From 1e8e4abbfe8a148bc8777bd4794197b7b0e31fc2 Mon Sep 17 00:00:00 2001 From: vllm-omni-review-bot Date: Sat, 12 Sep 2026 15:23:17 +0800 Subject: [PATCH] knowledge: daily intake batch 2026-09-12 --- knowledge/repos/vllm-omni/ci/rules-amd.md | 9 ++++++++- knowledge/repos/vllm-omni/ci/rules.md | 9 ++++++++- .../components/model-executor/rules-platform-backends.md | 9 ++++++++- .../vllm-omni/components/serving/rules-request-input.md | 9 ++++++++- .../vllm-omni/components/serving/rules-stage-startup.md | 9 ++++++++- knowledge/repos/vllm-omni/models/ltx2/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/models/qwen-omni/rules.md | 9 ++++++++- 8 files changed, 64 insertions(+), 8 deletions(-) diff --git a/knowledge/repos/vllm-omni/ci/rules-amd.md b/knowledge/repos/vllm-omni/ci/rules-amd.md index 7422b541..f05490ff 100644 --- a/knowledge/repos/vllm-omni/ci/rules-amd.md +++ b/knowledge/repos/vllm-omni/ci/rules-amd.md @@ -1,7 +1,7 @@ --- title: "AMD/ROCm CI 规则" created: 2026-09-05 -updated: 2026-09-10 +updated: 2026-09-12 type: rule tags: [vllm-omni, ci] sources: ["PR #6704", "PR #6830", "PR #6884", .buildkite/amd/, tests/helpers/clean.py, tests/helpers/stage_config.py, tests/buildkite/test_amd_pipeline.py, tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py, "PR #7234"] @@ -42,3 +42,10 @@ confidence: high - 强制：`mi300_1` 一类单卡 model job 的 pytest marker 必须 `not (cards_2 or … or cards_8)`，同时保留无 `cards_1` 的 legacy 单卡用例；真正需要双卡的用例（如 LTX2 Ulysses parity）改到已有双卡 lane，并声明 `rocm` 资源与 `device_count >= world_size` 早失败。 - 禁止：让 `cards_2+` 测试在单卡 worker 上 spawn rank1→GPU1 导致 `invalid device ordinal`；用邻近 green shard 宣称 multi-GPU routing 已修好。 - 验收：pipeline argv/collection 断言单卡 job 排除 multi-card markers、双卡 job 收集目标文件；硬件 marker helper 覆盖 ROCm 声明。^[PR #7234] + +## OMNI-CI-2i — AMD bootstrap 必须按 ready/merge-test 标签选择 L2/L3 suite + +- 触发：修改 `.buildkite/amd` bootstrap、`select_test_suites.py`、AMD PR label 路由，或 skip-ci 对 AMD suite 的过滤。 +- 强制：`ready` 选 L2（ready suite），`merge-test` 选 L3（merge suite）；两标签同时存在时合并多 suite 且共享一次 image build；`DEBUG_TEST_YAML` 优先；main 继续 L3；无 tier 标签的 PR 可保留 legacy ready fallback。PR labels 精确匹配且失败时 fail closed；skip-ci 必须对已选 L2/L3 独立过滤。 +- 禁止：凡 PR 一律上传 ready suite；用子串匹配 labels；在仓库侧假装已改变 Buildkite 外部 trigger 条件；把 `nightly-test` 当成已有 AMD L4 覆盖。 +- 验收：selector/bootstrap 单测覆盖 ready-only、merge-only、both、debug override、main、label fetch failure 与 per-suite skip-ci；日志报告实际 `TEST_SPECS`。^[PR #6966] diff --git a/knowledge/repos/vllm-omni/ci/rules.md b/knowledge/repos/vllm-omni/ci/rules.md index 78af0b42..4f88fb21 100644 --- a/knowledge/repos/vllm-omni/ci/rules.md +++ b/knowledge/repos/vllm-omni/ci/rules.md @@ -1,7 +1,7 @@ --- title: "vLLM-Omni CI 规则" created: 2026-08-23 -updated: 2026-09-08 +updated: 2026-09-12 type: rule tags: [vllm-omni, ci] sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", "PR #6613", "PR #6555", "PR #6650", .buildkite/common/scripts/run_cov_split.sh, pyproject.toml, tests/helpers/tests/test_mark.py, tools/pre_commit/check_test_marks.py, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py, .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, "PR #6704", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743", "PR #6696", vllm_omni/benchmarks/metrics/metrics.py, vllm_omni/benchmarks/patch/patch.py, tests/benchmarks/metrics/test_metrics.py, tests/benchmarks/patch/test_patch.py, "PR #6674", docker/Dockerfile.npu, docker/Dockerfile.npu.a3, docker/Dockerfile.npu.ci, docker/Dockerfile.npu.ci.a3, "PR #6818", "PR #6830", "PR #6884", tests/diffusion/conftest.py, tests/diffusion/attention/test_flash_attn.py, tests/buildkite/test_amd_pipeline.py, tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py, "PR #6947", "PR #7118"] @@ -271,3 +271,10 @@ PR 报告的 collection=1 和 local two-card B300 184.97 s 是该精确 workload - 强制：preset expansion tests 必须同时断言 pod env 中存在 alias、不再存在裸 `HF_TOKEN`，并验证 L4/H100/B200 全部 CUDA mirror preset 的 commands 都 prepend 同一恢复语句；非 CUDA preset 不得无故注入该 wrapper。 - 禁止：依赖 Buildkite/Kubernetes 的 env merge 顺序碰巧正确；只修一个 GPU family；或把本地 shell/export 成功外推为 secret 已进入 pod env。 - 验收：render 后的每个 CUDA mirror step 都保留 secret alias 和 runtime export，scheduled build 的同名 `HF_TOKEN` 不再导致 gated-model jobs 401；NPU steps 的 env 与 commands 保持不变。^[PR #7118] + +## OMNI-CI-1g — PR 上传的 CUDA/NPU 任务必须经 registry preset 做 diff 过滤 + +- 触发：修改 `ci_source_file_dependencies.yml`、`upload_pipeline.py` 的 source filter、或 CUDA L2–L5 / NPU L4 步骤的 `source_file_dependencies`。 +- 强制：步骤上的 key（或 key 列表）由 registry 展开为 path prefix；仅当变更文件命中 prefix 才保留该步，随后剥离该字段。无 key 的步骤始终保留；`--all`/`--e2e` 与 `BUILDKITE_BRANCH=main` 关闭过滤。模型业务路径用 YAML anchor，job key（`*_function`/`_perf`/…）别名 anchor 并显式列出该 job 脚本；pytest 路径写在 key 上，不从 `commands` 推断。 +- 禁止：PR label 上传整条 nightly/weekly/ready/merge 清单；把未注册 key 留给 Buildkite；用旧的模型族 label 代替 `nightly-test` 等有效触发。 +- 验收：uploader 单测覆盖 key 展开、无关 diff 丢弃、无 key 保留、doc 组合路径，以及 pipeline 引用的 key 均已注册。^[PR #6597] diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md index 83679869..faceb5cf 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md @@ -1,7 +1,7 @@ --- title: "平台后端合同" created: 2026-09-04 -updated: 2026-09-05 +updated: 2026-09-12 type: rule tags: [vllm-omni, components, model-executor] sources: ["PR #5886", "PR #6061", "PR #6096", vllm_omni/platforms/, "PR #5604", "PR #6293", "PR #5571", "vllm_omni/platforms/xpu/platform.py", "PR #5569", "vllm_omni/platforms/xpu/utils.py", "PR #5048", "PR #6350", "PR #6102", "PR #6563", "PR #6054", vllm_omni/platforms/npu/platform.py, tests/platforms/npu/test_diffusion_platform.py, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6674", vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_generation_model_runner.py, vllm_omni/platforms/npu/worker/npu_model_runner.py] @@ -114,3 +114,10 @@ confidence: high state；只在 start 处 gate、却无条件同步/导出，或将 gate 本身当作 NPU 性能结果。 - 验收：AR/generation 都覆盖 enabled+timed、disabled+timed、enabled+not-timed 及 per-output disable， 断言 start/sync/export 一致。PR #6674 没有这些专项测试，属于后续验收要求。^[PR #6674] + +## EXEC-10f — XPU W8A16 FP8 linear 必须 flatten-then-reshape 恢复 N-D 输出 + +- 触发：修改 `vllm_omni/platforms/xpu/patch.py`、`XPUOmniPlatform` 初始化补丁，或 XPU 上 diffusion `--quantization fp8` 的 ScaledMM 路径。 +- 强制：对 `XPUW8A16FP8LinearKernel.apply_weights` 在平台 init 幂等 patch：先按 `(*x.shape[:-1], out_features)` 记录输出形，把 activation reshape 为 2-D 再调用原实现，最后 `.view` 回 N-D。与其他 ScaledMM/W8A8 合同一致；kernel 不可用时 skip。 +- 禁止：把 3-D diffusion activation 直接交给 fake 恒返回 2-D 的 `fp8_gemm_w8a16`；把该 patch 宣称为 LLM 2-D 路径修复或上游永久替代（upstream 修好后应变 no-op）。 +- 验收：XPU 上 FLUX/Qwen-Image 等 FP8 diffusion 能完成 dummy warmup 与生成；2-D LLM 路径不受影响。^[PR #7301] diff --git a/knowledge/repos/vllm-omni/components/serving/rules-request-input.md b/knowledge/repos/vllm-omni/components/serving/rules-request-input.md index 495c8659..93d40990 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-request-input.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-request-input.md @@ -1,7 +1,7 @@ --- title: "请求输入合同" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-12 type: rule tags: [vllm-omni, components, serving] sources: ["PR #3805", "PR #5374", "PR #5885", "PR #6598", vllm_omni/data_entry_keys.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/, vllm_omni/entrypoints/omni_base.py, vllm_omni/engine/orchestrator.py, vllm_omni/inputs/, tests/engine/test_async_omni_engine_input.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_invalid_audio_speech.py, tests/entrypoints/openai_api/test_serving_speech.py, "PR #5181", "PR #6182"] @@ -139,3 +139,10 @@ engine 生命周期见 [engine 生命周期规则](rules-engine-lifecycle.md)； - 强制：transform 只操作 Stage-0 copy；原始 prompt 保留给 downstream，并在 transform 前后保持同一 global request ID。只把已处理的 metadata 合并回原始视图，先移除 transform-owned stale keys。临时目录只通过内部 `REQUEST_ARTIFACT_DIRS_KEY` 交给 request state；preprocess、companion build、enqueue 前失败立即回收，admit 后由 orchestrator 在所有 terminal cleanup 路径回收，且内部 key 不得进入 stage payload。 - 禁止：把 transformed prompt 当作 downstream 原始媒体、让旧 prepared descriptor 跨请求复用、在 ownership 已交给 orchestrator 后由 frontend 提前删除，或让异常路径泄漏转码目录。 - 验收：覆盖 transform replacement/copy、request ID 与 metadata merge、成功 terminal/abort、preprocess/companion/enqueue 异常以及无 artifact control；断言 downstream 看见原始媒体+允许的 processed meta，目录恰好由当前 owner 回收，内部 key 未传输。^[PR #5885] + +## SERV-4r — `/v1/audio/generate` 必须在协议层拒绝空 prompt 与越界采样字段 + +- 触发：修改 `OpenAICreateAudioGenerateRequest`、audio generate invalid-param 可靠性测试，或该入口的 prompt/采样字段限界。 +- 强制：`input` 以 field validator 拒绝空串与纯空白；`audio_length` 要求 `gt=0`；`guidance_scale` 限界 `ge=0, le=1000`；`num_inference_steps` 限界 `ge=1, le=1000`。非法值必须在 entrypoint Pydantic 校验失败，不得拖到 engine 超时或 HTTP 200。 +- 禁止：依赖 engine-level 检查代替协议拒绝；把越界 steps/guidance 留在 `_INT64_MAX` 一类无业务上界；用 skip mark 掩盖未接线的 invalid-param case。 +- 验收：空/空白 input、负/零 `audio_length`、负/过大 guidance、零/负/过大 steps 均返回字段定位的 4xx，并与 invalid-param 测试期望一致。^[PR #4741] diff --git a/knowledge/repos/vllm-omni/components/serving/rules-stage-startup.md b/knowledge/repos/vllm-omni/components/serving/rules-stage-startup.md index 2e5eb008..26b6bf71 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-stage-startup.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-stage-startup.md @@ -1,7 +1,7 @@ --- title: "Stage 启动与设备布局规则" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-12 type: rule tags: [vllm-omni, components, serving] sources: ["PR #6050", "PR #5445", "Issue #5003", "PR #5742", vllm_omni/engine/stage_init_utils.py, tests/engine/test_stage_device_layout.py] @@ -33,3 +33,10 @@ confidence: high - 强制：对声明 `runtime.devices` 的每个 LLM stage，在 `create_engine_config` 或 worker/executor 创建前，以同一公式验证并拆分设备：每个本地 replica 需要 `tensor_parallel_size × data_parallel_size_local × pipeline_parallel_size`；`data_parallel_size_local` 未设置时才回退到全局 DP，值为 `0` 的 head process 不做本地设备验证。显式列表可为单 replica template（恰为每副本大小）或完整 pool（该大小 × `num_replicas`）；未声明 `devices` 时保留 launcher 分配语义。仅当去掉 TP 后设备数恰好有效，错误才说明 top-level TP 会广播到全部 stage，并要求在每个 stage override 中一并设置 TP 与 `devices`；其他 TP/DP/PP/replica 不匹配给通用维度说明。 - 禁止：按集群全局 DP 校验本地 `runtime.devices`；让 guard 与 replica splitter 使用不同的每副本宽度；在 engine config/worker 启动后才暴露 `local rank ... out of bounds`；把 PP 或其他布局错误一律归因于 top-level TP；因为没有显式 devices 而拒绝 vLLM 的设备分配。 - 验收：覆盖 TP 广播的单卡 stage 在 engine-config/executor 前失败、匹配 TP/device 通过、未声明 devices 跳过、local-DP 小于 global-DP 与 local-DP=0、PP/local-DP 加多副本的 template 和 full-pool 拆分，以及 PP-only mismatch 不含 TP 专属 workaround。^[Issue #5003] ^[PR #5742] + +## SERV-8b — 全局 replica launch lock 只覆盖 spawn 与 runtime.env，READY 等待必须释放 + +- 触发：修改 `StageRuntime._initialize_local_llm_replica`、`_replica_launch_lock`、`parallel_stage_init`，或多 GPU 多副本启动超时。 +- 强制：全局 launch lock 仅包裹进程 spawn 与 process-global `runtime.env` overlay；launch context 的 READY 等待必须在锁外执行，使不同 device group 可并行完成 load/profile/compile/capture。同 GPU 仍由既有 per-device exclusive lock 串行到 READY；`ExitStack` 管理 launch context 并传播 READY 失败清理。 +- 禁止：默认路径把 lock 持有到 READY（跨 GPU 串行化整段 init）；为加速而削弱同设备 EX lock；改变 timeout/deploy 阈值来掩盖串行启动。 +- 验收：不同设备并发启动回归必须在未改代码时失败、修复后通过；同设备顺序与 READY 失败后的 resource/device-lock 清理仍成立。^[PR #7292] diff --git a/knowledge/repos/vllm-omni/models/ltx2/rules.md b/knowledge/repos/vllm-omni/models/ltx2/rules.md index 6f2f149b..1c0c02e9 100644 --- a/knowledge/repos/vllm-omni/models/ltx2/rules.md +++ b/knowledge/repos/vllm-omni/models/ltx2/rules.md @@ -1,7 +1,7 @@ --- title: "LTX-2.5 DiffVAE 规则" created: 2026-09-04 -updated: 2026-09-05 +updated: 2026-09-12 type: rule tags: [vllm-omni, models, ltx2, diffusion] sources: ["PR #6189", "PR #7000", "PR #7020", recipes/LTX/LTX-2.5.md, requirements/common.txt, docs/user_guide/diffusion/attention_backends/huggingface_hub.md, vllm_omni/diffusion/models/ltx2/ltx2_components.py, vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py, vllm_omni/diffusion/models/ltx2/ltx2_diffusion_decoder.py, vllm_omni/diffusion/models/ltx2/ltx2_diffusion_decoder_distributed.py, vllm_omni/diffusion/models/ltx2/ltx2_latents.py, vllm_omni/diffusion/models/ltx2/ltx2_request.py, vllm_omni/diffusion/models/ltx2/ltx2_runtime.py, tests/diffusion/models/ltx2/test_ltx25_pipeline.py, tests/diffusion/models/ltx2/test_ltx2_output_cuda.py, tests/diffusion/models/ltx2/test_ltx2_pipeline.py, tests/diffusion/models/ltx2/test_ltx2_vae.py] @@ -71,3 +71,10 @@ confidence: high 对比。有效 `[0, 1]` 的 `do_normalize=False` 输入须与 unclipped legacy path 比较；另测 intentional out-of-range defensive clamp。FP32 presentation bytes 必须 exact，BF16/FP16 相对 legacy FP32 presentation 最多相差一个 uint8 level；官方 similarity 仅是本 PR 所报 LTX matrix 的限定证据。^[PR #7000] + +## LTX25-3 — DiffVAE pointwise fusion 与 TileLang FNA 必须按 SM 独立门禁 + +- 触发：修改 `resolve_ltx2_vae_operators`、fusion/FNA compute-capability allowlist，或 LTX DiffVAE 加速准入。 +- 强制：pointwise fusion 与 TileLang FNA 分两个 capability 集合独立判定；已验证的 SM100/SM103 可开自校验 fusion（首用 bit-exact，失败回退普通数学），FNA 在未单独验证前保持 SM90-only。不得用单一 allowlist 绑死两条路径。 +- 禁止：因 FNA 未验证而关闭已合格 SM 上的 fusion；或把 fusion 资格静默推广成 FNA/NATTEN schedule 已验证。 +- 验收：平台单测覆盖 SM90 fusion+FNA、SM100/103 fusion-only、以及更低/未列 SM 全关。^[PR #7350] diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index 907b76e3..1e627a4a 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -1,7 +1,7 @@ --- title: "MiniCPM-o 4.5 规则" created: 2026-07-20 -updated: 2026-09-10 +updated: 2026-09-12 type: rule tags: [vllm-omni, models, model-executor] sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", "PR #6828", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/entrypoints/duplex/chat_fallback.py, vllm_omni/entrypoints/duplex/realtime_input.py, vllm_omni/entrypoints/duplex/session_runner.py, vllm_omni/entrypoints/duplex/serving.py, vllm_omni/entrypoints/duplex/vad.py, vllm_omni/model_executor/models/minicpmo_4_5/duplex/adapter.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_cfm_graph_capture_gating.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274", "PR #6346", "PR #6397", "PR #6406", "PR #6458", "PR #6587", "PR #6619", "PR #6757", "PR #6529", "PR #6772", vllm_omni/entrypoints/duplex/protocol.py, "PR #7071", "PR #5730"] @@ -300,3 +300,10 @@ confidence: high - 强制：embedding items 的 `modality` 必须是 `"audio"`（不得从 image 类 copy 成 `"image"`）。单条 `audio_embeds` 张量为 `(s, h)` 时，placeholder 长度取 `len(single_audio_embeds)` / 前导维 `s`，再经 `get_audio_len_by_num_chunks`；不得 `sum(map(len, …))` 把 hidden size 累进 token 数。 - 禁止：以 `modality="image"` 注册 audio embedding；让 `(10, 4096)` 请求约 40960 个 placeholder 并撞 `max_model_len`。 - 验收：modality 断言为 audio；经真实 `_get_prompt_updates` 覆盖 1s/3s/30s 与不同 `h`，placeholder 只随 `s` 变化。^[PR #5730] + +## MCPMO-2g — MiniCPM-o structured content 必须无分隔符拼接，对齐 native omni chat + +- 触发：修改 MiniCPM-o 4.5 chat template、online serving `--chat-template`、image/audio/video 多模态 content parts，或 `minicpmo45_native.jinja` 打包。 +- 强制：structured content 按 native `chat(omni_mode=True)` 直接拼接 text/media parts，不插入额外换行/分隔符；显式 text 空白保留。模板须作为 package data 发布，online/recipe 显式启用并设 `chat-template-content-format=openai`；image→speech 一致性用例保持 image-first。 +- 禁止：依赖通用 string-content 渲染在 media 与 text 间加换行；只改测试不改 shipping template/package data。 +- 验收：CPU 模板回归覆盖 structured parts；online image-to-text-audio 一致性在目标硬件通过。^[PR #7344] diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/rules.md b/knowledge/repos/vllm-omni/models/qwen-omni/rules.md index db1db3c9..5630a44f 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/rules.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/rules.md @@ -1,7 +1,7 @@ --- title: "Qwen-Omni 规则" created: 2026-09-04 -updated: 2026-09-09 +updated: 2026-09-12 type: rule tags: [vllm-omni, models, qwen-omni] sources: ["PR #5687", "PR #6284", "PR #6449", "PR #4322", "PR #6748", "PR #6886", "PR #7019", vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/deploy/qwen3_omni_moe_thinking.yaml, vllm_omni/engine/stage_init_utils.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/quantization/component_config.py, tests/config/test_config_factory.py, tests/diffusion/quantization/test_component_routing.py, tests/engine/test_stage_engine_args.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_quantization.py, "PR #7228"] @@ -86,3 +86,10 @@ confidence: high - 强制：构造 `inv_freq` 时优先 `config.rope_parameters["rope_theta"]`（若 mapping 存在），否则回退 top-level `rope_theta`，再默认 `10000.0`。Qwen3-Omni（Transformers 5.10+ 常把 checkpoint 的 `1_000_000` 只放在 nested 字段）与仍带 top-level 的 Qwen3-TTS/legacy 必须共用该顺序。 - 禁止：只读 top-level 导致静默回退 `10000`；把 nested 优先写成破坏 TTS top-level 兼容的唯一来源。 - 验收：参数化覆盖 nested-only、nested 优先于冲突 top-level、默认 `10000`，以及 TTS/legacy top-level（含删除 `rope_parameters`）重建正确 `inv_freq`。^[PR #7228] + +## QOMNI-1h — Qwen3-Omni 的 moe_backend 覆盖只认显式字段，字面 auto 不算隐式 + +- 触发：修改 `_maybe_set_qwen3_omni_moe_backend`、`OmniStageModelConfig` 的 `_TrackExplicitConfigFields`/`_omni_explicit_fields`，量化/NVFP4 deploy，或 stage engine-args 投影时机。 +- 强制：仅当 `moe_backend` 不在显式字段集合（typed 读 `_omni_explicit_fields`；dict 路径 key 缺失）时才默认写入 `triton`；deploy/CLI 显式写出的 `moe_backend: auto` 必须原样保留，交给 vLLM 量化 backend 选择。覆盖应在 stage projection 时带上 explicitness，不得仅靠 finalize 再猜字面值。 +- 禁止：用 `get("moe_backend", "auto") == "auto"` 把显式 `auto` 当成未设置；在 finalize 二次覆盖时丢弃 `_omni_explicit_fields`；只测 helper 不测 legacy+typed builder。 +- 验收：显式 `auto`、显式非 auto、未设置三路径分别经 legacy 与 typed builder；NVFP4 CI stage 0 最终为 `auto`，未量化默认 stage 仍为 `triton`。^[PR #7200]