From fa27b1cb629f1e5030f6174e4366575531a0321f Mon Sep 17 00:00:00 2001 From: vllm-omni-review-bot Date: Wed, 16 Sep 2026 15:32:45 +0800 Subject: [PATCH] knowledge: daily intake batch 2026-09-16 --- knowledge/repos/vllm-omni/ci/rules-test-fixtures.md | 9 ++++++++- .../vllm-omni/components/diffusion/rules-attention.md | 10 +++++++++- .../components/diffusion/rules-component-lifecycle.md | 10 +++++++++- .../model-executor/rules-platform-backends.md | 9 ++++++++- .../components/serving/rules-request-input.md | 9 ++++++++- .../repos/vllm-omni/models/mammoth-moda2/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/models/qwen-omni/rules.md | 10 +++++++++- knowledge/repos/vllm-omni/models/qwen3-tts/rules.md | 10 +++++++++- 8 files changed, 68 insertions(+), 8 deletions(-) diff --git a/knowledge/repos/vllm-omni/ci/rules-test-fixtures.md b/knowledge/repos/vllm-omni/ci/rules-test-fixtures.md index f54e75a4..eef057be 100644 --- a/knowledge/repos/vllm-omni/ci/rules-test-fixtures.md +++ b/knowledge/repos/vllm-omni/ci/rules-test-fixtures.md @@ -1,7 +1,7 @@ --- title: "CI 并行测试与 engine fixture 合同" created: 2026-08-23 -updated: 2026-09-10 +updated: 2026-09-16 type: rule tags: [vllm-omni, ci] sources: ["PR #3422", "PR #5074", "PR #5255", "PR #5310", "PR #5402", "PR #5524", "PR #5543", "PR #5670", "PR #5713", "PR #5780", "PR #5823", "PR #5836", "PR #5957", "PR #5976", docker/Dockerfile.ci, docker/Dockerfile.xpu, .buildkite/intel/scripts/run-xpu-test.sh, .buildkite/cuda/test-merge.yml, .buildkite/cuda/test-ready.yml, "PR #5845", "PR #5872", "PR #6008", "PR #6048", "PR #6056", "PR #6096", "PR #6102", "PR #6202", "PR #6208", "PR #6273", "PR #6293", "PR #6311", "PR #6339", "PR #6343", "PR #6468", "PR #6523", "PR #6613", "PR #6555", "PR #6650", .buildkite/common/scripts/run_cov_split.sh, pyproject.toml, tests/helpers/tests/test_mark.py, tools/pre_commit/check_test_marks.py, .buildkite/common/scripts/upload_pipeline.py, .buildkite/cuda/test-nightly.yml, .buildkite/cuda/test-weekly.yml, .buildkite/npu/test-npu-nightly.yml, .pre-commit-config.yaml, tests/helpers/clean.py, tests/helpers/client.py, tests/helpers/mark.py, tests/helpers/runtime.py, tests/helpers/stage_config.py, tests/buildkite/test_upload_pipeline.py, tests/dfx/perf/scripts/run_benchmark.py, tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/dfx/perf/tests/test_qwen_image_vllm_omni.json, tests/dfx/stability/, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_flux_kontext_expansion.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/test_qwen_image_expansion.py, tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py, tests/model_tests/diffusion/diff_model_builders.py, tests/model_tests/diffusion/model_settings.py, tests/model_tests/diffusion/test_alignment.py, tools/nightly/run_nightly_jobs.sh, tools/pre_commit/check_tts_adapter.py, tests/tools/test_check_tts_adapter.py, .buildkite/amd/scripts/bootstrap-amd-omni.sh, .buildkite/amd/test-amd-merge.yml, .buildkite/amd/test-amd-ready.yml, tests/diffusion/distributed/test_tensor_parallel.py, tests/diffusion/offloader/test_diffusion_layerwise_offload.py, "PR #6704", tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json, tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json, "PR #6743", "PR #6696", vllm_omni/benchmarks/metrics/metrics.py, vllm_omni/benchmarks/patch/patch.py, tests/benchmarks/metrics/test_metrics.py, tests/benchmarks/patch/test_patch.py, "PR #6674", docker/Dockerfile.npu, docker/Dockerfile.npu.a3, docker/Dockerfile.npu.ci, docker/Dockerfile.npu.ci.a3, "PR #6818", "PR #6830", "PR #6884", tests/diffusion/conftest.py, tests/diffusion/attention/test_flash_attn.py, tests/buildkite/test_amd_pipeline.py, tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py, "PR #6947", "PR #7225"] @@ -83,3 +83,10 @@ confidence: high - 强制：由成功重试路径设置 `asyncio.Event`（或等价信号），以有界 `wait_for` 等待恢复；在 `finally` 发 shutdown 并有界 await 任务。对 immediate 与 delayed 首次失败做参数化，使 delayed 超过旧观察窗口仍稳定。 - 禁止：`sleep(35ms)` 后断言 `calls >= 2`；依赖日志或调度速度填满重试窗口；漏关后台 task。 - 验收：immediate/delayed 均在超时前看到第二次成功调用；禁用 retry 时测试应 timeout 而非假绿。^[PR #7225] + +## OMNI-CI-2i — xdist online 测试不得用全局 GPU 占用做跨 worker 屏障 + +- 触发：修改 `tests/model_tests/diffusion` 等 online/xdist suite、`cleanup_test_environment`，或在并行 worker 上等待“整卡显存回落阈值”。 +- 强制：`cleanup_test_environment` 观察的是整卡占用，不是当前 worker 自有分配。在 `is_xdist_worker` 下，online diffusion 等并行 suite 必须禁用/no-op 该全局 cleanup，避免 server teardown 变成跨 worker 屏障；`OmniServer` 仍拆除自己的子进程树。非 xdist 运行保留更广的 cleanup。 +- 禁止：在 xdist 下等待 sibling worker 的分配消失才结束 teardown；把全局阈值等待说成单测资源隔离；或删除非 xdist 路径上仍需要的环境清理。 +- 验收：xdist worker 断言 cleanup 被 monkeypatch 为 no-op；非 worker 控制路径仍调用真实 cleanup；并行 online lane 不因全局 GPU wait 放大 wall time。^[PR #7534] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md index 7ebc0620..2d9f63f4 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md @@ -1,7 +1,7 @@ --- title: "Diffusion attention 规则" created: 2026-09-02 -updated: 2026-09-10 +updated: 2026-09-16 type: rule tags: [vllm-omni, components, diffusion] sources: ["PR #5543", "PR #5866", "PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", "PR #6037", "PR #6518", "PR #6563", "PR #6724", "PR #6909", docs/design/feature/skip_softmax.md, docs/user_guide/diffusion/attention_backends.md, docs/user_guide/diffusion/attention_backends/trtllm.md, docs/user_guide/diffusion/attention_backends/fastvideo_vsa.md, docs/user_guide/diffusion/attention_backends/rainfusion.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/fastvideo_vsa.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/diffusion_kv/paged_attention_adapter.py, vllm_omni/diffusion/models/minimax_h3/denoise_loop.py, vllm_omni/diffusion/models/minimax_h3/packed_sequence.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/cuda/platform.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_fastvideo_vsa.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_attention_config.py, tests/diffusion/attention/test_piecewise_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/diffusion_kv/test_paged_attention_adapter.py, tests/diffusion/models/minimax_h3/test_minimax_h3_packing.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py", "PR #6070", "vllm_omni/diffusion/attention/backends/cudnn_attn.py", "PR #5614", "PR #5194", "vllm_omni/diffusion/models/hidream_o1_image/hidream_o1_image_transformer.py", "vllm_omni/diffusion/models/hidream_o1_image/pipeline_hidream_o1_image.py", "PR #6181", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py", "PR #5717", "PR #6871"] @@ -217,3 +217,11 @@ confidence: high - 强制：仅当 `seq_lengths` 存在真实 padding（至少一档短于 `max(seq_lengths)`）时才物化 `(batch, max_len)` bool mask；单样本或全员等长 batch 必须返回 `None`，让 backend 直接走 dense path。all-true CUDA mask 会触发 `torch.any(~mask)` 的 device→host sync，不能当作“无害 no-op”。 - 禁止：无条件 `new_zeros` + 填 True；把 all-true device mask 交给 FlashAttention 只为“形状完整”；把省略 mask 说成改变了可变长 batch 的有效 token 集合。 - 验收：等长与单样本断言 helper/`forward` 得到 `None`；可变长断言 mask shape 与 True/False 边界；不得用一次端到端加速数字代替 sync 合同。^[PR #6871] + +## DIFF-1ai — Ring Attention 的 `valid_kv_length` 是全局前缀，且不得与 causal 并用 + +- 触发：修改 Ring Flash/PyTorch attention、`AttentionMetadata.valid_kv_length`、`ring_kv_block_valid_length`，或向 ring forward 传入前缀裁剪语义。 +- 强制：`valid_kv_length` 表示整条全局 K/V 序列的 contiguous 有效前缀（`0 < L <= block_size * world_size`）；每个 circulated block 按全局 `block_rank` 映射为本地 `max(0, min(block_size, L - block_start))`，再 `k/v[:, :local]`。`None` 表示整块有效。本地长度为 0 时跳过该 step 的 FA/LSE 更新，不得对空 K/V 调 kernel。 +- 强制：`causal=True` 且 `valid_kv_length is not None` 必须立即 `ValueError`。裁短 circulated K/V 会缩短 `seqlen_k` 而 query 仍是 padded 长度，FlashAttention causal 对角线 bottom-right 对齐会静默错位；caller 须先 unpad 再进 ring，或使用 `causal=False`。 +- 禁止：把 `valid_kv_length` 当成“本 rank 本地长度”直接切片；在 causal ring 路径上静默 trim；用 bool/非 int 冒充长度；或把 trim 当成慢路径而不是 unsupported 组合。 +- 验收：覆盖全局前缀落在首块/中块/末块、越界/非 int fail-closed、零长 block 跳过，以及 causal+valid_kv_length 拒绝；local dense backend 仍可直接消费同一全局前缀语义。^[PR #7047] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md b/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md index 8740242d..5b25eb30 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-component-lifecycle.md @@ -1,7 +1,7 @@ --- title: "Diffusion component lifecycle 规则" created: 2026-09-02 -updated: 2026-09-09 +updated: 2026-09-16 type: rule tags: [vllm-omni, components, diffusion] sources: ["PR #5720", "PR #5853", "PR #5882", "PR #5884", "PR #6486", "PR #6591", vllm_omni/diffusion/cache/base.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/cache/cachedit/runtime.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/models/interface.py, vllm_omni/diffusion/offloader/module_collector.py, vllm_omni/diffusion/offloader/startup.py, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/sched/interface.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/cache/test_cache_dit_request_runtime.py, tests/diffusion/models/sana_video/test_cache_offload.py, tests/diffusion/test_diffusion_model_runner.py, tests/diffusion/test_diffusion_scheduler.py, "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_recipes.py", "PR #6072", "PR #7078"] @@ -126,3 +126,11 @@ residency 留在模型 owner。规则入口与其他共享机制仍见 [Diffusio - 强制：凡 pipeline 用 “omitted vs explicit” provided 标志做 batch-local gate 的字段，都必须进入 request-mode 与 step-mode 的 sampling-params key；数值相等但 provided 不同的请求不得合批。当前合同要求 `guidance_scale_2_provided` 与数值 `guidance_scale_2` 一并参与 key。 - 禁止：只比较 auto-fill 后的数值；假设同数值即同语义；只更新 request scheduler 而漏掉 step scheduler；把 H3 等 single-request guard 当作可合批模型的例外模板。 - 验收：用真实 key builder 覆盖 omitted `guidance_scale_2` 与显式同值、数值不同，以及两套 scheduler 的 key 分离；并断言 pipeline 首请求 gate 不会跨不同 provided 语义共享 batch。^[PR #7078] + +## DIFF-2ag — pipeline 自管 staging 的组件不得进入 always-resident discovery 列表 + +- 触发：修改 `_vae_modules` / `ModuleDiscovery`、layerwise/HSDP/model-level offload 的 component 选择，或 pipeline 自己在 encode/decode 边界调用 `load_to_device`/`offload_to_cpu`。 +- 强制：`SupportsComponentDiscovery` 的 `_vae_modules` 语义是“VAE(s) always on GPU”。凡由 pipeline 自行互斥 staging、且不能交给 offloader/loader 搬迁的 VAE，必须从 discovery 列表省略（声明为空），使 layerwise/HSDP/model-level 路径都看不到它们；需要保留的 `pipeline.vae` 别名只服务 registry/patch，不算 discovery 声明。 +- 强制：因此退出 discovery 的组件，model-level CPU offload 入口不得再从 discovery 推导 stage 集，必须显式列出仍由 model-level 管理的 encoder/VAE 等；on-demand offload plan 同步丢掉已省略组件。 +- 禁止：一边在 pipeline 里互斥 staging，一边把同一 VAE 放进 `_vae_modules` 让 offloader/loader 再拉回 device；或用后端特殊分支掩盖 discovery 合同冲突。 +- 验收：discovery/offload plan 断言 pipeline-staged VAE 不出现；其余仍声明 VAE 的 pipeline 合同不变；model-level 显式 stage 列表仍覆盖应有组件。^[PR #7047] diff --git a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md index 83679869..b3d5cac1 100644 --- a/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md +++ b/knowledge/repos/vllm-omni/components/model-executor/rules-platform-backends.md @@ -1,7 +1,7 @@ --- title: "平台后端合同" created: 2026-09-04 -updated: 2026-09-05 +updated: 2026-09-16 type: rule tags: [vllm-omni, components, model-executor] sources: ["PR #5886", "PR #6061", "PR #6096", vllm_omni/platforms/, "PR #5604", "PR #6293", "PR #5571", "vllm_omni/platforms/xpu/platform.py", "PR #5569", "vllm_omni/platforms/xpu/utils.py", "PR #5048", "PR #6350", "PR #6102", "PR #6563", "PR #6054", vllm_omni/platforms/npu/platform.py, tests/platforms/npu/test_diffusion_platform.py, tests/platforms/npu/test_diffusion_attn_backend_selector.py, "PR #6674", vllm_omni/platforms/npu/worker/npu_ar_model_runner.py, vllm_omni/platforms/npu/worker/npu_generation_model_runner.py, vllm_omni/platforms/npu/worker/npu_model_runner.py] @@ -114,3 +114,10 @@ confidence: high state；只在 start 处 gate、却无条件同步/导出，或将 gate 本身当作 NPU 性能结果。 - 验收：AR/generation 都覆盖 enabled+timed、disabled+timed、enabled+not-timed 及 per-output disable， 断言 start/sync/export 一致。PR #6674 没有这些专项测试，属于后续验收要求。^[PR #6674] + +## EXEC-10f — AR runner 在 async PP 下必须配对 sampled-token broadcast 的 send + +- 触发：修改 `GPUARModelRunner.sample_tokens`、async scheduling + `pipeline_parallel_size > 1`，或继承/覆盖上游 `_pp_receive_prev_sampled_token_ids_to_input_batch` / `_pp_broadcast_prev_sampled_token_ids`。 +- 强制：非最终 PP rank 在 connector-only 分支 posted 的 async broadcast receive，必须由最终 rank 在 `_update_states_after_model_execute` 之后立刻执行匹配 send。守卫与上游一致：`use_async_scheduling` 且 `not broadcast_pp_output` 且 `pp.world_size > 1` 且 `pp.is_last_rank`。 +- 禁止：只继承 receive 而不补 send；在 `broadcast_pp_output=True`（external_launcher 已广播 logits/输出）时重复发送 sampled ids；把死锁归因于必须关闭 Thinker `async_scheduling` 而不修复配对集体通信。 +- 验收：ledger 或真实多进程 PP group 证明非最终 rank 的 receive 仅因最终 rank 的 Omni send 完成；PP1 与 `broadcast_pp_output` 路径不变。缺 send 时不得把“设 `async_scheduling: false`”当作长期合同。^[PR #7393] diff --git a/knowledge/repos/vllm-omni/components/serving/rules-request-input.md b/knowledge/repos/vllm-omni/components/serving/rules-request-input.md index 495c8659..1112c1e5 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-request-input.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-request-input.md @@ -1,7 +1,7 @@ --- title: "请求输入合同" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-16 type: rule tags: [vllm-omni, components, serving] sources: ["PR #3805", "PR #5374", "PR #5885", "PR #6598", vllm_omni/data_entry_keys.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/, vllm_omni/entrypoints/omni_base.py, vllm_omni/engine/orchestrator.py, vllm_omni/inputs/, tests/engine/test_async_omni_engine_input.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_invalid_audio_speech.py, tests/entrypoints/openai_api/test_serving_speech.py, "PR #5181", "PR #6182"] @@ -139,3 +139,10 @@ engine 生命周期见 [engine 生命周期规则](rules-engine-lifecycle.md)； - 强制：transform 只操作 Stage-0 copy；原始 prompt 保留给 downstream，并在 transform 前后保持同一 global request ID。只把已处理的 metadata 合并回原始视图，先移除 transform-owned stale keys。临时目录只通过内部 `REQUEST_ARTIFACT_DIRS_KEY` 交给 request state；preprocess、companion build、enqueue 前失败立即回收，admit 后由 orchestrator 在所有 terminal cleanup 路径回收，且内部 key 不得进入 stage payload。 - 禁止：把 transformed prompt 当作 downstream 原始媒体、让旧 prepared descriptor 跨请求复用、在 ownership 已交给 orchestrator 后由 frontend 提前删除，或让异常路径泄漏转码目录。 - 验收：覆盖 transform replacement/copy、request ID 与 metadata merge、成功 terminal/abort、preprocess/companion/enqueue 异常以及无 artifact control；断言 downstream 看见原始媒体+允许的 processed meta，目录恰好由当前 owner 回收，内部 key 未传输。^[PR #5885] + +## SERV-4r — pipeline 要求的 `stop_token_ids` 必须与 caller/deploy 停止集相加去重 + +- 触发：修改 `merge_sampling_constraints`、`OmniBase._apply_sampling_constraints`、pipeline `sampling_constraints.stop_token_ids`，或 deploy `default_sampling_params.stop_token_ids`。 +- 强制：标量约束仍由 pipeline 覆盖同名 caller/deploy 字段；惟 `stop_token_ids` 例外——先取 caller/deploy 列表，再追加 pipeline 要求的 ID，并以 `dict.fromkeys` 保序去重。legacy `yaml_extras` 与 structured config、以及 request 期重建 `SamplingParams`，必须共用同一 merge helper，且不 mutate 调用方对象。 +- 禁止：用整表 `constraints` 覆盖把 caller 自定义 stop 抹掉；只在 YAML merge 或只在 request apply 一侧实现相加；或让模型必需 terminator（如 MiniCPM-o TTS 边界）依赖重复 YAML/环境变量拷贝。 +- 验收：覆盖 deploy 已有 stop + pipeline 新增 stop 的并集、caller mapping/dataclass/msgspec 重建、immutability，以及至少一条真实 pipeline 的 Stage-0 必需 stop 进入全部 shipping deploy。^[PR #7463] diff --git a/knowledge/repos/vllm-omni/models/mammoth-moda2/rules.md b/knowledge/repos/vllm-omni/models/mammoth-moda2/rules.md index d4b5cca8..1419a7ff 100644 --- a/knowledge/repos/vllm-omni/models/mammoth-moda2/rules.md +++ b/knowledge/repos/vllm-omni/models/mammoth-moda2/rules.md @@ -1,7 +1,7 @@ --- title: "MammothModa2 规则" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-16 type: rule tags: [vllm-omni, models, model-executor, diffusion] sources: ["PR #6694", vllm_omni/model_extras/mammothmodal2_preview.py, vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py, vllm_omni/model_executor/stage_input_processors/mammoth_moda2.py, vllm_omni/diffusion/models/mammoth_moda2/pipeline_mammothmoda2_dit.py, tests/model_extras/test_model_extras.py, tests/e2e/offline_inference/test_mammoth_moda2_expansion.py] @@ -50,3 +50,10 @@ confidence: high empty visual condition、extra-body 参数透传和 AR-only text control。PR 作者报告的 54 个 CPU tests 与单次 L20X Preview T2I PNG smoke 只证明该提交环境的路径可运行；golden-image E2E 仍因既有 issue skip，不能据此声称质量、吞吐、Dev T2I 或跨硬件能力。^[PR #6694] + +## MAMMO-1c — AR→DiT `full_hidden_states` 必须保持源 FP16/BF16 直至 DiT 消费 + +- 触发：修改 `ar2dit`、`serialize_additional_information` / EngineCore payload，或 DiT `_split_ar_conditions` 的条件行选取与最终 `.to(device, dtype)`。 +- 强制：跨 stage 传输保持 AR 源 dtype（FP16/BF16）的 contiguous `full_hidden_states`；EngineCore/OmniSerializer 以 raw bytes 保留 bfloat16，不得为“过 numpy 边界”无条件 `.float()` 扩成 FP32。DiT 在选完 text/image 条件行之后，才在既有最终转换点落到 model device/dtype。 +- 禁止：在 bridge 或 condition split 处提前 host-side float32 加倍 staging 与 H2D；把精度优化扩成 request-end streaming / Layer-2 lifecycle 设计；或假定旧 FP32 wire 注释仍有效。 +- 验收：FP16/BF16 经 serialize/deserialize 往返 dtype 与数值不变；condition split 输出保持 transfer dtype 直至最终 cast；connector 级 BF16 round-trip 覆盖。^[PR #7102] diff --git a/knowledge/repos/vllm-omni/models/qwen-omni/rules.md b/knowledge/repos/vllm-omni/models/qwen-omni/rules.md index db1db3c9..ecd52a6b 100644 --- a/knowledge/repos/vllm-omni/models/qwen-omni/rules.md +++ b/knowledge/repos/vllm-omni/models/qwen-omni/rules.md @@ -1,7 +1,7 @@ --- title: "Qwen-Omni 规则" created: 2026-09-04 -updated: 2026-09-09 +updated: 2026-09-16 type: rule tags: [vllm-omni, models, qwen-omni] sources: ["PR #5687", "PR #6284", "PR #6449", "PR #4322", "PR #6748", "PR #6886", "PR #7019", vllm_omni/config/pipeline_registry.py, vllm_omni/deploy/qwen3_omni_moe.yaml, vllm_omni/deploy/qwen3_omni_moe_thinking.yaml, vllm_omni/engine/stage_init_utils.py, vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py, vllm_omni/model_executor/models/qwen3_omni/quantization.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py, vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py, vllm_omni/quantization/component_config.py, tests/config/test_config_factory.py, tests/diffusion/quantization/test_component_routing.py, tests/engine/test_stage_engine_args.py, tests/model_executor/models/qwen3_omni/test_qwen3_omni_quantization.py, "PR #7228"] @@ -86,3 +86,11 @@ confidence: high - 强制：构造 `inv_freq` 时优先 `config.rope_parameters["rope_theta"]`（若 mapping 存在），否则回退 top-level `rope_theta`，再默认 `10000.0`。Qwen3-Omni（Transformers 5.10+ 常把 checkpoint 的 `1_000_000` 只放在 nested 字段）与仍带 top-level 的 Qwen3-TTS/legacy 必须共用该顺序。 - 禁止：只读 top-level 导致静默回退 `10000`；把 nested 优先写成破坏 TTS top-level 兼容的唯一来源。 - 验收：参数化覆盖 nested-only、nested 优先于冲突 top-level、默认 `10000`，以及 TTS/legacy top-level（含删除 `rope_parameters`）重建正确 `inv_freq`。^[PR #7228] + +## QOMNI-1h — Thinker PP 必须把层捕获装进 IntermediateTensors 并让 runner 透传 + +- 触发：修改 Qwen3-Omni Thinker PP、`capture_layer_indices`/`PP_CAPTURE_PREFIX`、`make_empty_intermediate_tensors`，或 `GPUModelRunner.extract_multimodal_outputs` / `make_omni_output` 对非最终 PP rank 返回值的处理。 +- 强制：staged run 且存在 `accept_hidden_layer` 时，捕获层索引为 `[0, accept_layer]`。非最终 PP rank 在返回的 `IntermediateTensors` 中除 `hidden_states`/`residual` 外，必须附带 `capture_{idx}`；接收 rank 对 `idx < start_layer` 的捕获从 intermediate 读取并 `.clone()`，不得复用下一步会被覆盖的 receive buffer。 +- 强制：Thinker wrapper 在存在跨 rank 入站捕获时，用 `make_empty_intermediate_tensors_factory(["hidden_states", "residual", *incoming_captures], hidden_size)` 覆盖空张量工厂。runner 必须把 `IntermediateTensors` 当作合法 model output：不得强行 `make_omni_output`；`extract_multimodal_outputs` 透传；profiling/graph dummy 若需要 sampler 行，只取 `hidden_states` 张量字段。 +- 禁止：非最终 rank 只回传 hidden/residual 而丢掉 Talker 所需捕获；假定 stock vLLM 在非 staged 路径也要 return tuple；或因类型检查失败把 PP 路径改成同步-only workaround 却不修 handoff。 +- 验收：PP≥2 覆盖入站捕获 clone、非最终 rank 的 capture key 集合，以及 runner 对 `IntermediateTensors` 的 extract/dummy/make_omni_output 分支；同步与异步 PP 均不得以 `Invalid hidden states type` 启动失败。^[PR #7345] diff --git a/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md b/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md index 64e4db90..80d7ca98 100644 --- a/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md +++ b/knowledge/repos/vllm-omni/models/qwen3-tts/rules.md @@ -1,7 +1,7 @@ --- title: "Qwen3-TTS 规则" created: 2026-07-20 -updated: 2026-09-05 +updated: 2026-09-16 type: rule tags: [vllm-omni, models, serving, qwen-omni] sources: ["PR #5157", "PR #5202", "PR #5608", "PR #6001", "PR #6113", "PR #6523", "PR #6728", "PR #6861", vllm_omni/deploy/aura_omni.yaml, vllm_omni/deploy/qwen3_tts.yaml, vllm_omni/deploy/qwen3_tts_high_concurrency.yaml, vllm_omni/model_executor/models/aura_omni/pipeline.py, vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code2wav.py, vllm_omni/model_executor/models/qwen3_tts/prompt_embeds_builder.py, vllm_omni/model_executor/models/qwen3_tts/segmented_graph_wrapper.py, vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py, vllm_omni/model_executor/stage_input_processors/chunk_size_utils.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/serving_speech_stream.py, vllm_omni/entrypoints/openai/speech_usage.py, vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py, vllm_omni/model_executor/stage_input_processors/qwen3_tts.py, tests/e2e/online_serving/test_qwen3_tts_base.py, tests/e2e/online_serving/test_qwen3_tts_base_expansion.py, tests/entrypoints/openai_api/test_serving_speech.py, tests/entrypoints/openai_api/test_serving_speech_stream.py, tests/entrypoints/openai_api/test_tts_adapter.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_code2wav.py, tests/model_executor/models/qwen3_tts/test_qwen3_tts_incremental_decode.py, tests/model_executor/stage_input_processors/test_qwen3_tts_async_chunk.py, "PR #5048"] @@ -222,3 +222,11 @@ Qwen 家族入口见 [Qwen-Omni](../qwen-omni/_index.md)。 Ready CI 的 dummy-weight Base 即使到 192 tokens 仍可无 EOS 并返回 500，因而不是该 guard 的 Ready oracle；此事实不改变 runtime、YAML、EOS 或音质合同，也没有 real-weight merge pass log。 ^[PR #6861] ^[issue #6855] + +## Q3TTS-5a — 共享 code predictor 的 FP32 fallback 排除 NPU，且禁止外层 capture 时重放内层图 + +- 触发：修改 `qwen3_code_predictor` 的 FP16→FP32 stability fallback、NPUGraph/`_device_graphs` replay，或 Qwen3-TTS/Qwen3-Omni Talker 在 `enforce_eager=False` 下的外层 `talker_mtp` capture。 +- 强制：`use_fp32` 仅当 `input_dtype == float16` 且 `device.type not in ("cpu", "npu")`；CUDA/XPU/MUSA 等其它 accelerator 保留既有 FP32 upcast。Ascend fused RMSNorm 要求 activation 与 weight dtype 一致，不得在 NPU 上形成 FP32 activation + FP16 gamma。 +- 强制：每个 forward 采样一次 `is_npu_capturing = current_omni_platform.is_npu() and torch.npu.is_current_stream_capturing()`；存在 device graph 时，仅当**未**在外层 NPU capture 中才 `replay()`，capture 期间走普通/`compiled` forward 让外层图录制。无外层 capture 时仍使用内层 replay 快路径。 +- 禁止：把“非 CPU 即 FP32”重新扩到 NPU；在外层 capture 中嵌套 replay 内层 NPUGraph；或仅靠 stage 级 `PIECEWISE` workaround 声称已修复嵌套 replay 根因。 +- 验收：参数化断言 cpu/npu 不 upcast、cuda/xpu/musa upcast；`is_capturing` true/false 分别断言 skip/do replay。该合同属于共享 predictor，不证明每个 Qwen3 stage 已全图安全。^[PR #6639]