From 3685632afe06ecff10a393f8643055f67633ccc3 Mon Sep 17 00:00:00 2001 From: vllm-omni-review-bot Date: Tue, 15 Sep 2026 15:30:12 +0800 Subject: [PATCH] knowledge: daily intake batch 2026-09-15 --- knowledge/repos/vllm-omni/ci/rules-amd.md | 9 ++++++++- .../components/diffusion/rules-attention.md | 9 ++++++++- .../diffusion/rules-platform-kernels.md | 9 ++++++++- .../vllm-omni/components/distributed/rules.md | 9 ++++++++- .../components/serving/rules-request-input.md | 16 +++++++++++++++- .../vllm-omni/models/lingbot-video/rules.md | 9 ++++++++- .../vllm-omni/models/minicpm-o-4-5/rules.md | 9 ++++++++- 7 files changed, 63 insertions(+), 7 deletions(-) diff --git a/knowledge/repos/vllm-omni/ci/rules-amd.md b/knowledge/repos/vllm-omni/ci/rules-amd.md index 7422b541..39241f0f 100644 --- a/knowledge/repos/vllm-omni/ci/rules-amd.md +++ b/knowledge/repos/vllm-omni/ci/rules-amd.md @@ -1,7 +1,7 @@ --- title: "AMD/ROCm CI 规则" created: 2026-09-05 -updated: 2026-09-10 +updated: 2026-09-15 type: rule tags: [vllm-omni, ci] sources: ["PR #6704", "PR #6830", "PR #6884", .buildkite/amd/, tests/helpers/clean.py, tests/helpers/stage_config.py, tests/buildkite/test_amd_pipeline.py, tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py, "PR #7234"] @@ -42,3 +42,10 @@ confidence: high - 强制：`mi300_1` 一类单卡 model job 的 pytest marker 必须 `not (cards_2 or … or cards_8)`，同时保留无 `cards_1` 的 legacy 单卡用例；真正需要双卡的用例（如 LTX2 Ulysses parity）改到已有双卡 lane，并声明 `rocm` 资源与 `device_count >= world_size` 早失败。 - 禁止：让 `cards_2+` 测试在单卡 worker 上 spawn rank1→GPU1 导致 `invalid device ordinal`；用邻近 green shard 宣称 multi-GPU routing 已修好。 - 验收：pipeline argv/collection 断言单卡 job 排除 multi-card markers、双卡 job 收集目标文件；硬件 marker helper 覆盖 ROCm 声明。^[PR #7234] + +## OMNI-CI-2i — AMD nightly suite 必须可显式选中且不受 L2/L3 skip-ci 误杀 + +- 触发：修改 AMD bootstrap/`select_test_suites`、`NIGHTLY_TESTS` YAML、`nightly-test` label，或 skip-ci 对 suite spec 的过滤。 +- 强制：`nightly` 映射 `NIGHTLY_TESTS:test-amd-nightly.yml`；`main+NIGHTLY=1` 或 PR `nightly-test` 选中它；可与 ready/merge 组合共享一次 image build。`NIGHTLY_TESTS:*` 在 skip-ci 过滤中原样保留，docs-only 也不得剥掉显式/scheduled nightly。burn-in 叶子保持 `NonBlocking` 直至另有 gate。 +- 禁止：把 `nightly-test` 当成无 suite 的噪声 label；用 L2/L3 diff gate 静默丢掉已选 nightly；把实验 nightly 阈值外推为 CUDA H100 基线。 +- 验收：渲染 `DEBUG_TEST_YAML=nightly`、`NIGHTLY=1`、组合 label 与 docs-only+nightly，断言 suite spec 与子 pipeline 叶子集合。^[PR #6978] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md index 7ebc0620..ea80fc77 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md @@ -1,7 +1,7 @@ --- title: "Diffusion attention 规则" created: 2026-09-02 -updated: 2026-09-10 +updated: 2026-09-15 type: rule tags: [vllm-omni, components, diffusion] sources: ["PR #5543", "PR #5866", "PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", "PR #6037", "PR #6518", "PR #6563", "PR #6724", "PR #6909", docs/design/feature/skip_softmax.md, docs/user_guide/diffusion/attention_backends.md, docs/user_guide/diffusion/attention_backends/trtllm.md, docs/user_guide/diffusion/attention_backends/fastvideo_vsa.md, docs/user_guide/diffusion/attention_backends/rainfusion.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/fastvideo_vsa.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/diffusion_kv/paged_attention_adapter.py, vllm_omni/diffusion/models/minimax_h3/denoise_loop.py, vllm_omni/diffusion/models/minimax_h3/packed_sequence.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/cuda/platform.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_fastvideo_vsa.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_attention_config.py, tests/diffusion/attention/test_piecewise_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/diffusion_kv/test_paged_attention_adapter.py, tests/diffusion/models/minimax_h3/test_minimax_h3_packing.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py", "PR #6070", "vllm_omni/diffusion/attention/backends/cudnn_attn.py", "PR #5614", "PR #5194", "vllm_omni/diffusion/models/hidream_o1_image/hidream_o1_image_transformer.py", "vllm_omni/diffusion/models/hidream_o1_image/pipeline_hidream_o1_image.py", "PR #6181", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py", "PR #5717", "PR #6871"] @@ -217,3 +217,10 @@ confidence: high - 强制：仅当 `seq_lengths` 存在真实 padding（至少一档短于 `max(seq_lengths)`）时才物化 `(batch, max_len)` bool mask；单样本或全员等长 batch 必须返回 `None`，让 backend 直接走 dense path。all-true CUDA mask 会触发 `torch.any(~mask)` 的 device→host sync，不能当作“无害 no-op”。 - 禁止：无条件 `new_zeros` + 填 True；把 all-true device mask 交给 FlashAttention 只为“形状完整”；把省略 mask 说成改变了可变长 batch 的有效 token 集合。 - 验收：等长与单样本断言 helper/`forward` 得到 `None`；可变长断言 mask shape 与 True/False 边界；不得用一次端到端加速数字代替 sync 合同。^[PR #6871] + +## DIFF-1aj — NPU dense FlashAttention 在 `causal=True` 时必须物化因果掩码 + +- 触发：修改 `FlashAttentionImpl.forward_fa_npu`、NPU MindIE-SD / `npu_fusion_attention` dense path，或 `self.causal` 语义。 +- 强制：`causal=True` 且无显式 mask 时，必须走 bottom-right 对齐的压缩/物化因果 mask（`Sq!=Skv` 时按 `key_pos <= query_pos + (Skv-Sq)`），不能把 `attn_mask=None` 交给非因果 dense op。显式 keep-mask 须与因果条件组合后 contiguous 下发；`causal=False` 保持原无 mask 行为。 +- 禁止：依赖 MindIE-SD 隐式因果参数；让因果请求静默变成双向 attention；只靠“推理未崩”证明因果语义。 +- 验收：CPU/mock 覆盖 `Sq==Skv`、更短/更长 query、显式 mask 组合与非因果 `None`；目标 NPU 用已知因果模型对照。^[PR #7324] diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-platform-kernels.md b/knowledge/repos/vllm-omni/components/diffusion/rules-platform-kernels.md index 9a7c5b07..5857cb20 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-platform-kernels.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-platform-kernels.md @@ -1,7 +1,7 @@ --- title: "Diffusion 平台 kernel 与设备合同" created: 2026-07-20 -updated: 2026-09-07 +updated: 2026-09-15 type: rule tags: [vllm-omni, components, diffusion] sources: ["PR #4341", "PR #5001", "PR #5087", "PR #5088", "PR #5136", "PR #5255", "PR #5344", "PR #5543", "PR #5720", "PR #5737", "PR #5764", "PR #5801", "PR #5802", "PR #5838", "PR #5839", "PR #5848", "PR #5872", "PR #5881", "PR #5896", "PR #5981", "PR #6094", "PR #6102", "PR #6279", "PR #6385", "PR #6445", "PR #6651", "PR #6722", "PR #5831", "PR #6989", vllm_omni/diffusion/attention/backends/flashinfer_attn.py, vllm_omni/diffusion/attention/backends/ring/ring_kernels.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/cache/cachedit/backend.py, vllm_omni/diffusion/data.py, vllm_omni/diffusion/distributed/hsdp.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/layers/norm.py, vllm_omni/diffusion/layers/rope.py, vllm_omni/diffusion/lora/manager.py, vllm_omni/diffusion/model_loader/diffusers_loader.py, vllm_omni/diffusion/model_metadata.py, vllm_omni/diffusion/offloader/, vllm_omni/diffusion/registry.py, vllm_omni/diffusion/worker/diffusion_model_runner.py, vllm_omni/diffusion/worker/diffusion_worker.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/quantization/component_config.py, vllm_omni/quantization/factory.py, tests/diffusion/attention/test_attention_sp.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/cache/test_cache_backends.py, tests/diffusion/layers/test_norm.py, tests/diffusion/layers/test_rope_broadcast.py, tests/diffusion/offloader/test_distributed_layerwise_backend.py, tests/diffusion/test_diffusion_config_propagation.py, tests/diffusion/test_multiproc_engine_concurrency.py, "PR #4755", "PR #5990", "vllm_omni/diffusion/layers/fused_qk_norm_rope.py", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", "tests/diffusion/layers/test_fused_qk_norm_rope.py", "PR #6165", "PR #5677", "vllm_omni/diffusion/quantization/hsdp_fp8.py", "tests/diffusion/quantization/test_hsdp_fp8.py", "PR #4845", "PR #6173", "PR #6070", "vllm_omni/diffusion/models/ltx2/ltx2_components.py", "vllm_omni/diffusion/model_loader/hub_prefetch.py", "PR #5910", "PR #5676", "PR #6125", "PR #6320", "PR #5877", "vllm_omni/diffusion/vllm_config.py", "PR #6283", "vllm_omni/diffusion/layers/activation.py", "PR #6281", "vllm_omni/diffusion/attention/ops/minimax_h3_modulation.py", "PR #6130", "PR #6364", "PR #6073", "PR #4820", vllm_omni/diffusion/attention/backends/fastvideo_vsa.py, "PR #6150", "PR #6410", tests/diffusion/layers/test_activation.py, tests/diffusion/layers/test_fused_qk_norm_rope_npu.py] @@ -40,3 +40,10 @@ confidence: high - 禁止：只用 `x.is_cpu` 作为 Triton 门禁；让 NPU、XPU、MUSA 或其他非 CUDA tensor 进入 raw Triton launch；认为传入 row stride 就足以支持非连续列；对错误布局静默读写；或用 mock/wiring 结果宣称真实 kernel 数值 parity。 - 验收：CPU/mock 覆盖 CUDA、NPU、XPU、MUSA、unsupported dtype/shape、空行和 contiguous/strided 输入，断言非 CUDA 路径不启动 Triton 且 NPU 保留 native RMSNorm；CUDA 目标硬件以 FP32 reference 核对值、dtype 和容差，并单独记录 kernel 数量/延迟。当前 PR 未新增该模块测试且 NPU CI 不覆盖 H3，在补齐前不得称为生产级跨平台支持。^[PR #6281] + +## DIFF-1ai — fused_qk_norm_rope 的 interleaved 模式与 token gate 必须分模式闭合 + +- 触发：修改共享 `fused_qk_norm_rope`、packed RoPE table、`interleaved` 分派、Boogu/MiniMax consumer，或 `VLLM_OMNI_FUSED_QK_NORM_ROPE_MIN_TOKENS`。 +- 强制：`interleaved=False` 保持 half-split `(d, d+rotary_dim/2)` 与既有 MiniMax-H3 fused geometry（`head_dim==128`、`rotary_dim==96`）；`interleaved=True` 使用相邻对 `(2i, 2i+1)`，CUDA Triton 仅在 even `rotary_dim <= head_dim <= 256` 时融合，否则 eager。consumer 的 packed `[cos|sin]` table 必须匹配所选 pairing，不得把 half-split 与 adjacent-pair frequency 混用。融合路径的 token 门槛由 op 级 env 覆盖；短于门槛时整条 eager 链必须 bit-exact。新增 env 必须进入 public inventory 与 docs。 +- 禁止：把 interleaved 默认塞进 MiniMax half-split path；无门槛强制融合；只改一个 consumer 却不更新共享 op 合同或 inventory。 +- 验收：分别覆盖 half-split 与 interleaved 的 fused/eager 数值、geometry 拒绝、门槛上下切换，以及 env 缺省/覆盖；Boogu 集成断言 table 去重与 `[B*S, D]` 复用。^[PR #6982] diff --git a/knowledge/repos/vllm-omni/components/distributed/rules.md b/knowledge/repos/vllm-omni/components/distributed/rules.md index d693a2d0..ba538b74 100644 --- a/knowledge/repos/vllm-omni/components/distributed/rules.md +++ b/knowledge/repos/vllm-omni/components/distributed/rules.md @@ -1,7 +1,7 @@ --- title: "Distributed 传输规则" created: 2026-08-05 -updated: 2026-09-08 +updated: 2026-09-15 type: rule tags: [vllm-omni, components, distributed] sources: ["PR #5744", "PR #5976", "PR #6001", "PR #6089", "PR #6834", vllm_omni/diffusion/distributed/parallel_state.py, tests/diffusion/distributed/test_expert_parallel_layout.py, vllm_omni/distributed/omni_connectors/adapter.py, vllm_omni/distributed/omni_connectors/kv_transfer_manager.py, vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py, vllm_omni/distributed/omni_connectors/transfer_adapter/base.py, vllm_omni/worker/omni_connector_model_runner_mixin.py, tests/distributed/omni_connectors/test_kv_recv_tp_consensus.py, tests/distributed/omni_connectors/test_chunk_transfer_adapter.py, tests/worker/test_omni_connector_mixin.py, "PR #5146", "PR #6021", "PR #6033", "PR #6360", "PR #6406", "PR #6626", "PR #6529", "PR #7136"] @@ -142,3 +142,10 @@ confidence: high - 强制：CUDA Mooncake 依赖必须钉在包含 acknowledged TCP writes 的版本（`0.3.12` 起）；`batch_transfer_sync_write` 返回成功后，destination 内存必须已更新到可读状态，不能只表示本地 send 入队。回归必须在 receiver 被挂起时证明 completion 不会提前返回，恢复后再断言目的端字节一致。 - 禁止：降级到无 ACK 的 TCP 实现却不更新测试合同；把 RDMA 路径的 completion 语义外推到 TCP；或在 completion 前启动依赖 destination 内容的零拷贝消费。 - 验收：强制 TCP 的挂起-恢复测试覆盖「停止期间不完成、恢复后成功且字节匹配」；依赖声明与该测试同步变更。^[PR #7136] + +## DIST-1l — NIXL connector 超时后必须保留 native 资源直到终端态 + +- 触发：新增/修改 `NixlConnector`、NIXL READ、ZMQ claim/`XFER_DONE`、lease 或 receive timeout。 +- 强制：NIXL 无远端 READ 取消时，超时/放弃后仍保留 buffer、registration、descriptor、transfer handle 与 remote agent，直到 native 终端态；`close()` 等待后再释放。源 registration 有界 lease（`lease_seconds` / `VLLM_OMNI_NIXL_LEASE_S`）并由 reaper 回收。跨 stage 请求须把 producer endpoint/`payload_sender_info` 经 scheduler handle 传到 consumer。 +- 禁止：超时后立刻 deregister 或丢弃仍可能被 DMA 写入的内存；把 vLLM P/D block-table NIXL 接口当作 Omni 任意 payload connector；依赖未声明的固定 `zmq_port` 却不经 resolver。 +- 验收：覆盖超时保留、lease 回收、metadata claim/ack，以及 `OmniChunkRecvHandle.payload_sender_info` 贯通；缺 NIXL 依赖时 lazy import 为 None。^[PR #6093] diff --git a/knowledge/repos/vllm-omni/components/serving/rules-request-input.md b/knowledge/repos/vllm-omni/components/serving/rules-request-input.md index 495c8659..d81a8b71 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-request-input.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-request-input.md @@ -1,7 +1,7 @@ --- title: "请求输入合同" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-15 type: rule tags: [vllm-omni, components, serving] sources: ["PR #3805", "PR #5374", "PR #5885", "PR #6598", vllm_omni/data_entry_keys.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/, vllm_omni/entrypoints/omni_base.py, vllm_omni/engine/orchestrator.py, vllm_omni/inputs/, tests/engine/test_async_omni_engine_input.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_invalid_audio_speech.py, tests/entrypoints/openai_api/test_serving_speech.py, "PR #5181", "PR #6182"] @@ -139,3 +139,17 @@ engine 生命周期见 [engine 生命周期规则](rules-engine-lifecycle.md)； - 强制：transform 只操作 Stage-0 copy；原始 prompt 保留给 downstream，并在 transform 前后保持同一 global request ID。只把已处理的 metadata 合并回原始视图，先移除 transform-owned stale keys。临时目录只通过内部 `REQUEST_ARTIFACT_DIRS_KEY` 交给 request state；preprocess、companion build、enqueue 前失败立即回收，admit 后由 orchestrator 在所有 terminal cleanup 路径回收，且内部 key 不得进入 stage payload。 - 禁止：把 transformed prompt 当作 downstream 原始媒体、让旧 prepared descriptor 跨请求复用、在 ownership 已交给 orchestrator 后由 frontend 提前删除，或让异常路径泄漏转码目录。 - 验收：覆盖 transform replacement/copy、request ID 与 metadata merge、成功 terminal/abort、preprocess/companion/enqueue 异常以及无 artifact control；断言 downstream 看见原始媒体+允许的 processed meta，目录恰好由当前 owner 回收，内部 key 未传输。^[PR #5885] + +## SERV-4r — 图片 generations 与 edits 的 `output_compression` 必须同一闭环 + +- 触发：修改 `/v1/images/generations` 或 edits 的协议字段、handler 到 `encode_image_base64_with_compression` 的转发。 +- 强制：`ImageGenerationRequest` 声明 `output_compression`（0–100，默认 100）并原样传入 encoder；PNG 映射为 compress_level（100→0，1→9），jpeg/webp 映射为 quality。缺省字节合同与 edits 一致。 +- 禁止：只在 edits/`Form` 接受该字段而 generations 静默丢弃；用 HTTP 200 或非默认 format 冒充 compression 已生效。 +- 验收：L1 断言 PNG 100/1 与等价 compress_level 同大小且 1 更小，JPEG 高低 compression 体积方向正确；handler 用非默认值断言 encoder kwargs。^[PR #7447] + +## SERV-4s — 内存中的 image `file` 响应必须按固定块异步产出 + +- 触发：修改 `ImageGenerationResponse.stream_response`、`response_format=file`，或把已物化 PNG/ZIP 交给 Starlette `StreamingResponse`。 +- 强制：对已在内存中的单图/ZIP 使用异步迭代器按固定块（如 64 KiB）`memoryview` 切片产出；保留既有 headers/`Content-Length` 与完整 body 字节。 +- 禁止：把 `io.BytesIO` 直接交给 `StreamingResponse`（按行/`0x0A` 切分且同步 iterable 会触发 per-fragment threadpool）；为“流式”再无意义地按换行拆二进制。 +- 验收：构造含多处 `0x0A` 的 PNG/ZIP，断言 ASGI body 哈希不变、非空 frame 数约为 `ceil(size/chunk)` 而非 newline 次数。^[PR #7459] diff --git a/knowledge/repos/vllm-omni/models/lingbot-video/rules.md b/knowledge/repos/vllm-omni/models/lingbot-video/rules.md index f07d68b8..eece6177 100644 --- a/knowledge/repos/vllm-omni/models/lingbot-video/rules.md +++ b/knowledge/repos/vllm-omni/models/lingbot-video/rules.md @@ -1,7 +1,7 @@ --- title: "LingBot-Video 规则" created: 2026-08-10 -updated: 2026-09-04 +updated: 2026-09-15 type: rule tags: [vllm-omni, models, diffusion] sources: [vllm_omni/diffusion/models/lingbot_video/request_utils.py, vllm_omni/diffusion/models/lingbot_video/image_condition.py, vllm_omni/diffusion/models/lingbot_video/pipeline_lingbot_video.py, vllm_omni/entrypoints/openai/serving_video.py, vllm_omni/model_extras/lingbot_video.py, vllm_omni/model_extras/registry.py, examples/offline_inference/text_to_image/text_to_image.py, examples/offline_inference/image_to_video/image_to_video.py, tests/diffusion/models/lingbot_video/test_request_utils.py, tests/diffusion/models/lingbot_video/test_image_condition.py, tests/diffusion/models/lingbot_video/test_pipeline_lingbot_video.py, tests/entrypoints/openai_api/test_image_server.py, tests/entrypoints/openai_api/test_video_server.py, "PR #5311", "PR #5976", "PR #6049", "Issue #5883"] @@ -97,3 +97,10 @@ sources: [vllm_omni/diffusion/models/lingbot_video/request_utils.py, vllm_omni/d - 验收：同时覆盖 formatter 的 image/video key、T2I 单帧 shape、TI2V input cardinality、最终 image size limit、frame-count precedence 和 n>1 的 400。PR 所报 H200 单次耗时、峰值显存与 bitwise parity 缺少本知识树可复跑的命令/产物，不能作为性能或精度 gate。 + +## LBV-3a — AR 流式 VAE decode 状态必须按 session 持有并计入 admission + +- 触发：修改 LingBot/Wan 系 AR-Diffusion `post_decode`、streaming VAE decode、`SupportsStreamingDecode`，或 `model_owned_state_bytes_per_session`。 +- 强制：每个 `session_id`（与 `request_id` 同键）持有独立 `StreamingDecodeState`；chunk 间复用同一 temporal cache；`reset_ar_diffusion_session`/`close_ar_diffusion_session` 一并释放。`model_owned_state_bytes_per_session` 必须计入 decoder 按分辨率声明的常驻字节，不能只算 image condition。会走 VAE tiled decode 的 shape 不得冒充可跨 chunk 线程 cache。 +- 禁止：把 temporal cache 留在共享 VAE 模块上跨 session 覆写；块级独立 decode 却声称 timeline 连续；漏报 decode state 导致 admission 低估显存。 +- 验收：覆盖跨 chunk 连续性、session 隔离、release、非流式/tiling fallback，以及 admission 字节随 H×W 缩放不随 session 长度增长。^[PR #6533] diff --git a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md index 907b76e3..f672f6a0 100644 --- a/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md +++ b/knowledge/repos/vllm-omni/models/minicpm-o-4-5/rules.md @@ -1,7 +1,7 @@ --- title: "MiniCPM-o 4.5 规则" created: 2026-07-20 -updated: 2026-09-10 +updated: 2026-09-15 type: rule tags: [vllm-omni, models, model-executor] sources: ["PR #3642", "PR #5165", "PR #5382", "PR #5524", "PR #5638", "PR #5792", "PR #5869", "PR #6056", "PR #6154", "PR #6170", "PR #6318", "PR #6828", tests/dfx/perf/tests/test_minicpmo_4_5.json, tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json, tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py, tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py, tests/e2e/online_serving/test_minicpmo_4_5.py, tests/e2e/online_serving/test_minicpmo_4_5_duplex.py, tests/e2e/online_serving/test_minicpmo_4_5_expansion.py, tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py, vllm_omni/benchmarks/data_modules/seed_tts_dataset.py, vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/deploy/minicpmo_4_5.yaml, vllm_omni/experimental/fullduplex/client.py, vllm_omni/entrypoints/duplex/chat_fallback.py, vllm_omni/entrypoints/duplex/realtime_input.py, vllm_omni/entrypoints/duplex/session_runner.py, vllm_omni/entrypoints/duplex/serving.py, vllm_omni/entrypoints/duplex/vad.py, vllm_omni/model_executor/models/minicpmo_4_5/duplex/adapter.py, vllm_omni/model_executor/models/cosyvoice3/code2wav_core/hifigan.py, vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/cuda_graph_wrapper.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py, vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py, vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py, tests/model_executor/models/minicpmo_4_5/test_audio_chunk_mask.py, tests/model_executor/models/minicpmo_4_5/test_cfm_graph_capture_gating.py, tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py, tests/model_executor/models/minicpmo_4_5/test_cuda_graph_wrapper.py, tests/model_executor/models/minicpmo_4_5/test_pipeline.py, tests/model_executor/models/minicpmo_4_5/test_talker_batching.py, tests/model_executor/models/minicpmo_4_5/test_vision_flash_attention.py, "PR #6082", "PR #5604", "PR #6274", "PR #6346", "PR #6397", "PR #6406", "PR #6458", "PR #6587", "PR #6619", "PR #6757", "PR #6529", "PR #6772", vllm_omni/entrypoints/duplex/protocol.py, "PR #7071", "PR #5730"] @@ -300,3 +300,10 @@ confidence: high - 强制：embedding items 的 `modality` 必须是 `"audio"`（不得从 image 类 copy 成 `"image"`）。单条 `audio_embeds` 张量为 `(s, h)` 时，placeholder 长度取 `len(single_audio_embeds)` / 前导维 `s`，再经 `get_audio_len_by_num_chunks`；不得 `sum(map(len, …))` 把 hidden size 累进 token 数。 - 禁止：以 `modality="image"` 注册 audio embedding；让 `(10, 4096)` 请求约 40960 个 placeholder 并撞 `max_model_len`。 - 验收：modality 断言为 audio；经真实 `_get_prompt_updates` 覆盖 1s/3s/30s 与不同 `h`，placeholder 只随 `s` 变化。^[PR #5730] + +## MCPMO-3f — Thinker forward 必须返回裸 hidden_states tensor + +- 触发：修改 `MiniCPMO45OmniLLMForConditionalGeneration.forward`、Omni wrapper 的 shape 归一化，或 AR/`GPUModelRunner` 对 thinker 输出的消费。 +- 强制：thinker `forward` 只返回裸 `hidden_states` `torch.Tensor`；`inputs_embeds is None` 时只构建一次 embeddings。wrapper 仅当 `ndim==3 and shape[0]==1` 时 squeeze batch 维，得到 `(seq_len, H)`，并保留 2D `(1, H)` decode 行。 +- 禁止：返回 `(text_inputs_embeds, hidden_states)` tuple；用 `added_batch_dim` 误 squeeze 单 token 的 `(1, H)`；让 runner 用 `[0]` 把 embeddings 当最终 hidden 算 logits。 +- 验收：断言返回类型/值、单次 embedding 构建，以及 `(1,H)/(N,H)/(1,1,H)/(1,N,H)` 归一化。^[PR #7517]