Skip to content

Restore dpskv4 GB300 non-MTP disagg to staging image + deepep backend #1526

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
yhyang201 wants to merge 1 commit into from
Closed

Restore dpskv4 GB300 non-MTP disagg to staging image + deepep backend #1526

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/configs/nvidia-master.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8759,7 +8759,7 @@ dsv4-fp4-gb300-dynamo-vllm:
dp-attn: true

dsv4-fp4-gb300-dynamo-sglang:
image: lmsysorg/sglang:nightly-dev-cu13-20260519-dbac4647
image: lmsysorg/sglang-staging:deepseek-v4-grace-blackwell-dev
model: deepseek-ai/DeepSeek-V4-Pro
model-prefix: dsv4
runner: gb300-cw
Expand Down
40 changes: 28 additions & 12 deletions ...ode/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-10p1d-dep4-dep16-14-c8192.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ name: "disagg-gb300-10p1d-dep4-dep16-14-c8192"

model:
path: "deepseek-v4-pro"
container: "lmsysorg/sglang:nightly-dev-cu13-20260519-dbac4647"
container: "lmsysorg/sglang-staging:deepseek-v4-grace-blackwell-dev"
precision: "fp4"

dynamo:
Expand Down Expand Up @@ -74,14 +74,24 @@ backend:

prefill_environment:
PYTHONUNBUFFERED: "1"
SGLANG_RADIX_FORCE_MISS: "1"
SGLANG_RADIX_DISABLE_REUSE: "1"
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
SGLANG_DEFAULT_THINKING: "1"
SGLANG_DSV4_REASONING_EFFORT: "max"
SGLANG_ENABLE_THINKING: "1"
SGLANG_REASONING_EFFORT: "max"
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
SGLANG_OPT_USE_JIT_NORM: "1"
SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1"
SGLANG_OPT_USE_TOPK_V2: "1"
SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1"
SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: "1"
SGLANG_OPT_FIX_HASH_MEGA_MOE: "1"
SGLANG_OPT_USE_FAST_MASK_EP: "1"
SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1"
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192"
SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1"
SGLANG_OPT_USE_ONLINE_COMPRESS: "1"
SGLANG_OPT_FP8_WO_A_GEMM: "1"
SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "0"
NCCL_MNNVL_ENABLE: "1"
NCCL_CUMEM_ENABLE: "1"
Expand All @@ -94,17 +104,24 @@ backend:
SGLANG_LOG_FORWARD_ITERS: "1"
SGLANG_LOG_MS: "1"
SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60"
SGLANG_OPT_FP8_WO_A_GEMM: "0"

decode_environment:
PYTHONUNBUFFERED: "1"
SGLANG_RADIX_FORCE_MISS: "1"
SGLANG_RADIX_DISABLE_REUSE: "1"
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
SGLANG_DEFAULT_THINKING: "1"
SGLANG_DSV4_REASONING_EFFORT: "max"
SGLANG_ENABLE_THINKING: "1"
SGLANG_REASONING_EFFORT: "max"
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
SGLANG_OPT_USE_JIT_NORM: "1"
SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1"
SGLANG_OPT_USE_TOPK_V2: "1"
SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: "1"
SGLANG_OPT_FIX_HASH_MEGA_MOE: "1"
SGLANG_OPT_USE_FAST_MASK_EP: "1"
SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1"
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280"
SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1"
SGLANG_OPT_USE_ONLINE_COMPRESS: "1"
SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "0"
NCCL_MNNVL_ENABLE: "1"
Expand All @@ -119,7 +136,7 @@ backend:
SGLANG_LOG_FORWARD_ITERS: "1"
SGLANG_LOG_MS: "1"
SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60"
SGLANG_OPT_FP8_WO_A_GEMM: "0"
# SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 intentionally NOT set: CAR_V2
# is single-node only and corrupts results in 2-node decode setups.

sglang_config:
Expand All @@ -135,7 +152,7 @@ backend:
expert-parallel-size: 4

enable-dp-attention: true
moe-a2a-backend: "megamoe"
moe-a2a-backend: "deepep"
deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}'
moe-dense-tp-size: 1

Expand All @@ -154,7 +171,7 @@ backend:
stream-interval: 60

load-balance-method: "total_requests"
moe-a2a-backend: "megamoe"
moe-a2a-backend: "deepep"

disaggregation-mode: "decode"
disaggregation-transfer-backend: mooncake
Expand All @@ -179,4 +196,3 @@ benchmark:
concurrencies: "8192"
req_rate: "inf"
use_chat_template: false
custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"
40 changes: 28 additions & 12 deletions ...de/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-12p1d-dep4-dep12-15-c21504.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ name: "disagg-gb300-12p1d-dep4-dep12-15-c21504"

model:
path: "deepseek-v4-pro"
container: "lmsysorg/sglang:nightly-dev-cu13-20260519-dbac4647"
container: "lmsysorg/sglang-staging:deepseek-v4-grace-blackwell-dev"
precision: "fp4"

dynamo:
Expand Down Expand Up @@ -74,14 +74,24 @@ backend:

prefill_environment:
PYTHONUNBUFFERED: "1"
SGLANG_RADIX_FORCE_MISS: "1"
SGLANG_RADIX_DISABLE_REUSE: "1"
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
SGLANG_DEFAULT_THINKING: "1"
SGLANG_DSV4_REASONING_EFFORT: "max"
SGLANG_ENABLE_THINKING: "1"
SGLANG_REASONING_EFFORT: "max"
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
SGLANG_OPT_USE_JIT_NORM: "1"
SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1"
SGLANG_OPT_USE_TOPK_V2: "1"
SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1"
SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: "1"
SGLANG_OPT_FIX_HASH_MEGA_MOE: "1"
SGLANG_OPT_USE_FAST_MASK_EP: "1"
SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1"
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192"
SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1"
SGLANG_OPT_USE_ONLINE_COMPRESS: "1"
SGLANG_OPT_FP8_WO_A_GEMM: "1"
SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "0"
NCCL_MNNVL_ENABLE: "1"
NCCL_CUMEM_ENABLE: "1"
Expand All @@ -94,17 +104,24 @@ backend:
SGLANG_LOG_FORWARD_ITERS: "1"
SGLANG_LOG_MS: "1"
SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60"
SGLANG_OPT_FP8_WO_A_GEMM: "0"

decode_environment:
PYTHONUNBUFFERED: "1"
SGLANG_RADIX_FORCE_MISS: "1"
SGLANG_RADIX_DISABLE_REUSE: "1"
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
SGLANG_DEFAULT_THINKING: "1"
SGLANG_DSV4_REASONING_EFFORT: "max"
SGLANG_ENABLE_THINKING: "1"
SGLANG_REASONING_EFFORT: "max"
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
SGLANG_OPT_USE_JIT_NORM: "1"
SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1"
SGLANG_OPT_USE_TOPK_V2: "1"
SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: "1"
SGLANG_OPT_FIX_HASH_MEGA_MOE: "1"
SGLANG_OPT_USE_FAST_MASK_EP: "1"
SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1"
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280"
SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1"
SGLANG_OPT_USE_ONLINE_COMPRESS: "1"
SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "0"
NCCL_MNNVL_ENABLE: "1"
Expand All @@ -119,7 +136,7 @@ backend:
SGLANG_LOG_FORWARD_ITERS: "1"
SGLANG_LOG_MS: "1"
SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60"
SGLANG_OPT_FP8_WO_A_GEMM: "0"
# SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 intentionally NOT set: CAR_V2
# is single-node only and corrupts results in 2-node decode setups.

sglang_config:
Expand All @@ -135,7 +152,7 @@ backend:
expert-parallel-size: 4

enable-dp-attention: true
moe-a2a-backend: "megamoe"
moe-a2a-backend: "deepep"
deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}'
moe-dense-tp-size: 1

Expand All @@ -154,7 +171,7 @@ backend:
stream-interval: 60

load-balance-method: "total_requests"
moe-a2a-backend: "megamoe"
moe-a2a-backend: "deepep"

disaggregation-mode: "decode"
disaggregation-transfer-backend: mooncake
Expand All @@ -179,4 +196,3 @@ benchmark:
concurrencies: "21504"
req_rate: "inf"
use_chat_template: false
custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"
40 changes: 28 additions & 12 deletions ..._node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-dep4-dep16-5-c1024.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ name: "disagg-gb300-1p1d-dep4-dep16-5-c1024"

model:
path: "deepseek-v4-pro"
container: "lmsysorg/sglang:nightly-dev-cu13-20260519-dbac4647"
container: "lmsysorg/sglang-staging:deepseek-v4-grace-blackwell-dev"
precision: "fp4"

dynamo:
Expand Down Expand Up @@ -74,14 +74,24 @@ backend:

prefill_environment:
PYTHONUNBUFFERED: "1"
SGLANG_RADIX_FORCE_MISS: "1"
SGLANG_RADIX_DISABLE_REUSE: "1"
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
SGLANG_DEFAULT_THINKING: "1"
SGLANG_DSV4_REASONING_EFFORT: "max"
SGLANG_ENABLE_THINKING: "1"
SGLANG_REASONING_EFFORT: "max"
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
SGLANG_OPT_USE_JIT_NORM: "1"
SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1"
SGLANG_OPT_USE_TOPK_V2: "1"
SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1"
SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: "1"
SGLANG_OPT_FIX_HASH_MEGA_MOE: "1"
SGLANG_OPT_USE_FAST_MASK_EP: "1"
SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1"
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192"
SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1"
SGLANG_OPT_USE_ONLINE_COMPRESS: "1"
SGLANG_OPT_FP8_WO_A_GEMM: "1"
SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "0"
NCCL_MNNVL_ENABLE: "1"
NCCL_CUMEM_ENABLE: "1"
Expand All @@ -94,17 +104,24 @@ backend:
SGLANG_LOG_FORWARD_ITERS: "1"
SGLANG_LOG_MS: "1"
SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60"
SGLANG_OPT_FP8_WO_A_GEMM: "0"

decode_environment:
PYTHONUNBUFFERED: "1"
SGLANG_RADIX_FORCE_MISS: "1"
SGLANG_RADIX_DISABLE_REUSE: "1"
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
SGLANG_DEFAULT_THINKING: "1"
SGLANG_DSV4_REASONING_EFFORT: "max"
SGLANG_ENABLE_THINKING: "1"
SGLANG_REASONING_EFFORT: "max"
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
SGLANG_OPT_USE_JIT_NORM: "1"
SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1"
SGLANG_OPT_USE_TOPK_V2: "1"
SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: "1"
SGLANG_OPT_FIX_HASH_MEGA_MOE: "1"
SGLANG_OPT_USE_FAST_MASK_EP: "1"
SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1"
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280"
SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1"
SGLANG_OPT_USE_ONLINE_COMPRESS: "1"
SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "0"
NCCL_MNNVL_ENABLE: "1"
Expand All @@ -119,7 +136,7 @@ backend:
SGLANG_LOG_FORWARD_ITERS: "1"
SGLANG_LOG_MS: "1"
SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60"
SGLANG_OPT_FP8_WO_A_GEMM: "0"
# SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 intentionally NOT set: CAR_V2
# is single-node only and corrupts results in 2-node decode setups.

sglang_config:
Expand All @@ -135,7 +152,7 @@ backend:
expert-parallel-size: 4

enable-dp-attention: true
moe-a2a-backend: "megamoe"
moe-a2a-backend: "deepep"
deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}'
moe-dense-tp-size: 1

Expand All @@ -154,7 +171,7 @@ backend:
stream-interval: 60

load-balance-method: "total_requests"
moe-a2a-backend: "megamoe"
moe-a2a-backend: "deepep"

disaggregation-mode: "decode"
disaggregation-transfer-backend: mooncake
Expand All @@ -179,4 +196,3 @@ benchmark:
concurrencies: "1024"
req_rate: "inf"
use_chat_template: false
custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"
26 changes: 14 additions & 12 deletions .../multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@

model:
path: "deepseek-v4-pro"
container: "lmsysorg/sglang:nightly-dev-cu13-20260519-dbac4647"
container: "lmsysorg/sglang-staging:deepseek-v4-grace-blackwell-dev"
precision: "fp4"

# See ../1k1k/disagg-gb200-1p1d-dep8-tep8.yaml for the dynamo pin
Expand Down Expand Up @@ -80,37 +80,40 @@

prefill_environment:
PYTHONUNBUFFERED: "1"
SGLANG_RADIX_FORCE_MISS: "1"
SGLANG_RADIX_DISABLE_REUSE: "1"
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
SGLANG_DEFAULT_THINKING: "1"
SGLANG_DSV4_REASONING_EFFORT: "max"
SGLANG_ENABLE_THINKING: "1"
SGLANG_REASONING_EFFORT: "max"
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
SGLANG_OPT_USE_JIT_NORM: "1"
SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1"
SGLANG_OPT_USE_TOPK_V2: "1"
NCCL_MNNVL_ENABLE: "1"
NCCL_CUMEM_ENABLE: "1"
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
MC_FORCE_MNNVL: "1"
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1"
SGLANG_OPT_FP8_WO_A_GEMM: "0"

decode_environment:
PYTHONUNBUFFERED: "1"
SGLANG_RADIX_FORCE_MISS: "1"
SGLANG_RADIX_DISABLE_REUSE: "1"
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
SGLANG_DEFAULT_THINKING: "1"
SGLANG_DSV4_REASONING_EFFORT: "max"
SGLANG_ENABLE_THINKING: "1"
SGLANG_REASONING_EFFORT: "max"
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
SGLANG_OPT_USE_JIT_NORM: "1"
SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1"
SGLANG_OPT_USE_TOPK_V2: "1"
NCCL_MNNVL_ENABLE: "1"
NCCL_CUMEM_ENABLE: "1"
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
MC_FORCE_MNNVL: "1"
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1"
SGLANG_OPT_FP8_WO_A_GEMM: "0"
# SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 intentionally NOT set: CAR_V2

Check warning on line 116 in benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1.yaml

View check run for this annotation

Claude / Claude Code Review

Misapplied CAR_V2 comment in single-node decode config 

Nit: the CAR_V2 comment restored here doesn't match this file's topology. The `resources` block above declares `decode_nodes: 1, gpus_per_decode: 4` (single-node decode), but the comment justifies omitting `SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2` because "CAR_V2 is single-node only and corrupts results in 2-node decode setups" — by the comment's own logic, this is exactly the case where CAR_V2 is safe and intended. The comment was copy-pasted unchanged from the peer multi-node files (4p1d/8p1d/10p1
Comment on lines 115 to +116
Copy link
Copy Markdown
Contributor

@claude claude Bot May 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Nit: the CAR_V2 comment restored here doesn't match this file's topology. The resources block above declares decode_nodes: 1, gpus_per_decode: 4 (single-node decode), but the comment justifies omitting SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 because "CAR_V2 is single-node only and corrupts results in 2-node decode setups" — by the comment's own logic, this is exactly the case where CAR_V2 is safe and intended. The comment was copy-pasted unchanged from the peer multi-node files (4p1d/8p1d/10p1d/12p1d, all with decode_nodes ≥ 3). Since this PR is actively rewriting this exact comment block, it'd be a good time to either (a) enable CAR_V2 in decode here to match the rationale, or (b) replace the comment with the real reason it's omitted in this c=1 smoke-test config.

Extended reasoning...

What is the bug

In disagg-gb300-1p1d-tp4-tp4-2-c1.yaml, the decode_environment now ends with:

    SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1"
    # SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 intentionally NOT set: CAR_V2
    # is single-node only and corrupts results in 2-node decode setups.

The resources block earlier in the same file declares:

resources:
  gpu_type: "gb300"
  gpus_per_node: 4
  prefill_nodes: 1
  prefill_workers: 1
  gpus_per_prefill: 4
  decode_nodes: 1
  decode_workers: 1
  gpus_per_decode: 4

This is a single-node decode (decode_nodes=1, gpus_per_decode=4). The comment, however, justifies omitting CAR_V2 on the grounds that CAR_V2 is "single-node only and corrupts results in 2-node decode setups." By the comment's own rationale, single-node is the safe case for CAR_V2 — so the comment's justification is the opposite of what this file's topology calls for.

How it got here (step-by-step proof)

  1. In the peer files in this PR — 4p1d-dep4-dep16, 8p1d-dep4-dep16, 10p1d-dep4-dep16, 12p1d-dep4-dep12 — the decode is genuinely multi-node (decode_nodes: 4 or 3, gpus_per_decode: 16 or 12). In those files, the CAR_V2 comment is accurate: CAR_V2 would corrupt their multi-node decodes, so it's correctly omitted.
  2. The 1p1d-tp4-tp4-2-c1.yaml is a c=1 smoke-test config with a 1-node, 4-GPU decode (decode_nodes: 1, gpus_per_decode: 4).
  3. This PR restores the decode_environment block of 1p1d-tp4-tp4-2-c1.yaml to its pre-Update dpskv4 GB300 non-MTP disagg SGLang image to nightly-20260519 #1492 form, including reinstating the first line of the CAR_V2 comment. The result is a fully-formed two-line comment whose rationale doesn't logically apply to this file's actual topology.
  4. Verified independently: a peer file like 4p1d-dep4-dep16-8-c1024.yaml shows the comment in its correct context (multi-node decode, comment accurate). The 1p1d-tp4 file shows the comment in its incorrect context (single-node decode, comment's logic implies CAR_V2 should be enabled).

Why existing code doesn't prevent it

This is a YAML config comment — nothing parses or validates it. The mismatch is purely a readability/intent concern that only a human reviewer can catch.

Impact

  • No functional bug: env var behavior (CAR_V2 not set) is unchanged from the pre-PR state. The runtime sees the same configuration.
  • Readability: future readers will be confused — the comment's rationale ("2-node decode setups") doesn't apply to a 1-node-decode file. They'll be left wondering whether the author intended CAR_V2 to be enabled here.
  • Possible perf-on-the-table: if the omission was unintentional (i.e. carried in by copy-paste rather than deliberate choice for this smoke-test), enabling CAR_V2 in single-node decode is the documented intent. That said, since this is the c=1 smoke-test config, the perf angle is weak.

Addressing the refutation

The refuting verifier correctly notes that this comment existed in the file prior to PR #1492 — at commit 006bfab, the same two-line comment block was already present, and PR #1492 partially clobbered it (leaving an orphan dangling second line). This PR restores the original two-line comment intact.

That's true, but it doesn't mean the issue isn't worth flagging here:

  1. The PR is actively touching this exact line (diff line 115-116 of the file in question reinstates the first half of the comment). This PR is, materially, the act of restoring a misapplied comment to its full form. The fact that the misapplication is pre-existing doesn't change that this PR is the one putting the complete (and inapplicable) comment back in front of readers.
  2. It's the right moment to flag, even if not the right moment to require a fix. Hence: severity nit, not normal — purely a comment/documentation issue, no functional impact, no need to block the restore.
  3. The PR's stated goal of "verbatim restoration" is consistent with not fixing this in the same PR; the author may reasonably want to keep this PR scope-minimal. But a follow-up cleanup is worthwhile so future readers of this single-node file aren't left confused.

How to fix

Three reasonable options for a follow-up:

  1. Enable CAR_V2 here by adding SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" to this file's decode_environment, matching what the comment's rationale implies should happen in single-node decodes. (This would also align with the peer files that do enable CAR_V2 in their prefill environments, which are similarly single-node TP=4.)
  2. Remove the comment entirely if CAR_V2 absence is intentional for this debug-shape c=1 smoke-test.
  3. Rewrite the comment to state the real reason it's omitted here (e.g. "kept off for parity with debug-shape configs" or whatever the actual rationale is).

# is single-node only and corrupts results in 2-node decode setups.

sglang_config:
Expand Down Expand Up @@ -164,4 +167,3 @@
concurrencies: "1"
req_rate: "inf"
use_chat_template: false
custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"
Loading
Loading