From e9e91b4fe51ba3d11443993224c6032e511dee1d Mon Sep 17 00:00:00 2001 From: Matthias Gehre Date: Mon, 7 Sep 2026 05:58:42 -0600 Subject: [PATCH] fix(roofline): drop only cuda-graph capture, not inductor, for vLLM The profiling fallback appended --enforce-eager for vLLM, which turns off torch.compile/inductor as well as graph capture. A trace taken that way profiles the uncompiled kernels, not the ones the measured runs execute, so the roofline attributes device time to kernels production never runs. --compilation-config.cudagraph_mode NONE drops only the capture and keeps inductor compilation, so the profiled kernels match the measured ones. Changes: - The dotted form is deliberate. It is two plain tokens, so no shlex round-trip in the arg pipeline can damage it; the JSON form --compilation-config {"cudagraph_mode":"NONE"} only survives because _grid_server_args._repair_unquoted_json puts back the double quotes a round-trip strips. vLLM parses the dotted form in vllm/utils/argparse_utils.py: json.loads("NONE") fails, so it falls through to the raw string and merges to {"cudagraph_mode":"NONE"}. - _with_cuda_graph_disabled now dedups on the option's base name rather than the whole flag string. vLLM's flag is two tokens with a dotted option, so whole-string matching would never fire and the flag would be re-appended on every call. Comparing the part before any "." or "=" also treats an operator-supplied --compilation-config (JSON form) or --compilation-config. as already present, which matters because vLLM re-appends a synthesized --compilation-config for dotted args and argparse would then take the last one silently. The original property is kept: --disable-cuda-graph-extra is still not mistaken for --disable-cuda-graph, because base names are compared whole. - sglang is untouched; --disable-cuda-graph was already capture-only there. --- .../tests/test_baseline_param_overrides.py | 2 +- .../tests/test_roofline_executor.py | 6 ++++-- .../actions/executors/baseline.py | 21 +++++++++++++++---- .../actions/executors/roofline.py | 13 ++++++------ 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_param_overrides.py b/src/hyperloom/inference_optimizer/tests/test_baseline_param_overrides.py index bcbdd9cee6..4b9c4fe0d5 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_param_overrides.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_param_overrides.py @@ -572,7 +572,7 @@ def fake_run(cmd, *args, **kwargs): result = _run(executor(ctx)) assert result["status"] == "succeeded" - assert "--enforce-eager" in ctx.task.params["extra_server_args"] + assert "--compilation-config.cudagraph_mode NONE" in ctx.task.params["extra_server_args"] assert shared_state.baseline_eager_fallback is False diff --git a/src/hyperloom/inference_optimizer/tests/test_roofline_executor.py b/src/hyperloom/inference_optimizer/tests/test_roofline_executor.py index ca24a5bfa5..7b0ecc5383 100644 --- a/src/hyperloom/inference_optimizer/tests/test_roofline_executor.py +++ b/src/hyperloom/inference_optimizer/tests/test_roofline_executor.py @@ -192,7 +192,9 @@ async def fake_trace_analyze(payload, *, session_dir): assert result["status"] == "succeeded" assert calls == 2 - assert "--enforce-eager" in state.last_profile_workload["server_args"] + # vLLM drops only the capture (cudagraph_mode=NONE), keeping inductor + # compilation, so the profiled kernels match the measured ones. + assert "--compilation-config.cudagraph_mode NONE" in state.last_profile_workload["server_args"] assert state.last_profile_args == state.last_profile_workload["server_args"] @@ -1432,7 +1434,7 @@ async def test_431_zero_hot_with_degraded_trace_appends_warning(tmp_path): assert "cuda_graph_attribution_degraded" in codes, warnings w = next(w for w in warnings if w.get("code") == "cuda_graph_attribution_degraded") assert w["capture_traces_present"] is True - assert "--enforce-eager" in w["message"] + assert "--compilation-config.cudagraph_mode NONE" in w["message"] @pytest.mark.asyncio diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index ed1dc065db..6c7a184679 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -437,10 +437,11 @@ def _is_insufficient_gpu_memory(*texts: str) -> bool: return any(m in blob for m in _GPU_PREOCCUPIED_MARKERS) -# Disable cuda-graph capture per framework: sglang uses --disable-cuda-graph, vllm uses --enforce-eager. +# Disable cuda-graph capture per framework: sglang uses --disable-cuda-graph, vllm uses +# --compilation-config.cudagraph_mode NONE. _DISABLE_CUDA_GRAPH_FLAGS = { "sglang": "--disable-cuda-graph", - "vllm": "--enforce-eager", + "vllm": "--compilation-config.cudagraph_mode NONE", } @@ -604,9 +605,21 @@ def _disable_cuda_graph_flag(framework: str) -> str: def _with_cuda_graph_disabled(extra_server_args: str, framework: str) -> str: - """Append the framework-correct disable-cuda-graph flag once (idempotent).""" + """Append the framework-correct disable-cuda-graph flag once (idempotent). + + Dedup is on the option's base name, not the whole flag: vLLM's flag is two tokens with a + dotted option, and an operator-supplied ``--compilation-config`` in any form already covers + it. Base names are compared whole, so ``--disable-cuda-graph-extra`` is still not mistaken + for ``--disable-cuda-graph``. + """ + + def _base(token: str) -> str: + return token.split("=", 1)[0].split(".", 1)[0] + flag = _disable_cuda_graph_flag(framework) - if flag in (extra_server_args or "").split(): + option_base = _base(flag.split()[0]) + existing = (extra_server_args or "").split() + if any(_base(t) == option_base for t in existing if t.startswith("-")): return extra_server_args or "" return f"{extra_server_args} {flag}".strip() diff --git a/src/hyperloom/orchestrator/actions/executors/roofline.py b/src/hyperloom/orchestrator/actions/executors/roofline.py index af922de920..737591290b 100644 --- a/src/hyperloom/orchestrator/actions/executors/roofline.py +++ b/src/hyperloom/orchestrator/actions/executors/roofline.py @@ -364,16 +364,16 @@ async def _reported( # Track the last failure kind so the no-trace contract is preserved (profile_no_trace_failed) instead of # collapsing into profile_failed. last_phase = "profile" - # After a cuda-graph capture crash the next attempt boots eager so the torch-profiler stream capture cannot - # collide. + # After a cuda-graph capture crash the next attempt boots without capture so the torch-profiler stream + # capture cannot collide. disable_cuda_graph = os.environ.get("HYPERLOOM_PROFILE_DISABLE_CUDA_GRAPH", "").strip().lower() in { "1", "true", "yes", "on", } - # Resolve framework so the eager fallback picks the correct flag (vLLM --enforce-eager, sglang - # --disable-cuda-graph). + # Resolve framework so the fallback picks the correct flag (vLLM + # --compilation-config.cudagraph_mode NONE, sglang --disable-cuda-graph). framework = self._resolve_framework(ctx) from .baseline import ( _disable_cuda_graph_flag, @@ -956,8 +956,9 @@ def _note_analysis_run( "trace_analyze returned 0 hot kernels: the profile trace " "has no execute_*/user_annotation events, so per-kernel " "device time is folded into hipGraphLaunch wrappers under " - "cuda-graph capture (#431). Re-profile in eager mode " - "(append --enforce-eager to EXTRA_SGLANG_ARGS / " + "cuda-graph capture (#431). Re-profile without capture " + "(append --disable-cuda-graph to EXTRA_SGLANG_ARGS, or " + "--compilation-config.cudagraph_mode NONE to " "EXTRA_VLLM_ARGS) so per-step annotations fire, or enable " "a capture-fold fallback over capture_traces/." ),