From 95d2274020b88b82181fd87b66fb348b6fce1935 Mon Sep 17 00:00:00 2001 From: BaoYunkai <269084791+BaoYunkai@users.noreply.github.com> Date: Tue, 15 Sep 2026 10:19:30 +0000 Subject: [PATCH 1/3] fix(_workload_envs): assert profiler=torch alongside the profile bounds vLLM injects for EXTRA_VLLM_ARGS Magpie's vllm_mi355x.sh sets --profiler-config.profiler torch itself, but only in the flags it appends after EXTRA_VLLM_ARGS at actual launch. The argv preflight probe (now fixed to call parse_args, see previous commit) only sees EXTRA_VLLM_ARGS, so it correctly parses the delay_iterations/max_iterations/capture_torch_profiler/ detailed_trace_annotation flags this layer injects, then vLLM's own ProfilerConfig validator rejects capture_torch_profiler without profiler=='torch' present in that same fragment -- an argv that is valid once Magpie's flags are appended, rejected as VALUE_REJECTED (terminal, no repair) because the checked fragment was incomplete. Assert --profiler-config.profiler torch in this same injection so the fragment Hyperloom controls is self-consistent on its own. A duplicate from Magpie's script is a harmless last-wins repeat under vLLM's dotted-flag merge, not a conflict; an operator-set profiler flag is left untouched. --- .../tests/test_profile_and_kernel_handlers.py | 36 +++++++++++++++++++ .../actions/executors/_workload_envs.py | 21 +++++++++++ 2 files changed, 57 insertions(+) diff --git a/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py b/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py index 52288ca360..6573a7a42d 100644 --- a/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py +++ b/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py @@ -446,6 +446,40 @@ def test_materialize_profile_window_vllm_skill_formula_default_R( extra = rendered["benchmark"]["envs"]["EXTRA_VLLM_ARGS"] assert "--profiler-config.delay_iterations 6080" in extra, extra assert "--profiler-config.max_iterations 128" in extra, extra + # ``profiler=torch`` and a trace dir have to be asserted here too, not left to + # Magpie's launcher script alone: that script appends its own flags *after* + # EXTRA_VLLM_ARGS at actual launch, but the argv preflight probe only sees + # EXTRA_VLLM_ARGS, and vLLM's ProfilerConfig validator rejects + # delay/max_iterations without both present in the checked fragment. + assert "--profiler-config.profiler torch" in extra, extra + assert "--profiler-config.torch_profiler_dir" in extra, extra + + +def test_materialize_profile_does_not_duplicate_an_explicit_profiler_flag( + tmp_path, + monkeypatch, +): + """An operator-set ``profiler``/``torch_profiler_dir`` must not be doubled.""" + import yaml + + _clear_workload_env(monkeypatch) + src = _profile_yaml( + tmp_path, + "vllm", + { + "CONC": 32, + "ISL": 256, + "OSL": 1024, + "EXTRA_VLLM_ARGS": ( + "--profiler-config.profiler torch --profiler-config.torch_profiler_dir /tmp/operator-dir" + ), + }, + ) + out = _materialize_config_with_envs(src, tmp_path) + extra = yaml.safe_load(out.read_text())["benchmark"]["envs"]["EXTRA_VLLM_ARGS"] + assert extra.count("--profiler-config.profiler") == 1, extra + assert extra.count("--profiler-config.torch_profiler_dir") == 1, extra + assert "/tmp/operator-dir" in extra, extra def test_materialize_profile_window_vllm_skill_formula_explicit_R( @@ -734,6 +768,8 @@ def test_materialize_profile_restore_accepts_a_bound_that_already_holds( tmp_path, extra_envs={ "EXTRA_VLLM_ARGS": ( + "--profiler-config.profiler torch " + "--profiler-config.torch_profiler_dir /tmp/already-set " "--profiler-config.delay_iterations 6080 " "--profiler-config.max_iterations 64 " "--profiler-config.ignore_frontend True " diff --git a/src/hyperloom/orchestrator/actions/executors/_workload_envs.py b/src/hyperloom/orchestrator/actions/executors/_workload_envs.py index 7a2fcf2a2e..2b2a6a6d80 100644 --- a/src/hyperloom/orchestrator/actions/executors/_workload_envs.py +++ b/src/hyperloom/orchestrator/actions/executors/_workload_envs.py @@ -1542,6 +1542,27 @@ def materialize_config_with_envs( ("delay_iterations", f"--profiler-config.delay_iterations {delay_iters}"), ("max_iterations", f"--profiler-config.max_iterations {max_iters}"), ] + # ``profiler`` and ``torch_profiler_dir`` are normally set by + # Magpie's launcher script, not by this layer -- but that script + # appends its own flags *after* EXTRA_VLLM_ARGS in the real + # ``vllm serve`` invocation, so the argv preflight probe (which + # only sees EXTRA_VLLM_ARGS) checks capture_torch_profiler/ + # delay_iterations/max_iterations against a ProfilerConfig that + # never saw ``profiler=torch`` or a trace dir. vLLM's validator + # requires both whenever those bounds are present, so the probe + # fails an argv that will be valid once Magpie's flags are + # appended, and this layer's profiler bounds get treated as + # invalid and dropped instead of launched. Asserting placeholders + # here keeps the probed fragment self-consistent; the actual + # ``torch_profiler_dir`` Magpie computes from ``$WORKSPACE_DIR`` + # overrides this one at real launch time via vLLM's dotted-flag + # last-wins merge, so the value here only has to be a valid + # absolute path, not the directory the trace ends up under. An + # operator-set flag is left untouched either way. + if _profiler_flag_value(existing_vllm_args, "profiler") is None: + profiler_flags.append(("profiler", "--profiler-config.profiler torch")) + if _profiler_flag_value(existing_vllm_args, "torch_profiler_dir") is None: + profiler_flags.append(("torch_profiler_dir", f"--profiler-config.torch_profiler_dir {output_dir}")) if tracelens_patch_ok: profiler_flags.append(("capture_torch_profiler", "--profiler-config.capture_torch_profiler True")) profiler_flags.append(("detailed_trace_annotation", "--profiler-config.detailed_trace_annotation True")) From 275216c4821c0e4016fb0ed1fdb451ccbdb64163 Mon Sep 17 00:00:00 2001 From: BaoYunkai Date: Fri, 18 Sep 2026 14:43:06 +0000 Subject: [PATCH 2/3] fix(bypass): emit the profiler flags after EXTRA_VLLM_ARGS, not before ProfilerConfig refuses profiler=torch without a torch_profiler_dir, so the fragment the argv preflight probes has to name one even though the launcher computes the real directory. That injected value is a placeholder and depends on losing vLLM's last-wins dotted-flag merge. Magpie's launcher appends its own flags after EXTRA_VLLM_ARGS, so it wins. bypass_engine emitted its pair before extra_args, so the placeholder won instead and the trace landed under the round directory. Discovery probes workspace/torch_trace, workspace/capture_traces and workspace.parent/ capture_traces, none of which is where the steady-state trace then is, so the round reported no trace files. Order them the same way, and pin it: the backend's own trace dir must be the last one on the argv. --- CHANGELOG.md | 15 +++++++++++ .../tests/test_bypass_backend.py | 26 +++++++++++++++++++ .../actions/executors/bypass_engine.py | 20 ++++++++++---- 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abab92eee8..acf1e2b0a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Fixed +- **A vLLM profile round had its profiler bounds dropped before launch.** The + argv preflight probe sees only `EXTRA_VLLM_ARGS`, while the launcher appends + `--profiler-config.profiler torch` and a trace directory of its own + afterwards. `ProfilerConfig` refuses the iteration bounds this layer injects + unless both are present in the same fragment, so the probe rejected an argv + that is valid once the launcher's flags are appended, and the round then ran + with no bound on the capture window. Both flags are now asserted alongside + the bounds so the probed fragment is self-consistent on its own. The trace + directory is a placeholder: the launcher's own value has to win vLLM's + last-wins dotted-flag merge, so the bypass backend now emits its profiler + flags after `EXTRA_VLLM_ARGS` the way Magpie's launcher already does, rather + than before it where the placeholder would have won and sent the trace + somewhere trace discovery never looks. An operator-set profiler flag is left + untouched. + - **AgentX grading failures no longer fall back to throughput KEEP.** When an AgentX session cannot grade on interactivity because either side is missing the axis pair, explore, ``_lift_to_current_best``, and diff --git a/src/hyperloom/inference_optimizer/tests/test_bypass_backend.py b/src/hyperloom/inference_optimizer/tests/test_bypass_backend.py index da5593568f..e46980b87b 100644 --- a/src/hyperloom/inference_optimizer/tests/test_bypass_backend.py +++ b/src/hyperloom/inference_optimizer/tests/test_bypass_backend.py @@ -154,6 +154,32 @@ def test_server_command_sglang(): assert cmd[-2:] == ["--foo", "1"] +def test_the_backend_trace_dir_wins_over_the_preflight_placeholder(): + """EXTRA_VLLM_ARGS carries a trace dir only so the argv preflight accepts the bounds beside it. + + ProfilerConfig refuses ``profiler=torch`` without a ``torch_profiler_dir``, so the probed + fragment has to name one; it is a placeholder, and the launcher's own value has to win vLLM's + last-wins merge or the trace lands where this backend's discovery never looks. + """ + cmd = bypass_engine.build_server_command( + framework="vllm", + model="/m", + tp=1, + port=8888, + max_model_len=None, + extra_args=[ + "--profiler-config.profiler", + "torch", + "--profiler-config.torch_profiler_dir", + "/round-dir", + ], + profile_dir="/ws/torch_trace", + ) + + dirs = [cmd[i + 1] for i, token in enumerate(cmd) if token == "--profiler-config.torch_profiler_dir"] + assert dirs[-1] == "/ws/torch_trace" + + def test_sglang_atom_server_command_honors_python_exe(): """sglang/atom launch under the provided interpreter (not a PATH python3).""" sglang = bypass_engine.build_server_command( diff --git a/src/hyperloom/orchestrator/actions/executors/bypass_engine.py b/src/hyperloom/orchestrator/actions/executors/bypass_engine.py index 75b7772ff1..c6f5ba560c 100644 --- a/src/hyperloom/orchestrator/actions/executors/bypass_engine.py +++ b/src/hyperloom/orchestrator/actions/executors/bypass_engine.py @@ -85,16 +85,26 @@ def build_server_command( ] if max_model_len: cmd += ["--max-model-len", str(max_model_len)] - if profile_dir: - # vLLM enables the torch profiler via --profiler-config (the legacy VLLM_TORCH_PROFILER_DIR env is - # ignored), without which /start_profile returns 404 and no trace is written. - cmd += [ + if not profile_dir: + return cmd + list(extra_args) + # vLLM enables the torch profiler via --profiler-config (the legacy VLLM_TORCH_PROFILER_DIR env is + # ignored), without which /start_profile returns 404 and no trace is written. + # + # Last, after extra_args, the way Magpie's launcher orders its own: EXTRA_VLLM_ARGS has to carry a + # torch_profiler_dir for the argv preflight to accept the profile bounds beside it -- ProfilerConfig + # refuses `profiler=torch` without one -- and that value is a placeholder standing in for whatever the + # launcher computes. Emitting it first would let the placeholder win vLLM's last-wins dotted-flag merge + # and send the trace somewhere this backend's discovery never looks. + return ( + cmd + + list(extra_args) + + [ "--profiler-config.profiler", "torch", "--profiler-config.torch_profiler_dir", profile_dir, ] - return cmd + list(extra_args) + ) if fw == "atom": cmd = [ interp, From 5022752d9e4f21a09399ec920cf0280f356087e3 Mon Sep 17 00:00:00 2001 From: BaoYunkai Date: Fri, 18 Sep 2026 16:04:41 +0000 Subject: [PATCH 3/3] test(gemm-tuning): pin the backend this prerequisite test means to exercise The test names the geak branch and sets GEMM_TUNING_BACKEND=geak, but the dispatch reads KERNEL_OPT_BACKEND_ORDER instead, so the branch it took was whatever the ambient environment said. With that variable set to forge -- which a neighbour in the same xdist worker can leave behind -- the call goes down the forge branch, which reports model_path_missing, and the assertion on kernel_agent_root_missing fails for a reason that has nothing to do with either prerequisite. Clear the variable the dispatch actually reads, so the test exercises the branch it names rather than the one it inherits. Found by CI on this PR's shard 5; reproduced locally with KERNEL_OPT_BACKEND_ORDER=forge. The same fix is in #1536, which has not merged. --- .../tests/test_kernel_request_handlers_units.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/hyperloom/inference_optimizer/tests/test_kernel_request_handlers_units.py b/src/hyperloom/inference_optimizer/tests/test_kernel_request_handlers_units.py index 0270025777..ed211435d5 100644 --- a/src/hyperloom/inference_optimizer/tests/test_kernel_request_handlers_units.py +++ b/src/hyperloom/inference_optimizer/tests/test_kernel_request_handlers_units.py @@ -3658,6 +3658,10 @@ async def fake_run(cmd: list[str], *, timeout_sec: int): def test_handler_passes_non_fp8_geak_to_next_hyperloom_prereq(self, tmp_path, monkeypatch): monkeypatch.setenv("GEMM_TUNING_BACKEND", "geak") + # The backend is chosen by KERNEL_OPT_BACKEND_ORDER, not by GEMM_TUNING_BACKEND, so + # leaving it to the ambient environment sends this down the forge branch instead -- + # which reports model_path_missing, a prerequisite this test is not about. + monkeypatch.delenv("KERNEL_OPT_BACKEND_ORDER", raising=False) monkeypatch.delenv("HYPERLOOM_KERNEL_AGENT_ROOT", raising=False) state = SharedState(precision="bf16", framework="sglang") state.save(tmp_path)