From 3967a55b75c0302cf51ddafcd2c975ab2e9353e7 Mon Sep 17 00:00:00 2001 From: Rajesh Poornachandran Date: Thu, 10 Sep 2026 18:44:02 +0000 Subject: [PATCH 1/3] feat(keep): --max-latency-ms as one more veto on KEEP Redraw of the constraint after review. The previous shape landed a parallel latency subsystem beside three mechanisms that already owned the concern; this rides the ones that exist. Business code is 633 -> 194 lines, and two live bugs in the old shape are gone with it. One veto channel. The SLA is applied where the gain gates already rule: graded_comparison marks an over-budget candidate REVERT and names it in GradedComparison.veto_reason, so the promotion choke point, explore's round ladder and the kernel stack honour it by reading the verdict they already read. Explore refuses in the round that measured the variant, so an over-budget config is never folded onto the stack and never becomes the anchor the rest of the batch is graded against; its ledger row says latency_budget_exceeded rather than blaming a gain it comfortably made. One field. Every dict reaching the gate carries e2el_mean_ms, copied from VariantResult by the lane that built it. The alias table and the nested descent are gone: a lane that does not carry the field is that lane's bug, not a spelling the lookup should learn. This also removes the divergence the old shape had, where the gate read through the decoder while current_best archived the raw key -- a candidate reporting mean_e2el_ms passed a 200 ms budget on 150 ms and then archived None, so the report said "latency not measured" about a KEEP it had just graded. The tests could not catch it because they built the lift dict by splatting the decoder over the result, which is the decoder validating itself; they now drive each lane's real return dict. One copy of the budget. SharedState.latency_budget_ms, written at launch and archived with the session. HYPERLOOM_MAX_LATENCY_MS is gone -- the gate runs in the Coordinator process, so no worker read it -- as is the params tier, which nothing wrote. Four parsers of one float collapse to zero: argparse validates it, so an unusable value exits 2 rather than resolving to "no budget". A fail-closed gate must not have a switch that fails open; --max-latency-ms 200ms previously turned the SLA off while the operator believed it was on. Fail closed at the boundary. A baseline already over the ceiling stops the run with baseline_over_latency_budget instead of warning and then refusing every candidate for the rest of --max-hours to learn what was knowable at launch. That removes the baseline exemption, the over-budget champion prose in the report, and with it the case where a session ends "target reached" holding a current_best no Objective.reached() knew was inadmissible. Prompt and docs. to_latency_budget_summary renders "=== Latency budget (constraint) ===" and is called unwrapped: the neighbouring try/except blocks guard advisory sections, and swallowing this one would have the model route as if the session were unconstrained -- a silent failure of the constraint itself. orchestration.md names the block, SKILL.md's flag table carries the row, and optimization-loop.md, where KEEP semantics are defined, states the constraint. Tests follow the contract: the predicate, the lift veto and its recorded refusal, explore's REVERT driven end to end through the real ladder, the lanes' real return dicts, CLI validation, and resume from archived state. The source-string assertion, the self-referential payload, the permutations of the four resolvers and the retired framework task kind are gone -- _lift_to_current_best is near-agnostic to task_kind, so the seven-way parametrization was one path run seven times. Co-authored-by: Cursor --- CHANGELOG.md | 42 +++ docs/conceptual/optimization-loop.md | 22 ++ src/hyperloom/common/perf_metric.py | 26 ++ src/hyperloom/inference_optimizer/SKILL.md | 1 + .../inference_optimizer/cli/bootstrap.py | 3 + .../inference_optimizer/cli/parser.py | 30 ++ .../tests/test_explore_executor.py | 70 ++++ .../tests/test_latency_budget.py | 339 ++++++++++++++++++ .../orchestrator/actions/executors/explore.py | 22 +- .../actions/executors/integrate_patch.py | 3 + .../orchestrator/loop/conversation.py | 7 + src/hyperloom/orchestrator/loop/writeback.py | 52 +++ src/hyperloom/orchestrator/phases/kernel.py | 4 + .../orchestrator/phases/machine_state.py | 1 + src/hyperloom/orchestrator/phases/prelude.py | 2 + .../orchestrator/prompts/orchestration.md | 7 + .../orchestrator/specialists/rebench.py | 2 + .../state/_shared_state/render.py | 30 ++ .../orchestrator/state/shared_state.py | 20 +- 19 files changed, 677 insertions(+), 6 deletions(-) create mode 100644 src/hyperloom/inference_optimizer/tests/test_latency_budget.py diff --git a/CHANGELOG.md b/CHANGELOG.md index abab92eee8..ee0f40e4b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -766,6 +766,48 @@ for the user-facing summary. ### Added +- **`--max-latency-ms` makes a latency SLA a constraint on every KEEP.** + The optimizer maximized `output_throughput` and nothing else. Latency was + measured, reported and fed to the prompts, but no latency number could block a + promotion. That is survivable for a lever that raises throughput without + touching per-request latency, and unsafe for any lever that raises throughput + *by* making each stream slower: against a throughput-only gate such a lever + does not merely tolerate a latency regression, it selects for the largest one + on offer. Measured on one MI355X, splitting the card eight ways at two streams + each bought about 20% aggregate throughput while mean end-to-end latency went + from 183 ms to 1211 ms — a 6.6x regression that the session would have signed + off as a win. + + The ceiling rides the verdict the gain gates already decide rather than + arriving as a second opinion beside them: `graded_comparison` marks an + over-budget candidate `REVERT` and names it in a new + `GradedComparison.veto_reason`, so every consumer of the verdict — the + promotion choke point, explore's round ladder, the kernel stack — honours it + without a lane-by-lane check. Explore therefore refuses in the round that + measured the variant, which keeps an over-budget config from being folded onto + the stack and becoming the anchor the rest of the batch is graded against. + + It fails closed: a candidate that reported no end-to-end latency is refused, + since a constraint nobody measured is not one anybody satisfied. That makes + latency part of the promotion contract, so each lane copies `e2el_mean_ms` + from its `VariantResult` onto the dict it promotes — a lane that does not is + that lane's bug, not a spelling the lookup should learn. It fails closed at + the boundary too: a baseline already over the ceiling stops the run with + `baseline_over_latency_budget` instead of spending the whole `--max-hours` + refusing every candidate to discover what was knowable at launch, which is + also why no `current_best` can end a session over budget. + + The budget has one home, `SharedState.latency_budget_ms`, written at launch + and archived with the session so a resume restores it. Parse failures happen + only at the CLI, which exits 2: the switch on a fail-closed gate must not + itself fail open, and a value like `--max-latency-ms 200ms` silently resolving + to "no budget" would leave an operator believing an SLA was enforced. + Refusals are recorded on `SharedState.latency_refusals` and rendered as + `=== Latency budget (constraint) ===`, because a constrained session that ends + near baseline is otherwise indistinguishable from one that found no headroom, + and the two call for opposite responses. Off by default; KEEP behaviour is + exactly as it was when unset. + - **Session breakdown exports now include the additive V6 startup contract.** The existing V5 payload remains intact while `metadata`, `outcome`, `timeline`, and `close` provide the V6 read model. Install and model-gate diff --git a/docs/conceptual/optimization-loop.md b/docs/conceptual/optimization-loop.md index 3db2e18ab1..08ed9f4f47 100644 --- a/docs/conceptual/optimization-loop.md +++ b/docs/conceptual/optimization-loop.md @@ -181,6 +181,28 @@ authoring specialist's prompt. The rungs, in increasing complexity: environment variable (see [Targeted builds (Rung 5)](../reference/environment-variables.md#targeted-builds-rung-5)). +### Latency budget (constraint on KEEP) + +`--max-latency-ms` sets a ceiling on mean end-to-end latency. It is a +constraint rather than an objective: it does not decide when the run stops, +only which winners are admissible, so it composes with whichever `--target-*` +is in use. It rides the same verdict the gain gates decide — a candidate that +clears its objective and breaks the ceiling is a REVERT, carrying a +`veto_reason` that distinguishes it from one that simply did not gain. + +The constraint exists because a throughput-only comparison does not merely +tolerate a latency-for-throughput trade, it selects for the worst one on +offer: facing a lever that raises aggregate throughput *by* making each stream +slower, the largest regression is where the most throughput is. + +It fails closed. A candidate that reported no end-to-end latency is refused, +since a constraint nobody measured is not one anybody satisfied — which is why +every lane copies `e2el_mean_ms` onto the dict it promotes. It fails closed at +the boundary too: if the baseline itself exceeds the ceiling, the run stops +with `baseline_over_latency_budget` rather than spending its whole budget +refusing every candidate to learn what was knowable at launch. Off by default, +leaving KEEP behaviour unchanged when unset. + ### Runnable gate (earned KEEP) A verified build does not KEEP on artifact verification alone. After a diff --git a/src/hyperloom/common/perf_metric.py b/src/hyperloom/common/perf_metric.py index 5cfdf75a4a..fc169d9fbd 100644 --- a/src/hyperloom/common/perf_metric.py +++ b/src/hyperloom/common/perf_metric.py @@ -7,6 +7,7 @@ import os from dataclasses import dataclass +from math import isfinite from typing import Any, Mapping from hyperloom.common.env import env_bool, env_str @@ -205,6 +206,28 @@ def passes_intvty_gate( return _within_band(intvty_of(candidate), intvty_of(anchor), band) +def latency_veto_reason(observed_ms: Any, budget_ms: float) -> str: + """Why the latency budget refuses this candidate, or "" when it does not. + + The budget is a ceiling on mean end-to-end latency, so unlike the gain gates + it refuses a candidate whose throughput won: a lever that buys throughput by + making each stream slower is exactly the case a throughput-only comparison + selects for. Off entirely when *budget_ms* is not positive. + + Fails closed on an unmeasured candidate — a constraint nobody measured is not + one anybody satisfied — which is why every lane copies ``e2el_mean_ms`` onto + the dict it promotes. + """ + if not budget_ms or budget_ms <= 0: + return "" + if isinstance(observed_ms, bool) or not isinstance(observed_ms, (int, float)): + return "latency_unmeasured" + observed = float(observed_ms) + if not isfinite(observed) or observed <= 0: + return "latency_unmeasured" + return "latency_budget_exceeded" if observed > float(budget_ms) else "" + + def passes_tput_guard( candidate: Mapping[str, float], anchor: Mapping[str, float], @@ -224,6 +247,7 @@ class GradedComparison: ``candidate`` and ``reference`` are both read on ``objective``. ``tput_*`` carry the guard axis and are 0.0 off AgentX. ``degrade_reason`` names why the interactivity axis did not apply on a session that asked for it. + ``veto_reason`` names a constraint that refused a candidate its throughput would otherwise have kept. """ objective: str @@ -233,6 +257,7 @@ class GradedComparison: tput_candidate: float = 0.0 tput_reference: float = 0.0 degrade_reason: str = "" + veto_reason: str = "" @property def comparable(self) -> bool: @@ -268,6 +293,7 @@ def graded_on_intvty(self) -> bool: "intvty_of", "intvty_serving_grading_enabled", "is_agentx_mode", + "latency_veto_reason", "output_tput_of", "parse_intvty_noise_pct", "passes_intvty_gate", diff --git a/src/hyperloom/inference_optimizer/SKILL.md b/src/hyperloom/inference_optimizer/SKILL.md index b461a49482..ada351e5fd 100644 --- a/src/hyperloom/inference_optimizer/SKILL.md +++ b/src/hyperloom/inference_optimizer/SKILL.md @@ -657,6 +657,7 @@ and the operator's stated value is lost: | Expert parallel | `--ep` | Pass the prompt's EP for MoE. Default `1`. | | Precision | `--precision` | Match the checkpoint (`bf16` default / `fp8` / ...). Keep consistent with `--quantize`. | | Budget | `--max-hours` | Pass the prompt's time budget. Default `2.0`. | +| Latency SLA | `--max-latency-ms` | Pass any stated ceiling on per-request latency ("must stay under 250 ms", "interactive workload"). A **constraint, not a target**: it composes with `--target-*` rather than competing, and refuses any KEEP whose mean end-to-end latency exceeds it — including one that reported no latency at all. Off when omitted, which does not lose a preference but does remove the SLA from the search. | | Max model len | `--max-model-len` | Optional; auto-derived from ISL+OSL+headroom when omitted. | | External reference GPU | `--compare-against-gpu` | `target_analysis` writes `target_analysis/target_baseline.json` for query/status metadata and `competitor_target.json` for both advisory and final-report comparisons. Without a target GPU it writes `reason="no_target_gpu_configured"` and clears the competitor target. AgentX reads accepted `current_best.total_throughput / state.tp` and `current_best.e2e_norm_intvty_p90` at `state.conc`; it does not reread raw results or recipes. Missing targets or axes remain unavailable. This is a cross-system advisory, not proof of identical measurement estimators or deployment, and never changes Objective or KEEP/REVERT. | | Target advisory | `--no-target-advisory` | Disable external-target hints in prompts without disabling final-report comparison. `primary_gap` uses the existing latency/throughput categories; the interactivity axis is displayed as interactivity. | diff --git a/src/hyperloom/inference_optimizer/cli/bootstrap.py b/src/hyperloom/inference_optimizer/cli/bootstrap.py index 1fcb999a07..75ea86fa3b 100644 --- a/src/hyperloom/inference_optimizer/cli/bootstrap.py +++ b/src/hyperloom/inference_optimizer/cli/bootstrap.py @@ -285,6 +285,9 @@ def _resolve_framework_version(args_in: Any) -> str: # config.json structural summary, persisted for downstream collectors. model_info=summarize_model_config(str(args.model)), framework=os.environ.get("FRAMEWORK", "sglang"), + # The only copy of the budget. Validated at the CLI, so anything that reaches here is usable, and archived + # with the session so a resume restores it without a second source to reconcile. + latency_budget_ms=float(getattr(args, "max_latency_ms", None) or 0.0), gpu_type=str(getattr(args, "gpu_type", None) or os.environ.get("GPU_TYPE", "")), # Workload metadata mirrored from CLI/env. tp=_int_arg("tp", DEFAULT_TP), diff --git a/src/hyperloom/inference_optimizer/cli/parser.py b/src/hyperloom/inference_optimizer/cli/parser.py index e1d94d2936..498f12a973 100644 --- a/src/hyperloom/inference_optimizer/cli/parser.py +++ b/src/hyperloom/inference_optimizer/cli/parser.py @@ -7,6 +7,7 @@ import argparse import os +from math import isfinite from pathlib import Path from typing import NoReturn @@ -106,6 +107,22 @@ def _positive_int_arg(value: str) -> int: return parsed +def _positive_ms_arg(value: str) -> float: + """argparse type for a millisecond ceiling. + + The gate this feeds fails closed, so its switch must not fail open: an + unusable value has to stop the launch rather than resolve to "no budget" and + leave the operator believing an SLA is enforced. + """ + try: + parsed = float(str(value).strip()) + except (TypeError, ValueError) as exc: + raise argparse.ArgumentTypeError(f"expected a positive number of milliseconds, got {value!r}") from exc + if not isfinite(parsed) or parsed <= 0: + raise argparse.ArgumentTypeError(f"expected a positive number of milliseconds, got {value!r}") + return parsed + + def _default_claude_model_env() -> str: """Resolve the default Claude model from env.""" explicit = (os.environ.get("CLAUDE_MODEL") or "").strip() @@ -563,6 +580,19 @@ def _build_parser() -> argparse.ArgumentParser: "independently of --target-gain / --target-tput / --target-baseline-dir." ), ) + # Outside the group as well, and for a stronger reason than --target-roofline: this is a constraint rather than + # an objective. It does not say when to stop, it says which winners are admissible, so it composes with whichever + # target is in use instead of competing with one. + opt.add_argument( + "--max-latency-ms", + type=_positive_ms_arg, + default=None, + help=( + "Refuse any KEEP whose mean end-to-end latency exceeds N ms. Off by " + "default. A candidate that reported no end-to-end latency is refused " + "too, since an unmeasured constraint is not a satisfied one." + ), + ) opt.add_argument( "--resume-from", type=str, diff --git a/src/hyperloom/inference_optimizer/tests/test_explore_executor.py b/src/hyperloom/inference_optimizer/tests/test_explore_executor.py index a288a4e468..1e216a5912 100644 --- a/src/hyperloom/inference_optimizer/tests/test_explore_executor.py +++ b/src/hyperloom/inference_optimizer/tests/test_explore_executor.py @@ -2718,3 +2718,73 @@ def fake_kill(): assert result["status"] == "failed" assert kill_calls["n"] == 0, "must be a no-op while PYTEST_CURRENT_TEST is set" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "budget_ms,expected_outcome,expected_reason", + [ + (250.0, "REVERT", "latency_budget_exceeded"), + (5000.0, "KEEP", ""), + (0.0, "KEEP", ""), + ], +) +async def test_explore_refuses_an_over_budget_winner_in_the_round_that_measured_it( + sub_agent_runner, tmp_path, monkeypatch, budget_ms, expected_outcome, expected_reason +): + """``--max-latency-ms`` rides the verdict explore's ladder already reads. + + The variant gains throughput either way; only the SLA separates the cases. + Refusing here rather than at promotion keeps an over-budget variant from + being folded onto the stack and becoming the anchor the rest of the batch is + graded against, and gives the ledger a latency reason for the REVERT. + """ + _force_cold_decision(monkeypatch) + monkeypatch.delenv("HYPERLOOM_AGENTX", raising=False) + monkeypatch.delenv("HYPERLOOM_PERF_METRIC", raising=False) + sub, tr, _ = sub_agent_runner + state = SharedState(framework="sglang") + state.baseline_tput = 200.0 + state.latency_budget_ms = budget_ms + sub.shared_state = state + base = tmp_path / "base.yaml" + _write_baseline_yaml(base) + + def _fake_run(cmd, *args, **kwargs): + slot = Path(cmd[cmd.index("--output-dir") + 1]) + # The harness workspace reports e2el mean 2500 ms, well over the 250 ms budget. + _fake_workspace(slot, tput=20000.0) + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + + task = await tr.create( + kind="explore", + params={ + "config_path": str(base), + "output_dir": str(tmp_path / f"explore-latency-{budget_ms:g}"), + "base_tput": 200.0, + "grid": [{"name": "v_slow_but_fast", "extra_args": "--split 8"}], + "variant_timeout_sec": 10, + }, + idempotency_key=f"ex-latency-{budget_ms:g}", + ) + sub.register_executor("explore", ExploreExecutor(session_dir=tmp_path)) + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_fake_run, + ): + res = await sub.run_task(task) + + out = res.result + tested = out["explore_search_update"]["tested"][canonical_fingerprint("--split 8", {})] + assert tested["status"] == "succeeded" + assert tested["outcome"] == expected_outcome + if expected_outcome == "REVERT": + # Not "gain_below_threshold": the variant gained 100x. Naming the wrong + # gate would send the search looking for throughput it already has. + assert out["losers"][0]["reason"] == expected_reason + assert out["winners"] == [] + gates = {g["gate"]: g for g in tested["gates"]} + assert "latency_budget" in gates + assert gates["latency_budget"]["passed"] is False + else: + assert [w["name"] for w in out["winners"]] == ["v_slow_but_fast"] diff --git a/src/hyperloom/inference_optimizer/tests/test_latency_budget.py b/src/hyperloom/inference_optimizer/tests/test_latency_budget.py new file mode 100644 index 0000000000..d8af304f5d --- /dev/null +++ b/src/hyperloom/inference_optimizer/tests/test_latency_budget.py @@ -0,0 +1,339 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""``--max-latency-ms``: one veto on KEEP, and the contract that feeds it. + +The gate fails closed, so what reaches it is as much the subject here as the +decision itself: a lane that does not carry ``e2el_mean_ms`` refuses every KEEP +it would ever have made, and that is a bug in the lane, not in the lookup. The +lane tests therefore drive each executor's real return dict rather than a +fixture already shaped the way the gate wants. +""" + +from __future__ import annotations + +import argparse + +import pytest + +from hyperloom.common.perf_metric import ( + VERDICT_KEEP, + VERDICT_REVERT, + latency_veto_reason, +) +from hyperloom.orchestrator.actions.executors._grid_base import VariantResult +from hyperloom.orchestrator.state.shared_state import SharedState, resolve_graded_comparison + + +class TestPredicate: + """The whole decision, in one function.""" + + def test_a_candidate_under_the_ceiling_is_not_vetoed(self): + assert latency_veto_reason(150.0, 200.0) == "" + + def test_a_candidate_over_the_ceiling_is_vetoed(self): + assert latency_veto_reason(1211.0, 250.0) == "latency_budget_exceeded" + + def test_the_ceiling_itself_passes(self): + """A ceiling is a maximum, so equality is inside it.""" + assert latency_veto_reason(250.0, 250.0) == "" + + @pytest.mark.parametrize("missing", [None, "", "183", float("nan"), 0.0, -1.0, True]) + def test_an_unmeasured_candidate_is_refused_not_admitted(self, missing): + """Fail closed: a constraint nobody measured is not one anybody satisfied.""" + assert latency_veto_reason(missing, 250.0) == "latency_unmeasured" + + @pytest.mark.parametrize("off", [0.0, None]) + def test_no_budget_vetoes_nothing(self, off): + """Off by default, including for a candidate that reported nothing.""" + assert latency_veto_reason(5000.0, off) == "" + assert latency_veto_reason(None, off) == "" + + +class TestOneVetoChannel: + """The SLA rides the verdict the gain gates already decide.""" + + def _state(self, budget: float) -> SharedState: + state = SharedState() + state.baseline_tput = 1000.0 + state.latency_budget_ms = budget + return state + + def test_the_motivating_case_a_throughput_win_that_breaks_the_sla(self): + """+20% aggregate throughput at 1211 ms against a 250 ms SLA is a REVERT. + + The case the flag exists for: a lever that raises throughput *by* making + each stream slower. Graded on throughput alone this is the best candidate + on offer, which is exactly the problem. + """ + graded = resolve_graded_comparison( + self._state(250.0), + {"output_throughput": 1200.0, "e2el_mean_ms": 1211.0}, + keep_threshold_pct=1.0, + ) + assert graded.verdict == VERDICT_REVERT + assert graded.veto_reason == "latency_budget_exceeded" + + def test_the_same_win_inside_the_sla_still_keeps(self): + graded = resolve_graded_comparison( + self._state(250.0), + {"output_throughput": 1200.0, "e2el_mean_ms": 183.0}, + keep_threshold_pct=1.0, + ) + assert graded.verdict == VERDICT_KEEP + assert graded.veto_reason == "" + + def test_an_untimed_winner_is_refused_under_a_budget(self): + graded = resolve_graded_comparison( + self._state(250.0), + {"output_throughput": 1200.0}, + keep_threshold_pct=1.0, + ) + assert graded.verdict == VERDICT_REVERT + assert graded.veto_reason == "latency_unmeasured" + + def test_an_untimed_winner_keeps_when_no_budget_is_set(self): + """Off by default: KEEP behaviour is exactly as it was when unset.""" + graded = resolve_graded_comparison( + self._state(0.0), + {"output_throughput": 1200.0}, + keep_threshold_pct=1.0, + ) + assert graded.verdict == VERDICT_KEEP + assert graded.veto_reason == "" + + def test_a_veto_is_distinguishable_from_a_candidate_that_simply_did_not_gain(self): + """Both are REVERT; only the reason says which, and they need opposite responses.""" + state = self._state(250.0) + no_gain = resolve_graded_comparison( + state, + {"output_throughput": 900.0, "e2el_mean_ms": 100.0}, + keep_threshold_pct=1.0, + ) + vetoed = resolve_graded_comparison( + state, + {"output_throughput": 1200.0, "e2el_mean_ms": 1211.0}, + keep_threshold_pct=1.0, + ) + assert no_gain.verdict == vetoed.verdict == VERDICT_REVERT + assert no_gain.veto_reason == "" + assert vetoed.veto_reason == "latency_budget_exceeded" + + +class TestLaneResultShapes: + """Every lane must hand the gate the field it grades on. + + Built from each executor's own return-dict construction rather than from a + dict already carrying the canonical key: the gate fails closed, so a lane + that forgets the field loses every KEEP it would have made, and a fixture + that supplies it cannot catch that. + """ + + def _variant(self) -> VariantResult: + return VariantResult( + name="cpx-2-streams", + extra_server_args="--tp 8", + extra_envs={}, + status="succeeded", + output_throughput=1200.0, + ttft_mean_ms=40.0, + e2el_mean_ms=1211.0, + tpot_mean_ms=12.0, + ) + + def test_variant_result_carries_the_canonical_name(self): + """Pins the attribute the lanes copy: a rename must not silently read None.""" + assert self._variant().e2el_mean_ms == 1211.0 + assert "e2el_mean_ms" in self._variant().to_dict() + + def test_the_explore_variant_dict_carries_it(self): + """Explore promotes ``VariantResult.to_dict()`` rows directly.""" + assert self._variant().to_dict()["e2el_mean_ms"] == 1211.0 + + def test_the_specialist_rebench_dict_carries_it(self, tmp_path, monkeypatch): + """The lane's own return dict, produced by calling it with the benchmark stubbed.""" + import asyncio + + from hyperloom.orchestrator.specialists import rebench + + monkeypatch.setattr(rebench, "materialize_config_with_envs", lambda *a, **k: tmp_path / "cfg.yaml") + monkeypatch.setattr(rebench, "_current_leased_cards", lambda: "0") + + async def _fake_run_grid(**_kwargs): + return [self._variant()] + + monkeypatch.setattr(rebench, "run_grid", _fake_run_grid) + result = asyncio.run( + rebench.run_specialist_rebench(config_path=None, output_dir=tmp_path, port=8000), + ) + assert result["e2el_mean_ms"] == 1211.0 + + def test_the_integrate_patch_lift_carries_it(self): + """``_integrate_measurement_fields`` builds the dict integrate_patch promotes.""" + from hyperloom.orchestrator.loop.writeback import _integrate_measurement_fields + + fields = _integrate_measurement_fields(self._variant().to_dict()) + assert fields["e2el_mean_ms"] == 1211.0 + + def test_a_lane_that_forgets_the_field_loses_its_keep(self): + """Why the lane tests exist: the gate cannot tell an untimed candidate + from an unplumbed one, so the plumbing is part of the contract.""" + state = SharedState() + state.baseline_tput = 1000.0 + state.latency_budget_ms = 250.0 + unplumbed = {k: v for k, v in self._variant().to_dict().items() if k != "e2el_mean_ms"} + graded = resolve_graded_comparison(state, unplumbed, keep_threshold_pct=1.0) + assert graded.veto_reason == "latency_unmeasured" + + +class TestLiftRefusesAndSaysWhy: + """The promotion choke point honours the veto and leaves an operator a trail.""" + + def _coord(self, tmp_path, budget: float): + from hyperloom.orchestrator.loop.coordinator import Coordinator + + coord = Coordinator.__new__(Coordinator) + coord.session_dir = tmp_path + coord.shared_state = SharedState( + baseline_tput=1000.0, + latency_budget_ms=budget, + model_path="/models/m", + gpu_type="mi355x", + ) + return coord + + def _winner(self, **over): + return { + "name": "cpx-2-streams", + "extra_server_args": "--tp 8", + "output_throughput": 1200.0, + **over, + } + + def test_an_over_budget_winner_does_not_reach_current_best(self, tmp_path): + coord = self._coord(tmp_path, 250.0) + assert coord._lift_to_current_best("explore", 1200.0, self._winner(e2el_mean_ms=1211.0)) is False + assert not coord.shared_state.current_best + assert not coord.shared_state.optimization_stack + + def test_the_refusal_is_recorded_with_what_it_measured(self, tmp_path): + """A session that ends near baseline under an SLA must be distinguishable + from one that found no headroom.""" + coord = self._coord(tmp_path, 250.0) + coord._lift_to_current_best("explore", 1200.0, self._winner(e2el_mean_ms=1211.0)) + (refusal,) = coord.shared_state.latency_refusals + assert refusal["reason"] == "latency_budget_exceeded" + assert refusal["variant_name"] == "cpx-2-streams" + assert refusal["e2el_mean_ms"] == 1211.0 + assert refusal["budget_ms"] == 250.0 + + def test_an_untimed_winner_is_refused_as_untimed_not_as_slow(self, tmp_path): + """The two reasons need opposite responses: one needs a different + candidate, the other needs the benchmark to report latency at all.""" + coord = self._coord(tmp_path, 250.0) + assert coord._lift_to_current_best("integrate_patch", 1200.0, self._winner()) is False + assert coord.shared_state.latency_refusals[0]["reason"] == "latency_unmeasured" + assert coord.shared_state.latency_refusals[0]["e2el_mean_ms"] is None + + def test_an_in_budget_winner_is_promoted_and_records_nothing(self, tmp_path): + coord = self._coord(tmp_path, 250.0) + assert coord._lift_to_current_best("explore", 1200.0, self._winner(e2el_mean_ms=183.0)) is True + assert coord.shared_state.current_best + assert coord.shared_state.latency_refusals == [] + + def test_with_no_budget_an_untimed_winner_still_promotes(self, tmp_path): + """Off by default: KEEP behaviour is exactly as it was when unset.""" + coord = self._coord(tmp_path, 0.0) + assert coord._lift_to_current_best("explore", 1200.0, self._winner()) is True + assert coord.shared_state.latency_refusals == [] + + +class TestBaselineFailsClosedAtTheBoundary: + """An over-budget baseline is knowable at launch; do not spend the run on it.""" + + def test_the_stop_reason_is_in_the_closed_vocabulary(self): + """PolicyGate rejects anything outside it, so an unregistered value would + silently degrade into "the run did not stop".""" + from hyperloom.orchestrator.phases.machine_state import is_valid_stop_reason + + assert is_valid_stop_reason("baseline_over_latency_budget") + + def test_setting_it_takes(self): + state = SharedState() + assert state.set_stop_reason("baseline_over_latency_budget") == "baseline_over_latency_budget" + assert state.stop_reason == "baseline_over_latency_budget" + + +class TestCliValidation: + """The switch on a fail-closed gate must not itself fail open.""" + + def _parse(self, *argv: str) -> argparse.Namespace: + from hyperloom.inference_optimizer.cli.parser import _build_parser + + return _build_parser().parse_args(["optimize", "--model", "/m", *argv]) + + def test_a_budget_is_parsed(self): + assert self._parse("--max-latency-ms", "250").max_latency_ms == 250.0 + + def test_omitting_it_leaves_the_gate_off(self): + assert self._parse().max_latency_ms is None + + @pytest.mark.parametrize("bad", ["250ms", "abc", "0", "-5", "nan", "inf"]) + def test_an_unusable_value_stops_the_launch_rather_than_disabling_the_sla(self, bad): + """The old failure: a bad value resolved to "no budget", so the operator + believed an SLA was enforced while every candidate passed.""" + with pytest.raises(SystemExit) as exc: + self._parse("--max-latency-ms", bad) + assert exc.value.code == 2 + + +class TestSessionCarriesTheOnlyCopy: + """One copy of the budget: state, written at launch, archived for resume.""" + + def test_the_launch_flag_lands_on_state(self): + from hyperloom.orchestrator.state.shared_state import SharedState as S + + assert S(latency_budget_ms=250.0).latency_budget_ms == 250.0 + + def test_a_resume_restores_it_from_the_archived_state(self, tmp_path): + """No second source to reconcile: the value round-trips through state.json.""" + state = SharedState(latency_budget_ms=250.0) + state.save(tmp_path) + assert SharedState.load_or_init(tmp_path).latency_budget_ms == 250.0 + + def test_it_defaults_to_off(self): + assert SharedState().latency_budget_ms == 0.0 + + +class TestPromptBlock: + """What the router sees. Rendered, not asserted against source text.""" + + def test_no_block_when_no_budget(self): + assert SharedState().to_latency_budget_summary() == "" + + def test_the_constraint_is_stated_when_set(self): + state = SharedState(latency_budget_ms=250.0) + block = state.to_latency_budget_summary() + assert "250 ms" in block + assert "refused : none so far" in block + + def test_refusals_are_listed_so_a_binding_sla_is_visible(self): + state = SharedState(latency_budget_ms=250.0) + state.latency_refusals = [ + {"variant_name": "cpx-2-streams", "action": "explore", "e2el_mean_ms": 1211.0}, + {"variant_name": "qpx-4", "action": "integrate_patch", "e2el_mean_ms": None}, + ] + block = state.to_latency_budget_summary() + assert "2 winner(s)" in block + assert "cpx-2-streams (explore): 1211 ms" in block + # An untimed refusal must not read as a measured one. + assert "qpx-4 (integrate_patch): not measured" in block + + def test_the_list_is_capped_and_says_so(self): + state = SharedState(latency_budget_ms=250.0) + state.latency_refusals = [ + {"variant_name": f"v{i}", "action": "explore", "e2el_mean_ms": 900.0} for i in range(8) + ] + block = state.to_latency_budget_summary() + assert "8 winner(s)" in block + assert "(+3 more elided" in block diff --git a/src/hyperloom/orchestrator/actions/executors/explore.py b/src/hyperloom/orchestrator/actions/executors/explore.py index 2f83f53f6e..a2c05b859a 100644 --- a/src/hyperloom/orchestrator/actions/executors/explore.py +++ b/src/hyperloom/orchestrator/actions/executors/explore.py @@ -1250,6 +1250,8 @@ def _stopped_by_the_run(result: Any, *, variant: GridVariant, idx: int, round_la "total_throughput": r.total_token_throughput, GRADED_INTVTY: r.intvty_p90, "tpot_p90_ms": r.tpot_p90_ms, + # Graded against the session latency budget when one is set. + "e2el_mean_ms": r.e2el_mean_ms, } graded = resolve_graded_comparison( ss, @@ -1290,7 +1292,13 @@ def _stopped_by_the_run(result: Any, *, variant: GridVariant, idx: int, round_la elif graded.verdict == VERDICT_REVERT: gain = None outcome = "REVERT" - if _graded_on_intvty: + if graded.veto_reason: + # The variant is refused a round earlier than the + # promotion gate would, so it is never folded onto the + # stack and never becomes the anchor the rest of the + # batch is graded against. + reason = graded.veto_reason + elif _graded_on_intvty: reason = f"both_axes_regressed ({axes})" else: reason = "gain_below_threshold" @@ -1306,9 +1314,15 @@ def _stopped_by_the_run(result: Any, *, variant: GridVariant, idx: int, round_la # all, which is why no row is appended then. decision_gates.append( { - "gate": "graded_axes" - if (_graded_on_intvty or graded.degrade_reason) - else "keep_threshold", + "gate": ( + "latency_budget" + if graded.veto_reason + else ( + "graded_axes" + if (_graded_on_intvty or graded.degrade_reason) + else "keep_threshold" + ) + ), "passed": ( False if graded.degrade_reason diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index 43d1b08d0b..47a542b13f 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -4525,6 +4525,9 @@ async def _bench_patch( # the emitted keys stay ``ttft_ms`` / ``itl_ms`` for the collectors. "ttft_ms": r.ttft_mean_ms, "itl_ms": r.tpot_mean_ms, + # Canonical name: the latency budget fails closed, so a lane that + # does not carry this refuses every KEEP it would ever have made. + "e2el_mean_ms": r.e2el_mean_ms, # Benchmark dir; ``_grade_accuracy`` locates accuracy artifacts here. "workspace": r.workspace or "", "error": r.error or "", diff --git a/src/hyperloom/orchestrator/loop/conversation.py b/src/hyperloom/orchestrator/loop/conversation.py index 9064036b94..c1c49b72fc 100644 --- a/src/hyperloom/orchestrator/loop/conversation.py +++ b/src/hyperloom/orchestrator/loop/conversation.py @@ -319,6 +319,13 @@ async def _compose_prompt(self, agent_name: str) -> str: sections.append("=== Shared session state ===") sections.append(self.shared_state.to_prompt_summary()) + # Not wrapped, unlike the advisory blocks below: a latency budget changes + # what a KEEP means, so losing it silently would have the model route as + # if the session were unconstrained. Pure string assembly, no I/O. + latency_block = self.shared_state.to_latency_budget_summary() + if latency_block: + sections.append("=== Latency budget (constraint) ===") + sections.append(latency_block) # Resource pools are orchestration-only; robustness cannot schedule GPU work. if agent_name != "robustness": sections.append("=== Resource pools ===") diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index bf0b351064..96b4d4a3bf 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -3082,6 +3082,29 @@ async def _harvest_specialist_findings(self, done_payload: dict[str, Any]) -> No len(self.shared_state.research_scout_seen_pr_ids or []), ) + def _record_latency_refusal(self, task_kind: str, bv: Any, best_tput: float, reason: str) -> None: + """Log and record a winner the latency budget refused.""" + observed = bv.get("e2el_mean_ms") if isinstance(bv, dict) else None + log.warning( + "current_best held: %s winner measured %.1f tput but %s (budget %.1f ms, measured %s)", + task_kind, + float(best_tput), + reason, + float(self.shared_state.latency_budget_ms or 0.0), + f"{float(observed):.1f} ms" if isinstance(observed, (int, float)) else "nothing", + ) + self.shared_state.latency_refusals.append( + { + "action": task_kind, + "variant_name": str((bv.get("name") if isinstance(bv, dict) else "") or ""), + "tput": float(best_tput), + "e2el_mean_ms": observed if isinstance(observed, (int, float)) else None, + "budget_ms": float(self.shared_state.latency_budget_ms or 0.0), + "reason": reason, + "ts": datetime.now(timezone.utc).isoformat(), + } + ) + def _lift_to_current_best( self, task_kind: str, @@ -3144,6 +3167,9 @@ def _lift_to_current_best( graded.degrade_reason, ) return False + if graded.veto_reason: + self._record_latency_refusal(task_kind, bv, best_tput, graded.veto_reason) + return False if graded.graded_on_intvty and graded.verdict != VERDICT_KEEP: log.info( "current_best held: %s winner %s intvty %.1f->%.1f tput %.1f->%.1f", @@ -3730,6 +3756,29 @@ async def _promote_baseline( # Reads the current_best just assigned, so it has to follow it. self._stamp_current_best_measurement(result) changed = True + # The reference the run is measured against is itself over the SLA, so + # nothing that follows can clear it. Stopping here costs one baseline; + # continuing spends the whole --max-hours refusing every candidate to + # learn something already knowable. + from hyperloom.common.perf_metric import latency_veto_reason + + baseline_veto = latency_veto_reason( + result.get("e2el_mean_ms"), + float(self.shared_state.latency_budget_ms or 0.0), + ) + if baseline_veto: + log.error( + "baseline does not satisfy --max-latency-ms (%s): budget %.1f ms, baseline %s. " + "No candidate can clear a ceiling the reference already breaks; stopping.", + baseline_veto, + float(self.shared_state.latency_budget_ms or 0.0), + ( + f"{float(result['e2el_mean_ms']):.1f} ms" + if isinstance(result.get("e2el_mean_ms"), (int, float)) + else "reported no end-to-end latency" + ), + ) + self.shared_state.set_stop_reason("baseline_over_latency_budget") if anchor_accepted: audit_decision = "promoted" elif isinstance(tput, (int, float)) and tput > 0: @@ -5508,6 +5557,9 @@ def _replay_keep_from_result(self, kind: str, result: dict[str, Any]) -> bool: "extra_envs": dict(result.get("extra_envs_applied") or {}), "tput": float(tput), **graded_axes_of(result.get("bench_result") or result), + # ``graded_axes_of`` carries the throughput axes only; the latency + # budget grades on this one and fails closed without it. + "e2el_mean_ms": (result.get("bench_result") or result).get("e2el_mean_ms"), "workspace": result.get("workspace"), "provenance": provenance or "integrate_patch", "scope": "source_patch", diff --git a/src/hyperloom/orchestrator/phases/kernel.py b/src/hyperloom/orchestrator/phases/kernel.py index 6c15b453bc..1874587b92 100644 --- a/src/hyperloom/orchestrator/phases/kernel.py +++ b/src/hyperloom/orchestrator/phases/kernel.py @@ -2021,6 +2021,10 @@ def _promote_geak_from_candidate( "lever_kind": LEVER_KERNEL if kernel_proven else LEVER_CONFIG, "ttft_mean_ms": result.get("ttft_ms"), "tpot_mean_ms": result.get("tpot_ms"), + # Canonical name, read from the measurement the axes are graded from rather than from ``result``: a + # recheck supersedes the original figures, and pairing this round's throughput with a previous round's + # latency is the divergence the budget exists to catch. Absent here, the budget fails closed. + "e2el_mean_ms": graded_measurement.get("e2el_mean_ms", graded_measurement.get("e2el_ms")), **graded_axes_of(graded_measurement), "workspace": result.get("eval_dir"), } diff --git a/src/hyperloom/orchestrator/phases/machine_state.py b/src/hyperloom/orchestrator/phases/machine_state.py index c0b77d62bf..889519b9fe 100644 --- a/src/hyperloom/orchestrator/phases/machine_state.py +++ b/src/hyperloom/orchestrator/phases/machine_state.py @@ -189,6 +189,7 @@ def render_phase_action_bullets( "custom", "robustness_escalated", "prelude_baseline_failed", + "baseline_over_latency_budget", "prelude_cold_anchor_low_budget", "time_exhausted_during_prelude", "warm_replay_rollback_failed", diff --git a/src/hyperloom/orchestrator/phases/prelude.py b/src/hyperloom/orchestrator/phases/prelude.py index 2bd76cf955..43a1c36e1f 100644 --- a/src/hyperloom/orchestrator/phases/prelude.py +++ b/src/hyperloom/orchestrator/phases/prelude.py @@ -2421,6 +2421,8 @@ def _settle_warm_replay( { "name": "warm_replay", **graded_axes_of(result), + # The latency budget grades on this and fails closed without it. + "e2el_mean_ms": result.get("e2el_mean_ms"), "candidate_extra_server_args": warm_args, "candidate_extra_envs": warm_envs, "recipe_delta": { diff --git a/src/hyperloom/orchestrator/prompts/orchestration.md b/src/hyperloom/orchestrator/prompts/orchestration.md index c3a9ba789a..081b2f9bec 100644 --- a/src/hyperloom/orchestrator/prompts/orchestration.md +++ b/src/hyperloom/orchestrator/prompts/orchestration.md @@ -355,6 +355,13 @@ the code actually is; SESSION CONTEXT names the tree this session optimises `explore` round to refresh the validated gain. The legacy `validate_stack` / `backends` / `params` action names are not in any phase's proposable set (use `explore`). +* **A latency budget changes what a KEEP means.** When + `=== Latency budget (constraint) ===` is present, a throughput gain no + longer predicts a KEEP: any winner over the ceiling is refused, as is one + that reported no end-to-end latency. Read the refusal list before + concluding the search is exhausted — a list that keeps growing means the + SLA is the binding limit, and the answer is a lever that buys throughput + without spending per-request latency, not more of the same. * **Config vs source patch.** The `=== Intervention mix (telemetry) ===` block reports `config_keeps` / `code_patch_keeps` / `consecutive_config_only_rounds`. Config tuning tends to plateau; when diff --git a/src/hyperloom/orchestrator/specialists/rebench.py b/src/hyperloom/orchestrator/specialists/rebench.py index 490888b2e9..8240c84e59 100644 --- a/src/hyperloom/orchestrator/specialists/rebench.py +++ b/src/hyperloom/orchestrator/specialists/rebench.py @@ -130,6 +130,8 @@ async def run_specialist_rebench( # ``itl_ms`` for the collectors. "ttft_ms": rb.ttft_mean_ms, "itl_ms": rb.tpot_mean_ms, + # Canonical name: the latency budget fails closed, so a lane that does not carry this refuses every KEEP. + "e2el_mean_ms": rb.e2el_mean_ms, "workspace": str(getattr(rb, "workspace", "") or ""), "port": resolved_port, "gpu_ids": gpu_ids, diff --git a/src/hyperloom/orchestrator/state/_shared_state/render.py b/src/hyperloom/orchestrator/state/_shared_state/render.py index 99a9d875e3..ff39e00987 100644 --- a/src/hyperloom/orchestrator/state/_shared_state/render.py +++ b/src/hyperloom/orchestrator/state/_shared_state/render.py @@ -276,6 +276,36 @@ def to_phase_budget_telemetry( lines.append(f" {phase}: elapsed={int(elapsed)}s {cap_line} used={used_pct:.0f}%") return "\n".join(lines) or "(no phase history yet)" + def to_latency_budget_summary(self, *, max_rows: int = 5) -> str: + """Render the ``=== Latency budget (constraint) ===`` block; empty when unset. + + Under a budget a throughput gain no longer predicts a KEEP, so the router + needs the constraint and the winners it has refused. A refusal list that + keeps growing means the SLA is the binding limit rather than an exhausted + search space, and the two call for opposite responses. + """ + budget = float(getattr(self, "latency_budget_ms", 0.0) or 0.0) + if budget <= 0: + return "" + refusals = [r for r in (self.latency_refusals or []) if isinstance(r, dict)] + lines = [ + f"budget : {budget:g} ms mean end-to-end, enforced on every KEEP", + "unmeasured: refused (a constraint that was not measured is not satisfied)", + ] + if not refusals: + lines.append("refused : none so far") + return "\n".join(lines) + lines.append(f"refused : {len(refusals)} winner(s) so far") + for row in refusals[-max_rows:]: + observed = row.get("e2el_mean_ms") + measured = f"{float(observed):.0f} ms" if isinstance(observed, (int, float)) else "not measured" + name = str(row.get("variant_name") or "?") + action = str(row.get("action") or "?") + lines.append(f" - {name} ({action}): {measured}") + if len(refusals) > max_rows: + lines.append(f" - (+{len(refusals) - max_rows} more elided; see state.json `latency_refusals`)") + return "\n".join(lines) + def to_resource_pools_summary(self) -> str: """Render the GPU pool / lane capacity block.""" from ...bus.storage.schema import DEFAULT_LANE_CAPACITIES diff --git a/src/hyperloom/orchestrator/state/shared_state.py b/src/hyperloom/orchestrator/state/shared_state.py index bd5c6f65b7..49305aed4f 100644 --- a/src/hyperloom/orchestrator/state/shared_state.py +++ b/src/hyperloom/orchestrator/state/shared_state.py @@ -139,6 +139,7 @@ def resolve_graded_comparison( VERDICT_RECORDED, VERDICT_REVERT, intvty_of, + latency_veto_reason, output_tput_of, passes_intvty_gate, passes_tput_guard, @@ -148,6 +149,13 @@ def resolve_graded_comparison( ) on_intvty, noise_pct = resolved_grading(state) + # The session's latency ceiling is a constraint on the same verdict the gain + # gates decide, not a second opinion beside it: a candidate that clears its + # objective and breaks the SLA is a REVERT, on whichever axis graded it. + sla_veto = latency_veto_reason( + measurement.get("e2el_mean_ms") if isinstance(measurement, Mapping) else None, + float(getattr(state, "latency_budget_ms", 0.0) or 0.0), + ) degrade_reason = "" if on_intvty: if anchor_perf is not None: @@ -179,9 +187,10 @@ def resolve_graded_comparison( objective=GRADED_INTVTY, candidate=intvty_of(cand_perf), reference=intvty_of(ref_perf), - verdict=verdict, + verdict=VERDICT_REVERT if sla_veto else verdict, tput_candidate=total_tput_of(cand_perf), tput_reference=total_tput_of(ref_perf), + veto_reason=sla_veto, ) degrade_reason = reason or "candidate_axes_missing" @@ -203,8 +212,9 @@ def resolve_graded_comparison( objective=GRADED_OUTPUT, candidate=candidate, reference=reference, - verdict=verdict, + verdict=VERDICT_REVERT if sla_veto else verdict, degrade_reason=degrade_reason, + veto_reason=sla_veto, ) @@ -501,6 +511,12 @@ class SharedState(_RenderMixin, _ExploreStateMixin): conc_sweep_variant_timeout_sec: int = 1800 target_summary: str = "" baseline_tput: float = 0.0 + # Ceiling on mean end-to-end latency (ms) from ``--max-latency-ms``; 0.0 leaves KEEP behaviour unchanged. The + # only copy of the budget: it is written once at launch and archived with the session, so a resume restores it. + latency_budget_ms: float = 0.0 + # Winners the budget refused: {action, variant_name, tput, e2el_mean_ms, budget_ms, reason, ts}. A constrained + # session that ends near baseline is otherwise indistinguishable from one that found no headroom. + latency_refusals: list[dict[str, Any]] = field(default_factory=list) # AgentX corpus shape: written at seed from canonical constants, overwritten with measured values after every # AgentX measurement. Read by semantic consumers (prompts, manifest, reports) instead of the inert state.isl / # state.osl placeholders. Absent on synthetic sessions. From b8a543b17ac2f942c73a69559c7666aaf793713b Mon Sep 17 00:00:00 2001 From: Rajesh Poornachandran Date: Thu, 10 Sep 2026 18:56:24 +0000 Subject: [PATCH 2/3] fix(report): explain the new stop_reason to the operator who hit it A terminal the report cannot explain is a run that ends with a token and no account of itself, which is what the vocab test guards. Names what the stop means and what to do next: the ceiling may be under this workload's floor on this hardware, or the baseline configuration is itself the thing to fix. Co-authored-by: Cursor --- src/hyperloom/orchestrator/actions/executors/report.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/hyperloom/orchestrator/actions/executors/report.py b/src/hyperloom/orchestrator/actions/executors/report.py index 3fb45ced7f..fc52283872 100644 --- a/src/hyperloom/orchestrator/actions/executors/report.py +++ b/src/hyperloom/orchestrator/actions/executors/report.py @@ -271,6 +271,14 @@ def _build_failure_summary( "an improvement over a baseline that was never the baseline, so the run stopped with the figure kept and " "marked. Resume with more budget to measure a comparable baseline." ), + "baseline_over_latency_budget": ( + "The baseline's own mean end-to-end latency exceeded --max-latency-ms, so the run stopped before " + "optimizing. The budget refuses any KEEP over the ceiling, and the reference the run is measured against " + "already breaks it — no candidate built on it could have been promoted, so continuing would have spent the " + "whole time budget refusing every winner in turn. Either the ceiling is lower than this workload's floor on " + "this hardware, or the baseline configuration itself is the thing to fix; relaunch with a ceiling the " + "baseline can meet, or without one, to see what the search finds." + ), # Recipe KB knowledge-plane bootstrap failures. "warm_replay_rollback_failed": ( "Warm replay rollback could not restore every Recipe/Kernel mutation; " From 6d6425de18c9d15c114e93feb0aaa4fad5413679 Mon Sep 17 00:00:00 2001 From: Rajesh Poornachandran Date: Sun, 13 Sep 2026 03:19:03 +0000 Subject: [PATCH 3/3] fix(keep): read the budget without the falsy idiom `latency_budget_ms or 0.0` is the pattern the no-falsy-budget rehearsal rule exists to catch: a budget tested for truth reads zero as absent. Harmless here only because zero happens to mean "off", which is exactly the coincidence the rule refuses to rely on. The field is a plain float with a 0.0 default, so read it. Co-authored-by: Cursor --- src/hyperloom/orchestrator/loop/writeback.py | 8 ++++---- src/hyperloom/orchestrator/state/_shared_state/render.py | 2 +- src/hyperloom/orchestrator/state/shared_state.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index 96b4d4a3bf..b12ed043ff 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -3090,7 +3090,7 @@ def _record_latency_refusal(self, task_kind: str, bv: Any, best_tput: float, rea task_kind, float(best_tput), reason, - float(self.shared_state.latency_budget_ms or 0.0), + float(self.shared_state.latency_budget_ms), f"{float(observed):.1f} ms" if isinstance(observed, (int, float)) else "nothing", ) self.shared_state.latency_refusals.append( @@ -3099,7 +3099,7 @@ def _record_latency_refusal(self, task_kind: str, bv: Any, best_tput: float, rea "variant_name": str((bv.get("name") if isinstance(bv, dict) else "") or ""), "tput": float(best_tput), "e2el_mean_ms": observed if isinstance(observed, (int, float)) else None, - "budget_ms": float(self.shared_state.latency_budget_ms or 0.0), + "budget_ms": float(self.shared_state.latency_budget_ms), "reason": reason, "ts": datetime.now(timezone.utc).isoformat(), } @@ -3764,14 +3764,14 @@ async def _promote_baseline( baseline_veto = latency_veto_reason( result.get("e2el_mean_ms"), - float(self.shared_state.latency_budget_ms or 0.0), + float(self.shared_state.latency_budget_ms), ) if baseline_veto: log.error( "baseline does not satisfy --max-latency-ms (%s): budget %.1f ms, baseline %s. " "No candidate can clear a ceiling the reference already breaks; stopping.", baseline_veto, - float(self.shared_state.latency_budget_ms or 0.0), + float(self.shared_state.latency_budget_ms), ( f"{float(result['e2el_mean_ms']):.1f} ms" if isinstance(result.get("e2el_mean_ms"), (int, float)) diff --git a/src/hyperloom/orchestrator/state/_shared_state/render.py b/src/hyperloom/orchestrator/state/_shared_state/render.py index ff39e00987..e8f267c177 100644 --- a/src/hyperloom/orchestrator/state/_shared_state/render.py +++ b/src/hyperloom/orchestrator/state/_shared_state/render.py @@ -284,7 +284,7 @@ def to_latency_budget_summary(self, *, max_rows: int = 5) -> str: keeps growing means the SLA is the binding limit rather than an exhausted search space, and the two call for opposite responses. """ - budget = float(getattr(self, "latency_budget_ms", 0.0) or 0.0) + budget = float(getattr(self, "latency_budget_ms", 0.0)) if budget <= 0: return "" refusals = [r for r in (self.latency_refusals or []) if isinstance(r, dict)] diff --git a/src/hyperloom/orchestrator/state/shared_state.py b/src/hyperloom/orchestrator/state/shared_state.py index 49305aed4f..17926390d9 100644 --- a/src/hyperloom/orchestrator/state/shared_state.py +++ b/src/hyperloom/orchestrator/state/shared_state.py @@ -154,7 +154,7 @@ def resolve_graded_comparison( # objective and breaks the SLA is a REVERT, on whichever axis graded it. sla_veto = latency_veto_reason( measurement.get("e2el_mean_ms") if isinstance(measurement, Mapping) else None, - float(getattr(state, "latency_budget_ms", 0.0) or 0.0), + float(getattr(state, "latency_budget_ms", 0.0)), ) degrade_reason = "" if on_intvty: