diff --git a/docs/reference/session-breakdown.md b/docs/reference/session-breakdown.md index 05756ccb84..2228d9fd04 100644 --- a/docs/reference/session-breakdown.md +++ b/docs/reference/session-breakdown.md @@ -136,8 +136,8 @@ started, …). ## `metadata` — `V6Metadata` Task identity, recorded as each fact is decided rather than re-derived at -export. Four blocks: `session`, `task_config`, `versions` and `langfuse`, plus -the export's own `exported_at_utc` and `warnings`. +export. Five blocks: `session`, `task_config`, `grading`, `versions` and +`langfuse`, plus the export's own `exported_at_utc` and `warnings`. `metadata.session` — identity and lifecycle: @@ -171,6 +171,24 @@ treat the `objective.kind` enum as the canonical optimisation goal. Its `architecture` sub-object is the structural model summary parsed from the model's own `config.json`, and is empty on non-transformers models. +`metadata.grading` — which axis this session was configured to grade on: +`benchmark_mode` (`agentx` or `synthetic`), `objective`, and the `tput_guard` +that rides along with the interactivity objective (`enabled`, `noise_pct`). + +An AgentX replay is ranked on the slow-tail interactivity percentile +(`e2e_norm_intvty_p90`) with total throughput held as a guard; a synthetic run +is ranked on output throughput alone. Every throughput field elsewhere in this +document is the output axis by construction, so without this block a consumer +cannot tell the two kinds of session apart — and on the canonical corpus the +two axes differ by roughly two orders of magnitude. + +This is the session-level *setting*. What the run actually decided a given +promotion on is `outcome.validation.graded_on`, read off the promotion itself: +a session configured for interactivity still grades an individual comparison +on output whenever either side of it cannot supply the axis pair. Neither field +resolves the other. `tput_guard.noise_pct` is null on a session that predates +the band being recorded. + `metadata.versions` — the schema version, the Hyperloom revision, the framework and its version, and a `tools` map carrying `{tool, root_dir, commit, version}` per external tool. @@ -205,6 +223,14 @@ the exact baseline benchmark. `extra_envs` is allowlist-filtered to keep secrets out of the breakdown. Do not assume it contains every env var the session ran with. +`baseline.perf` and `final.perf` carry the four AgentX axes the measurement +reported — `e2e_norm_intvty_p90`, `total_throughput`, `input_throughput`, +`tpot_p90_ms` — each an explicit `null` where nothing measured it. Absent would +be indistinguishable from an axis the framework failed to report, and zero +reads as "measured, and it was zero", so a synthetic run publishes four nulls. +`final.graded_on` names the axis `final.gain_pct` is on, and always agrees with +`outcome.validation.graded_on`: they are the same figure read twice. + --- ## `outcome.final` — `Final` (SaFE contract core) @@ -237,7 +263,11 @@ downstream consumers: `outcome` is the terminal result: `status`, `stop_reason`, `stage_reached`, the `baseline` and `final` blocks documented above, and the `validation` -block that reconciles the optimization stack's parts against its total. +block that reconciles the optimization stack's parts against its total. That +reconciliation is single-axis and `validation.graded_on` names the axis: an +attributed figure on one axis against an unattributed figure on another makes +the gap meaningless. `validation.notes` reports any adoption that fell off +that axis, because its contribution sits in the same sum as the rest. `timeline` is the run itself — one event per stage, oldest first. An event carries its `type`, its identity (`event_id`, `phase`, `macro_cycle`), its @@ -369,6 +399,11 @@ The following example shows a complete `session_breakdown.json` for a finished G "launch_server_args": "", "architecture": { "model_class": "moe_mla_nsa", "model_type": "glm5", "is_moe": true } }, + "grading": { + "benchmark_mode": "synthetic", + "objective": "output_throughput", + "tput_guard": { "enabled": false, "noise_pct": 5.0 } + }, "langfuse": { "enabled": false, "disabled_reason": "no_credentials", "trace_url": null, "counts": {} }, "warnings": [] }, @@ -383,6 +418,12 @@ The following example shows a complete `session_breakdown.json` for a finished G "accuracy": 0.812, "ttft_mean_ms": 0.0, "e2el_mean_ms": 0.0, + "perf": { + "e2e_norm_intvty_p90": null, + "total_throughput": null, + "input_throughput": null, + "tpot_p90_ms": null + }, "ttft_e2el_source": "state_workspace", "config_path": "runs/baseline/baseline_config.with_envs.yaml", "benchmark_report_path": "runs/baseline/report.json", diff --git a/src/hyperloom/common/gain_math.py b/src/hyperloom/common/gain_math.py index 9a1b44d6b8..f544df9844 100644 --- a/src/hyperloom/common/gain_math.py +++ b/src/hyperloom/common/gain_math.py @@ -8,6 +8,7 @@ from typing import Any from hyperloom.common.coerce import to_float +from hyperloom.common.perf_metric import GRADED_INTVTY, GRADED_TOTAL, graded_axes_of, passes_tput_guard def gain_pct(new: float | None, base: float) -> float | None: @@ -37,8 +38,18 @@ def conc_pair_comparison( optimized_points: list[dict[str, Any]], *, metric_key: str = "output_throughput", + guard_noise_pct: float | None = None, ) -> tuple[list[dict[str, Any]], dict[str, Any]]: - """Pair curve points by CONC (outer join), compute per-conc speedup, and aggregate.""" + """Pair curve points by CONC (outer join), compute per-conc speedup on *metric_key*, and aggregate. + + Under the interactivity objective each pair also reports whether throughput held within the noise band the + session grades under. It is reported, not enforced: InferenceX publishes a 2-D frontier with no fixed + interactivity target, so a rung that traded throughput for interactivity moved along that frontier rather than + violating a constraint -- and a sweep exists to draw the frontier. Gating on the guard here would drop half the + curve. The KEEP path enforces it because a stack promotion at one concurrency is a different question. + """ + # The guard belongs to the interactivity objective; on the output axis there is no second axis to hold. + guard_axis = GRADED_TOTAL if metric_key == GRADED_INTVTY else "" def _norm_conc(p: dict[str, Any]) -> int | float | str: raw = p.get("conc") @@ -68,23 +79,42 @@ def _norm_conc(p: dict[str, Any]) -> int | float | str: successful_pairs += 1 else: failed_pairs += 1 + # ``graded_axes_of`` is what normalises the sweep's ``total_token_throughput`` onto GRADED_TOTAL, so the + # guard cannot read a differently-named axis as an absent one. + base_axes = graded_axes_of(b) if guard_axis else {} + opt_axes = graded_axes_of(o) if guard_axis else {} + guard_holds: bool | None = None + if guard_axis and base_axes.get(guard_axis) and opt_axes.get(guard_axis): + guard_holds = passes_tput_guard(opt_axes, base_axes, noise_pct=guard_noise_pct) rows.append( { "conc": c, - "baseline_tput": bt, - "optimized_tput": ot, + # Named for the axis rather than for throughput: under the interactivity objective these hold a + # slow-tail percentile, and ``summary.metric`` is what says which. + "baseline_value": bt, + "optimized_value": ot, "speedup": speedup, "delta_pct": delta_pct, "baseline_status": b.get("status"), "optimized_status": o.get("status"), + # The guard axis beside the objective, so the frontier this rung sits on is readable rather than + # only the one number it was ranked by. Null off the interactivity objective, and null when a side + # did not measure the axis -- which is not the same as a rung that measured it and fell outside. + "baseline_guard": base_axes.get(guard_axis) if guard_axis else None, + "optimized_guard": opt_axes.get(guard_axis) if guard_axis else None, + "guard_holds": guard_holds, } ) summary: dict[str, Any] = { "metric": metric_key, + # The axis held beside the objective, empty off the interactivity objective. Named here so a reader of + # ``best_conc_guard_holds`` does not have to infer which axis the verdict is about. + "guard_axis": guard_axis, "successful_pairs": successful_pairs, "failed_pairs": failed_pairs, "best_conc": None, "best_speedup": None, + "best_conc_guard_holds": None, "median_speedup": None, "mean_speedup": None, } @@ -100,6 +130,9 @@ def _norm_conc(p: dict[str, Any]) -> int | float | str: { "best_conc": rows[best_idx]["conc"], "best_speedup": round(best_val, 4), + # The headline rung is the best on the objective alone. Whether the session's own KEEP rule would + # have accepted it is a second fact, and the two disagreeing is worth seeing rather than resolving. + "best_conc_guard_holds": rows[best_idx]["guard_holds"], "median_speedup": round(median, 4), "mean_speedup": round(sum(speedups) / len(speedups), 4), } diff --git a/src/hyperloom/common/perf_metric.py b/src/hyperloom/common/perf_metric.py index 3abfb4b667..326c2c95a5 100644 --- a/src/hyperloom/common/perf_metric.py +++ b/src/hyperloom/common/perf_metric.py @@ -26,6 +26,11 @@ GRADED_TOTAL = "total_throughput" GRADED_OUTPUT = "output_throughput" +# The axes ``graded_axes_of`` can carry, for a consumer that must publish all four including the ones a measurement +# did not supply. Absent and null are not the same fact: a recorder that omits an axis leaves a reader unable to tell +# an unmeasured axis from one the framework failed to report, and zero reads as "measured, and it was zero". +GRADED_AXIS_KEYS = (GRADED_INTVTY, GRADED_TOTAL, "input_throughput", "tpot_p90_ms") + # Upstream reports run-to-run noise on this workload as 1-5% depending on the concurrency regime, so the band opens # to the top of that range instead of rejecting movement upstream would call noise. _DEFAULT_INTVTY_NOISE_PCT = 5.0 @@ -237,6 +242,7 @@ def graded_on_intvty(self) -> bool: __all__ = [ "AGENTX_KEEP_THRESHOLD_FLOOR_PCT", "GradedComparison", + "GRADED_AXIS_KEYS", "GRADED_INTVTY", "GRADED_OUTPUT", "GRADED_TOTAL", diff --git a/src/hyperloom/inference_optimizer/breakdown/SKILL.md b/src/hyperloom/inference_optimizer/breakdown/SKILL.md index 091a08631b..98abb19461 100644 --- a/src/hyperloom/inference_optimizer/breakdown/SKILL.md +++ b/src/hyperloom/inference_optimizer/breakdown/SKILL.md @@ -32,8 +32,8 @@ authoritative. | `schema_version` | The wire contract. Gate features on the **major** version, not the exact string. | | `exported_at_utc` | When this export was built. | | `exporter_version` | Which exporter built it. | -| `metadata` | Session identity, launch configuration, component versions, Langfuse receipt, and `warnings` -- how the export itself went, reported once and only here. | -| `outcome` | Terminal status, stage reached, stop reason, the `baseline` and `final` measured results, and the `validation` that reconciles the stack's parts against its total. | +| `metadata` | Session identity, launch configuration, `grading` -- the axis this session was configured to rank on -- component versions, Langfuse receipt, and `warnings`, how the export itself went, reported once and only here. | +| `outcome` | Terminal status, stage reached, stop reason, the `baseline` and `final` measured results with the graded axes each reported, and the `validation` that reconciles the stack's parts against its total on one axis, named by `graded_on`. | | `timeline` | The run itself: one event per stage, oldest first, each carrying its span, status, and an `ext` block of what that kind of stage records. Startup source events live under `reports/sbd_v6/timeline/`. | | `close` | What the session settled at close: the steps the sequencer ran, the artifacts it published, the robustness findings it collected. | | `critic` | The critic agent's own run, iteration by iteration: what it was asked about, how its rulings fell (`verdict_counts`), and the four artifacts each pass left behind. Per-proposal verdicts stay with the proposals, on the timeline. | @@ -122,7 +122,7 @@ this reference is partial — `breakdown/exporter.py` is authoritative. | Section | Read from | |-------------|-----------------------------------------------------------------------------------------------------------------| -| `metadata` | `manifest.json` + `state.json`, overlaid by the recorder's own `session` / `task_config` / `versions` fragments | +| `metadata` | `manifest.json` + `state.json`, overlaid by the recorder's own `session` / `task_config` / `grading` / `versions` fragments. `grading` is recorded only: the axis is resolved at seed, where the run can still see its own configuration, and re-deriving it here would read the exporting subprocess's environment | | `outcome` | The recorder's `close` and stack fragments, plus `state.{current_best, cumulative_gain_validated, optimization_stack}` for the sessions that predate them | | `timeline` | The event fragments in the spool, closed and assembled per event; orphans left open by a killed phase are closed on first build | | `close` | The CLOSE sequencer's own `close` / `close_step` fragments | diff --git a/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py b/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py index a2d1b3486d..807508e7a9 100644 --- a/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py +++ b/src/hyperloom/inference_optimizer/breakdown/collectors/v6.py @@ -349,6 +349,25 @@ def _stage_reached( _ANCHORING_BASELINE_STATUSES = frozenset({"succeeded", "degraded"}) +def _graded_axes(recorded: Any) -> dict[str, Any]: + """Publish the four graded axes a recorder projected, absent ones as explicit nulls. + + The recorder already filled all four, so this only has to hold the shape for a session recorded before it did. + All four are always present because absent would be indistinguishable from an axis the framework failed to + report, and zero reads as "measured, and it was zero". + + Args: + recorded (Any): The recorded ``perf`` block, or ``None`` on a session that has none. + + Returns: + dict[str, Any]: The four axes, each ``None`` where nothing measured it. + """ + from hyperloom.common.perf_metric import GRADED_AXIS_KEYS + + source = _mapping(recorded) + return {key: _optional_float(source.get(key)) for key in GRADED_AXIS_KEYS} + + def _baseline_from_timeline(timeline: list[dict[str, Any]]) -> dict[str, Any]: """Read the session's anchoring baseline off the ``baseline`` events. @@ -382,10 +401,13 @@ def _baseline_from_timeline(timeline: list[dict[str, Any]]) -> dict[str, Any]: # no chronology of its own. anchors.append((str(action.get("end_time") or action.get("start_time") or ""), action)) if not anchors: - return dict.fromkeys(_BASELINE_OUTCOME_FIELDS) + return {**dict.fromkeys(_BASELINE_OUTCOME_FIELDS), "perf": _graded_axes(None)} anchors.sort(key=lambda row: row[0]) measurement = _mapping(anchors[-1][1].get("measurement")) - return {field: _optional_float(measurement.get(field)) for field in _BASELINE_OUTCOME_FIELDS} + return { + **{field: _optional_float(measurement.get(field)) for field in _BASELINE_OUTCOME_FIELDS}, + "perf": _graded_axes(measurement.get("perf")), + } def _validation_from_timeline(timeline: list[dict[str, Any]]) -> dict[str, Any]: @@ -432,6 +454,15 @@ def _bucket(*names: str) -> dict[str, Any]: validations = _mapping(ledger.get("validations")) settled = _mapping(validations.get("settled")) return { + # The axis every percentage below shares. The reconciliation has to be single-axis: an attributed figure on + # one axis against an unattributed figure on another makes the gap meaningless. Read off the validation row + # that produced the settled figure, falling back to the axis the adoptions were graded on for a session that + # adopted but never validated -- and never from the session's configured axis, which says what was asked + # for rather than what this figure was measured on. + "graded_on": str(settled.get("graded_objective") or ledger.get("objective") or "") or None, + # The settled measurement's own axes, carried beside the gain they produced rather than read off + # ``current_best``: a revalidation moves the cumulative figure without re-promoting the recipe. + "perf": _graded_axes(settled.get("perf")), "attributed_gain_pct": _optional_float(ledger.get("attributed_gain_pct")) or 0.0, "unattributed_gain_pct": _optional_float(ledger.get("unattributed_gain_pct")) or 0.0, "reconciliation_gap_pct": _optional_float(ledger.get("reconciliation_gap_pct")), @@ -501,6 +532,24 @@ def _bucket_of(backends: dict[str, Any], name: str, available: bool) -> dict[str _RECONCILIATION_NOISE_PP = 0.01 +def _degraded_adoptions(ledger: dict[str, Any]) -> dict[str, int]: + """Count the ledger's adoptions per reason their comparison fell off the configured axis. + + Args: + ledger (dict[str, Any]): The stack event's ``ext``. + + Returns: + dict[str, int]: Adoption count per ``degrade_reason``, empty when every adoption graded on the axis the + session asked for. + """ + counts: dict[str, int] = {} + for row in _dict_rows(_mapping(ledger.get("adoptions")).get("rows")): + reason = str(row.get("degrade_reason") or "").strip() + if reason: + counts[reason] = counts.get(reason, 0) + 1 + return counts + + def _validation_notes(ledger: dict[str, Any]) -> list[str]: """Name what the ledger's own figures say is wrong with it. @@ -536,6 +585,16 @@ def _validation_notes(ledger: dict[str, Any]) -> list[str]: "either an adoption is missing from the ledger or its recorded throughputs disagree " "with the end-to-end measurement" ) + degraded = _degraded_adoptions(ledger) + if degraded: + # A session configured for the interactivity axis still grades an individual adoption on output whenever + # either side of that comparison cannot supply the axis pair, and its contribution then sits in the same + # sum as the axis-graded ones. Naming the count is what keeps the total from reading as single-axis. + reasons = ", ".join(sorted(degraded)) + notes.append( + f"{sum(degraded.values())} adoption(s) were graded on the output axis rather than the axis the " + f"session was configured for ({reasons}); their contributions are not on the same axis as the rest" + ) validations = _mapping(ledger.get("validations")) if not _optional_int(validations.get("count")): notes.append("no whole-stack validation was measured, so the ledger has nothing to reconcile against") @@ -590,6 +649,12 @@ def collect_v6_outcome( # session's total means, and asking two sources the same question # is how the export came to publish an answer nothing measured. "gain_pct": validation.get("validated_total_gain_pct") or 0.0, + # The same axis and the same measurement as the gain above, from the one row that produced both. A + # consumer sorting sessions has to be able to tell an interactivity-graded AgentX result from an + # output-graded synthetic one: on the canonical corpus the two axes differ by two orders of magnitude, + # and every other throughput field in this document is the output axis by construction. + "graded_on": validation.get("graded_on"), + "perf": validation.get("perf"), "action_path": [str(step) for step in recipe.get("action_path") or []], "extra_envs": dict(_mapping(recipe.get("extra_envs"))), "extra_server_args": str(recipe.get("extra_server_args") or ""), diff --git a/src/hyperloom/inference_optimizer/breakdown/recorder/baseline_event.py b/src/hyperloom/inference_optimizer/breakdown/recorder/baseline_event.py index 6485859d23..4db63ab78c 100644 --- a/src/hyperloom/inference_optimizer/breakdown/recorder/baseline_event.py +++ b/src/hyperloom/inference_optimizer/breakdown/recorder/baseline_event.py @@ -15,6 +15,7 @@ as_list as _as_list, failure_row as _failure_row, float_or_none as _float_or_none, + graded_axes as _graded_axes, int_or_none as _int_or_none, now_iso_seconds as _now_iso, worst_status as _worst_status, @@ -217,6 +218,10 @@ def _measurement(result: Mapping[str, Any], framework: str) -> dict[str, Any]: # Separate because TPOT alone can be computed from the other two, and # a computed figure must not be read as a measured one. "tpot_source": str(result.get("tpot_source") or ""), + # The graded axes this round measured. Recorded here rather than read off ``state.baseline_perf`` at export + # because this block is already where ``outcome.baseline`` comes from, and a second source for one baseline + # is a second answer to the same question. + "perf": _graded_axes(result), "accuracy": _float_or_none(result.get("accuracy")), "accuracy_task": str(result.get("accuracy_task") or ""), "accuracy_metric": str(result.get("accuracy_metric") or ""), diff --git a/src/hyperloom/inference_optimizer/breakdown/recorder/conc_sweep_event.py b/src/hyperloom/inference_optimizer/breakdown/recorder/conc_sweep_event.py index 27b5ddbbea..12ffeccad5 100644 --- a/src/hyperloom/inference_optimizer/breakdown/recorder/conc_sweep_event.py +++ b/src/hyperloom/inference_optimizer/breakdown/recorder/conc_sweep_event.py @@ -29,6 +29,7 @@ from .event_fields import ( as_dict as _as_dict, as_list as _as_list, + bool_or_none as _bool_or_none, clip as _clip, float_or_none as _float_or_none, int_or_none as _int_or_none, @@ -620,6 +621,13 @@ def record_progress(self, *, comparison: Any, summary: Any) -> None: concurrency, so a later pass revises a row rather than adding one, and a failure reason is settled here because the arm that broke is the only one that can say why. + + ``baseline_value``/``optimized_value`` are on the axis named by + ``result.metric``, which is an interactivity percentile rather than a + throughput whenever the session grades on one. The guard axis rides + along beside them, null off the interactivity objective, so a rung that + bought interactivity by giving up throughput is visible as such instead + of reading as a clean win. """ for row in _as_list(comparison): if not isinstance(row, Mapping): @@ -630,10 +638,13 @@ def record_progress(self, *, comparison: Any, summary: Any) -> None: { "task_id": self._action_id, "conc": rung, - "baseline_throughput": _float_or_none(row.get("baseline_tput")), - "optimized_throughput": _float_or_none(row.get("optimized_tput")), + "baseline_value": _float_or_none(row.get("baseline_value")), + "optimized_value": _float_or_none(row.get("optimized_value")), "speedup": _float_or_none(row.get("speedup")), "delta_pct": _float_or_none(row.get("delta_pct")), + "baseline_guard": _float_or_none(row.get("baseline_guard")), + "optimized_guard": _float_or_none(row.get("optimized_guard")), + "guard_holds": _bool_or_none(row.get("guard_holds")), "baseline_status": _text(row.get("baseline_status")), "optimized_status": _text(row.get("optimized_status")), "error": _pair_error( @@ -650,8 +661,10 @@ def record_progress(self, *, comparison: Any, summary: Any) -> None: { "result": { "metric": _text(roll_up.get("metric")), + "guard_axis": _text(roll_up.get("guard_axis")), "best_conc": _int_or_none(roll_up.get("best_conc")), "best_speedup": _float_or_none(roll_up.get("best_speedup")), + "best_conc_guard_holds": _bool_or_none(roll_up.get("best_conc_guard_holds")), "successful_pairs": _int_or_none(roll_up.get("successful_pairs")), "failed_pairs": _int_or_none(roll_up.get("failed_pairs")), "median_speedup": _float_or_none(roll_up.get("median_speedup")), @@ -697,8 +710,10 @@ def finish(self, payload: Mapping[str, Any] | None, *, stop_reason: Any = None) result = { "status": status, "metric": _text(roll_up.get("metric")), + "guard_axis": _text(roll_up.get("guard_axis")), "best_conc": _int_or_none(roll_up.get("best_conc")), "best_speedup": _float_or_none(roll_up.get("best_speedup")), + "best_conc_guard_holds": _bool_or_none(roll_up.get("best_conc_guard_holds")), "successful_pairs": _int_or_none(roll_up.get("successful_pairs")), "failed_pairs": _int_or_none(roll_up.get("failed_pairs")), "median_speedup": _float_or_none(roll_up.get("median_speedup")), diff --git a/src/hyperloom/inference_optimizer/breakdown/recorder/event_fields.py b/src/hyperloom/inference_optimizer/breakdown/recorder/event_fields.py index 10119f1dd7..56c538bddf 100644 --- a/src/hyperloom/inference_optimizer/breakdown/recorder/event_fields.py +++ b/src/hyperloom/inference_optimizer/breakdown/recorder/event_fields.py @@ -21,10 +21,12 @@ "analysis_detail", "as_dict", "as_list", + "bool_or_none", "bounded_block", "clip", "failure_row", "float_or_none", + "graded_axes", "int_or_none", "now_iso_micros", "now_iso_seconds", @@ -88,6 +90,16 @@ def float_or_none(value: Any) -> float | None: return None +def bool_or_none(value: Any) -> bool | None: + """``bool(value)`` when a value was recorded, else ``None``. + + A tri-state flag needs the coercion to stop at ``None`` rather than fold it + to ``False``: "the framework never answered" and "the answer was no" are + different facts, and ``bool(None)`` erases the difference. + """ + return None if value is None else bool(value) + + def text_or_none(value: Any) -> str | None: """Distinguish "not recorded" from "recorded empty". @@ -100,6 +112,23 @@ def text_or_none(value: Any) -> str | None: return text or None +def graded_axes(source: Any) -> dict[str, Any]: + """The four graded axes a measurement carries, as explicit nulls where it carries none. + + A synthetic run measures none of them and an AgentX round can be missing any one. Absent keys would leave a + reader unable to tell an unmeasured axis from one the framework failed to report, and zero reads as "measured, + and it was zero", so all four are always present. + + Recorded beside a round's output-axis figures rather than instead of them: an AgentX session is ranked on the + slow-tail interactivity percentile with total throughput held as a guard, and none of that is recoverable from + the output axis -- on the canonical corpus the two throughputs differ by roughly two orders of magnitude. + """ + from hyperloom.common.perf_metric import GRADED_AXIS_KEYS, graded_axes_of + + axes = graded_axes_of(source) + return {key: float_or_none(axes.get(key)) for key in GRADED_AXIS_KEYS} + + def summarize_hot_kernels(rows: Any) -> dict[str, Any]: """Project the hot-kernel ranking head into the event.""" candidates = [row for row in as_list(rows) if isinstance(row, dict)] diff --git a/src/hyperloom/inference_optimizer/breakdown/recorder/session_metadata.py b/src/hyperloom/inference_optimizer/breakdown/recorder/session_metadata.py index 9424871ad4..ff8d6049fc 100644 --- a/src/hyperloom/inference_optimizer/breakdown/recorder/session_metadata.py +++ b/src/hyperloom/inference_optimizer/breakdown/recorder/session_metadata.py @@ -182,7 +182,11 @@ def snapshot_metadata(rec: Recorder, state: Any) -> None: "tick_count": int(getattr(state, "tick", 0) or 0), "recovery": _recovery(state), } - payload: dict[str, Any] = {"session": session, "task_config": _launch_config(state)} + payload: dict[str, Any] = { + "session": session, + "task_config": _launch_config(state), + "grading": _grading(state), + } architecture = _architecture( getattr(state, "model_info", None) or {}, model_class=_text(getattr(state, "model_class", "")), @@ -192,6 +196,38 @@ def snapshot_metadata(rec: Recorder, state: Any) -> None: rec.record_upsert_singleton(SECTION, payload) +def _grading(state: Any) -> dict[str, Any]: + """Declare the axis this session was configured to grade on, and the band it grades under. + + An AgentX replay is ranked on the slow-tail interactivity percentile with throughput held as a guard; a synthetic + run is ranked on output throughput alone. On the canonical corpus the two axes differ by roughly two orders of + magnitude, so a consumer that cannot tell them apart will happily sort one against the other -- and nothing else + in this document carries the distinction, because every throughput field in it is the output axis by + construction and ``benchmark_mode`` never reaches the breakdown at all. + + This is the session-level setting and only that. What the run actually decided a given promotion on is a + different fact, recorded on the promotion itself and published as ``outcome.validation.graded_on``: a session + configured for interactivity still grades an individual comparison on output whenever either side of it cannot + supply the axis pair. Resolving one of the two from the other would put a label on a figure it does not describe. + + Read from the live state rather than resolved here, which is why this reaches the export with no environment read + anywhere on the path: ``SharedState.grading`` was resolved once at seed, where the run could still see its own + configuration. + """ + from hyperloom.common.perf_metric import GRADED_INTVTY, GRADED_OUTPUT + from hyperloom.orchestrator.state.shared_state import resolved_grading + + on_intvty, noise_pct = resolved_grading(state) + return { + "benchmark_mode": _text(getattr(state, "benchmark_mode", "")) or "synthetic", + "objective": GRADED_INTVTY if on_intvty else GRADED_OUTPUT, + # The throughput guard that rides along with the interactivity objective. ``noise_pct`` is null on a session + # seeded before the band was recorded: the band it applied is unknown, and today's default is not evidence + # of it. + "tput_guard": {"enabled": on_intvty, "noise_pct": noise_pct}, + } + + def _architecture(model_info: Any, *, model_class: str = "") -> dict[str, Any]: """The structural model block, or ``{}`` when nothing is known.""" info = dict(model_info or {}) if isinstance(model_info, Mapping) else {} diff --git a/src/hyperloom/inference_optimizer/breakdown/recorder/stack_event.py b/src/hyperloom/inference_optimizer/breakdown/recorder/stack_event.py index 4f0e71ef5e..1964d927f5 100644 --- a/src/hyperloom/inference_optimizer/breakdown/recorder/stack_event.py +++ b/src/hyperloom/inference_optimizer/breakdown/recorder/stack_event.py @@ -27,6 +27,7 @@ from .event_fields import ( as_list as _as_list, float_or_none as _float_or_none, + graded_axes as _graded_axes, now_iso_seconds as _now, text_or_none as _text_or_none, ) @@ -249,6 +250,7 @@ def record_validation( source: str = "", measurement_basis: str = "", graded_objective: str = "", + measurement: Mapping[str, Any] | None = None, ts: str = "", ttft_mean_ms: Any = None, e2el_mean_ms: Any = None, @@ -266,11 +268,16 @@ def record_validation( ``stack_len`` keys the row, so a later validation at one length supersedes the earlier. ``measurement_basis`` is ``e2e_rebench`` for a full-stack revalidation or ``e2e_decision_round`` for the round a variant was graded - on. ``graded_objective`` names the axis the figure was measured on, so a - total- or intvty-graded gain is not later read as an output gain. The - latency pair and ``server_launch_flags`` are carried here because the run - that produced ``validated_tput`` resolves them and they cannot be recovered - afterwards. + on. ``graded_objective`` names the axis the figure was measured on, so an + intvty-graded gain is not later read as an output gain; the caller only + records a comparison it found comparable, so this is always the axis the + session was configured for. ``measurement`` is projected to its graded + axes and recorded beside the gain they produced, because a later + revalidation moves the cumulative figure without re-promoting the recipe, + so reading the axes off ``current_best`` at export can pair this gain with + a different measurement. The latency pair and ``server_launch_flags`` are + carried here because the run that produced ``validated_tput`` resolves them + and they cannot be recovered afterwards. """ try: sink = _sink() @@ -288,6 +295,7 @@ def record_validation( "source": str(source or ""), "measurement_basis": str(measurement_basis or ""), "graded_objective": str(graded_objective or ""), + "perf": _graded_axes(measurement), "ttft_mean_ms": _float_or_none(ttft_mean_ms), "e2el_mean_ms": _float_or_none(e2el_mean_ms), "ttft_e2el_source": str(ttft_e2el_source or ""), diff --git a/src/hyperloom/inference_optimizer/breakdown/recorder/warm_start_event.py b/src/hyperloom/inference_optimizer/breakdown/recorder/warm_start_event.py index 3202d3a24e..4efd799484 100644 --- a/src/hyperloom/inference_optimizer/breakdown/recorder/warm_start_event.py +++ b/src/hyperloom/inference_optimizer/breakdown/recorder/warm_start_event.py @@ -28,6 +28,7 @@ from .event_fields import ( as_dict as _as_dict, + bool_or_none as _bool_or_none, float_or_none as _float_or_none, now_iso_seconds as _now, text_or_none as _text_or_none, @@ -435,11 +436,6 @@ def _origin(recipe: Mapping[str, Any]) -> dict[str, Any] | None: return {"session_id": session_id, "gain_pct": gain} -def _bool_or_none(value: Any) -> bool | None: - """``bool(value)`` when a value was recorded, else ``None``.""" - return None if value is None else bool(value) - - __all__ = [ "EVENT_COMPONENT", "EVENT_KIND", diff --git a/src/hyperloom/inference_optimizer/breakdown/schema.py b/src/hyperloom/inference_optimizer/breakdown/schema.py index 3884b56c55..405b0445e4 100644 --- a/src/hyperloom/inference_optimizer/breakdown/schema.py +++ b/src/hyperloom/inference_optimizer/breakdown/schema.py @@ -139,6 +139,48 @@ class V6MetadataLangfuse(TypedDict, total=False): counts: dict[str, int] +class V6GradedAxes(TypedDict, total=False): + """The four axes an AgentX measurement is ranked on. + + Every axis is present on every measurement, ``None`` where nothing measured + it: absent would be indistinguishable from an axis the framework failed to + report, and zero reads as "measured, and it was zero". A synthetic run + carries four nulls. + """ + + e2e_norm_intvty_p90: float | None + total_throughput: float | None + input_throughput: float | None + tpot_p90_ms: float | None + + +class V6GradingTputGuard(TypedDict, total=False): + """The throughput constraint riding along with the interactivity objective. + + ``noise_pct`` is ``None`` on a session seeded before the band was recorded: + the band that session applied is unknown, and today's environment is not + evidence of it. + """ + + enabled: bool + noise_pct: float | None + + +class V6Grading(TypedDict, total=False): + """The axis this session was configured to grade on. + + The session-level setting and only that. What the run actually decided a + given promotion on is ``outcome.validation.graded_on``, read off the + promotion itself: a session configured for interactivity still grades an + individual comparison on output whenever either side of it cannot supply + the axis pair. Neither field resolves the other. + """ + + benchmark_mode: str + objective: str + tput_guard: V6GradingTputGuard + + class V6Metadata(TypedDict, total=False): """V6 task identity, configuration, versions, and trace entrypoint.""" @@ -146,6 +188,7 @@ class V6Metadata(TypedDict, total=False): versions: V6MetadataVersions session: V6MetadataSession task_config: V6TaskConfig + grading: V6Grading langfuse: V6MetadataLangfuse warnings: list[str] @@ -196,6 +239,15 @@ class V6OutcomeValidation(TypedDict, total=False): disagreeing means one of them is wrong. """ + #: The axis every percentage here shares, read off the row that produced + #: the settled figure. The reconciliation has to be single-axis: an + #: attributed figure on one axis against an unattributed figure on another + #: makes the gap meaningless. + graded_on: str | None + #: The settled measurement's own axes, on the same row as the gain they + #: produced -- a revalidation moves the cumulative figure without + #: re-promoting the recipe, so ``current_best`` can be a later measurement. + perf: V6GradedAxes attributed_gain_pct: float unattributed_gain_pct: float chain_total_gain_pct: float | None @@ -534,7 +586,7 @@ class V6ConcSweepPoint(TypedDict, total=False): request_throughput: float | None total_token_throughput: float | None input_throughput: float | None - intvty_p90: float | None + e2e_norm_intvty_p90: float | None tpot_p90_ms: float | None ttft_mean_ms: float | None e2el_mean_ms: float | None @@ -581,13 +633,25 @@ class V6ConcSweepArm(TypedDict, total=False): class V6ConcSweepPair(TypedDict, total=False): - """The two arms joined at one concurrency.""" + """The two arms joined at one concurrency. + + The pair is ranked on one axis and reports a second. ``*_value`` is on the + axis ``result.metric`` names -- a slow-tail interactivity percentile + whenever the session grades on one, which is why these are not named for + throughput. ``*_guard`` and ``guard_holds`` carry the throughput the + session would have held a promotion to, reported rather than enforced: a + sweep exists to draw the interactivity/throughput frontier, so a rung that + moved along it is a result and not a failure. They are null off the + interactivity objective, where there is no second axis to hold.""" conc: int - baseline_throughput: float | None - optimized_throughput: float | None + baseline_value: float | None + optimized_value: float | None speedup: float | None delta_pct: float | None + baseline_guard: float | None + optimized_guard: float | None + guard_holds: bool | None baseline_status: str optimized_status: str error: str | None @@ -939,9 +1003,14 @@ class V6StackValidation(TypedDict, total=False): validated_gain_pct: float | None source: str measurement_basis: str - #: The axis the figure was graded on, so a total- or intvty-graded gain is - #: not later read as an output gain. + #: The axis the figure was graded on, so an intvty-graded gain is not later + #: read as an output gain. Always the axis the session was configured for: + #: only a comparison the orchestrator found comparable is recorded here. graded_objective: str + #: The graded axes of the measurement that produced the figure, recorded + #: beside it because a later revalidation moves the cumulative gain without + #: re-promoting the recipe. + perf: V6GradedAxes class V6StackExt(TypedDict, total=False): @@ -1769,6 +1838,9 @@ class SessionBreakdown(TypedDict, total=False): "V6EnablementExt", "V6EnablementRevalidation", "V6GeakCandidate", + "V6GradedAxes", + "V6Grading", + "V6GradingTputGuard", "V6KBWriteBackExt", "V6KernelAdoptedRow", "V6KernelAnalysisArtifacts", diff --git a/src/hyperloom/inference_optimizer/cli/bootstrap.py b/src/hyperloom/inference_optimizer/cli/bootstrap.py index 83781be8a3..9a523d2c91 100644 --- a/src/hyperloom/inference_optimizer/cli/bootstrap.py +++ b/src/hyperloom/inference_optimizer/cli/bootstrap.py @@ -62,6 +62,33 @@ def resolve_model_display_name(args: argparse.Namespace) -> str: AGENTX_MEASUREMENT_EPOCH = 1 +def seed_grading(framework: str, benchmark_mode: str) -> dict[str, Any]: + """Resolve the grading axis and its noise band once, at seed, so they can be recorded. + + The resolution reads ``HYPERLOOM_PERF_METRIC`` and ``HYPERLOOM_PERF_NOISE_PCT``. Deriving it again later -- in a + resumed process, a re-baseline subprocess, or the breakdown export CLOSE drives from a subprocess that often did + not inherit them -- can name an axis the session never graded on. This is the same reasoning that put + ``benchmark_mode`` in the state rather than leaving it to the ambient var. + """ + from hyperloom.common.perf_metric import ( + GRADED_INTVTY, + GRADED_OUTPUT, + intvty_serving_grading_enabled, + parse_intvty_noise_pct, + ) + + from .. import framework_registry + + on_intvty = intvty_serving_grading_enabled( + scriptable=framework_registry.is_scriptable(framework), + benchmark_mode=benchmark_mode, + ) + return { + "objective": GRADED_INTVTY if on_intvty else GRADED_OUTPUT, + "noise_pct": parse_intvty_noise_pct(), + } + + def agentx_state_is_stale(state: Any) -> str: """Return why a resumed session's AgentX state is unusable, or ``\"\"``.""" want_mode = "agentx" if _agentx_enabled() else "synthetic" @@ -322,6 +349,7 @@ def _resolve_framework_version(args_in: Any) -> str: conc_sweep_enabled=bool(getattr(args, "enable_conc_sweep", not _agentx_enabled())), benchmark_mode=benchmark_mode, agentx_epoch=AGENTX_MEASUREMENT_EPOCH if _agentx_enabled() else 0, + grading=seed_grading(os.environ.get("FRAMEWORK", "sglang"), benchmark_mode), conc_sweep_concs=_parse_conc_sweep_concs(args, benchmark_mode), conc_sweep_total_budget_sec=int( getattr(args, "conc_sweep_total_budget_sec", 9000) or 0, diff --git a/src/hyperloom/inference_optimizer/tests/test_conc_sweep.py b/src/hyperloom/inference_optimizer/tests/test_conc_sweep.py index 09ed783149..53c83476d3 100644 --- a/src/hyperloom/inference_optimizer/tests/test_conc_sweep.py +++ b/src/hyperloom/inference_optimizer/tests/test_conc_sweep.py @@ -28,6 +28,7 @@ _flush_conc_sweep_report, _flush_partial_conc_sweep_report, _granted_cap_sec, + _grading_of, _has_optimization, _order_concs_desc, _point_from_variant, @@ -175,8 +176,8 @@ def test_build_comparison_mismatched_concs_outer_join(): ] rows, summary = _build_comparison(baseline, optimized) assert [r["conc"] for r in rows] == [1, 4, 16] - assert rows[0]["optimized_tput"] is None - assert rows[2]["baseline_tput"] is None + assert rows[0]["optimized_value"] is None + assert rows[2]["baseline_value"] is None assert summary["successful_pairs"] == 1 @@ -1324,7 +1325,6 @@ def test_the_key_follows_the_mode(self): assert graded_metric_key(benchmark_mode="") == "output_throughput" def test_an_explicit_grading_override_wins_over_the_mode(self, monkeypatch): - """The summary follows the axis the KEEP verdicts were taken on.""" monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "output_throughput") assert graded_metric_key(benchmark_mode="agentx") == "output_throughput" from hyperloom.common.perf_metric import INTVTY_V1 @@ -1339,6 +1339,112 @@ def test_the_ambient_agentx_signal_reaches_the_summary(self, monkeypatch): assert graded_metric_key(benchmark_mode="") == "e2e_norm_intvty_p90" +class TestTheSweepGradesOnTheAxisTheSessionKeepsOn: + """A curve drawn on one axis beside promotions decided on another is two answers to one question.""" + + @pytest.fixture(autouse=True) + def _no_ambient_grading(self, monkeypatch): + for name in ("HYPERLOOM_PERF_METRIC", "HYPERLOOM_PERF_NOISE_PCT", "HYPERLOOM_AGENTX"): + monkeypatch.delenv(name, raising=False) + + def _state(self, **fields: Any) -> SharedState: + state = SharedState() + for key, value in fields.items(): + setattr(state, key, value) + return state + + def test_the_recorded_axis_beats_the_environment(self, monkeypatch): + """A resume is a new process, and the shell it landed in is not evidence about the axis.""" + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "output_throughput") + state = self._state(benchmark_mode="agentx", grading={"objective": "e2e_norm_intvty_p90", "noise_pct": 3.5}) + assert _grading_of(state) == ("e2e_norm_intvty_p90", 3.5) + + def test_the_recorded_band_travels_with_the_axis(self): + """A lost band silently widens a 3.5% guard back to the 5% default.""" + state = self._state(benchmark_mode="agentx", grading={"objective": "e2e_norm_intvty_p90", "noise_pct": 3.5}) + assert _grading_of(state)[1] == 3.5 + + def test_a_session_with_nothing_recorded_falls_back_to_its_mode(self): + assert _grading_of(self._state(benchmark_mode="agentx"))[0] == "e2e_norm_intvty_p90" + assert _grading_of(self._state(benchmark_mode="synthetic"))[0] == "output_throughput" + + def test_a_scriptable_framework_is_carved_out(self): + """An image framework reports no interactivity axis, so ranking every rung on it fails the whole sweep.""" + state = self._state(benchmark_mode="agentx", framework="xdit") + assert _grading_of(state)[0] == "output_throughput" + + +class TestTheGuardAxisIsReportedNotEnforced: + """A sweep exists to draw the frontier, so a rung that moved along it is a result and not a failure.""" + + def _pts(self, arm: str, *, intvty: float, total: float) -> list[dict[str, Any]]: + return [ + { + "arm": arm, + "conc": 8, + "status": "succeeded", + "e2e_norm_intvty_p90": intvty, + "total_token_throughput": total, + } + ] + + def test_a_rung_that_bought_interactivity_with_throughput_still_pairs(self): + """It ranks on the objective it gained on, and carries the throughput it gave up beside it.""" + comparison, summary = _build_comparison( + self._pts("baseline", intvty=20.0, total=20000.0), + self._pts("optimized", intvty=30.0, total=10000.0), + metric_key="e2e_norm_intvty_p90", + ) + row = comparison[0] + assert row["speedup"] == pytest.approx(1.5) + assert summary["successful_pairs"] == 1 + assert summary["best_conc"] == 8 + # Halving throughput is far outside any band, and that is visible without having dropped the rung. + assert row["guard_holds"] is False + assert summary["best_conc_guard_holds"] is False + assert row["baseline_guard"] == 20000.0 + assert row["optimized_guard"] == 10000.0 + + def test_a_rung_that_held_throughput_says_so(self): + comparison, summary = _build_comparison( + self._pts("baseline", intvty=20.0, total=20000.0), + self._pts("optimized", intvty=24.0, total=19800.0), + metric_key="e2e_norm_intvty_p90", + ) + assert comparison[0]["guard_holds"] is True + assert summary["guard_axis"] == "total_throughput" + + def test_the_recorded_band_decides_the_verdict(self): + """The same pair holds under the default band and fails under a tighter one.""" + arms = ( + self._pts("baseline", intvty=20.0, total=20000.0), + self._pts("optimized", intvty=24.0, total=19200.0), + ) + assert _build_comparison(*arms, metric_key="e2e_norm_intvty_p90")[0][0]["guard_holds"] is True + tight, _ = _build_comparison(*arms, metric_key="e2e_norm_intvty_p90", guard_noise_pct=1.0) + assert tight[0]["guard_holds"] is False + + def test_the_output_objective_has_no_second_axis_to_hold(self): + comparison, summary = _build_comparison( + [{"arm": "baseline", "conc": 8, "status": "succeeded", "output_throughput": 100.0}], + [{"arm": "optimized", "conc": 8, "status": "succeeded", "output_throughput": 130.0}], + metric_key="output_throughput", + ) + assert summary["guard_axis"] == "" + assert summary["best_conc_guard_holds"] is None + assert comparison[0]["guard_holds"] is None + + def test_an_unmeasured_guard_axis_is_null_not_a_failure(self): + """Null says the axis was never measured; False would say it was, and fell outside.""" + comparison, _summary = _build_comparison( + [{"arm": "baseline", "conc": 8, "status": "succeeded", "e2e_norm_intvty_p90": 20.0}], + [{"arm": "optimized", "conc": 8, "status": "succeeded", "e2e_norm_intvty_p90": 30.0}], + metric_key="e2e_norm_intvty_p90", + ) + assert comparison[0]["guard_holds"] is None + assert comparison[0]["baseline_guard"] is None + + class TestAnUnreportedTotalComesFromItsHalves: """A row graded on the total axis must not read as unmeasured.""" diff --git a/src/hyperloom/inference_optimizer/tests/test_sbd_v6_conc_sweep_timeline.py b/src/hyperloom/inference_optimizer/tests/test_sbd_v6_conc_sweep_timeline.py index 766143372e..54d9da0fb7 100644 --- a/src/hyperloom/inference_optimizer/tests/test_sbd_v6_conc_sweep_timeline.py +++ b/src/hyperloom/inference_optimizer/tests/test_sbd_v6_conc_sweep_timeline.py @@ -87,7 +87,7 @@ def _point(conc: int, *, arm: str = ARM_OPTIMIZED, status: str = "succeeded", ** "request_throughput": 1.5 * conc, "total_token_throughput": 200.0 * conc, "input_throughput": 100.0 * conc, - "intvty_p90": 42.0, + "e2e_norm_intvty_p90": 42.0, "tpot_p90_ms": 13.5, "ttft_mean_ms": 130.0, "e2el_mean_ms": 4000.0, @@ -406,7 +406,7 @@ def test_a_rung_carries_the_agentic_axis_the_projection_dropped(_bound_session): point = _ext(_bound_session)["arms"][ARM_OPTIMIZED]["points"][0] assert point["arm"] == ARM_OPTIMIZED assert point["total_token_throughput"] == 12800.0 - assert point["intvty_p90"] == 42.0 + assert point["e2e_norm_intvty_p90"] == 42.0 assert point["tpot_p90_ms"] == 13.5 assert point["request_throughput"] == 96.0 assert point["input_throughput"] == 6400.0 @@ -548,8 +548,8 @@ def test_the_pair_table_keeps_the_gain_columns_the_projection_dropped(_bound_ses comparison=[ { "conc": 32, - "baseline_tput": 1000.0, - "optimized_tput": 1350.0, + "baseline_value": 1000.0, + "optimized_value": 1350.0, "speedup": 1.35, "delta_pct": 35.0, "baseline_status": "succeeded", @@ -570,13 +570,18 @@ def test_the_pair_table_keeps_the_gain_columns_the_projection_dropped(_bound_ses pair = _ext(_bound_session)["comparison"][0] assert pair["conc"] == 32 - assert pair["baseline_throughput"] == 1000.0 - assert pair["optimized_throughput"] == 1350.0 + assert pair["baseline_value"] == 1000.0 + assert pair["optimized_value"] == 1350.0 assert pair["speedup"] == 1.35 assert pair["delta_pct"] == 35.0 assert pair["baseline_status"] == "succeeded" assert pair["optimized_status"] == "succeeded" assert pair["error"] is None + # The output objective has no second axis to hold, so the guard columns stay null rather than reading as a + # rung whose throughput was measured and fell outside the band. + assert pair["baseline_guard"] is None + assert pair["optimized_guard"] is None + assert pair["guard_holds"] is None def test_a_failed_pair_is_explained_by_the_arm_that_broke(_bound_session): diff --git a/src/hyperloom/inference_optimizer/tests/test_sbd_v6_grading.py b/src/hyperloom/inference_optimizer/tests/test_sbd_v6_grading.py new file mode 100644 index 0000000000..aeb99c736d --- /dev/null +++ b/src/hyperloom/inference_optimizer/tests/test_sbd_v6_grading.py @@ -0,0 +1,375 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Coverage for the grading axis the SBD V6 document declares and the axes it publishes. + +An AgentX replay is ranked on the slow-tail interactivity percentile with throughput held as a guard; a +synthetic run is ranked on output throughput alone. On the canonical corpus the two axes differ by roughly +two orders of magnitude, and every throughput field in the breakdown is the output axis by construction, so +without the declaration a consumer would sort one kind of session against the other and every number would +look plausible. + +Two separate facts, deliberately not resolved from one another. ``metadata.grading`` is the axis the session +was configured for; ``outcome.validation.graded_on`` is the axis the run actually decided its last promotion +on, which differs whenever a comparison could not supply the axis pair. +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from hyperloom.common.perf_metric import GRADED_AXIS_KEYS, GRADED_INTVTY, GRADED_OUTPUT +from hyperloom.inference_optimizer.breakdown.collectors.v6 import collect_v6_metadata, collect_v6_outcome +from hyperloom.inference_optimizer.breakdown.recorder import assemble_parts, recorder_for, snapshot_metadata +from hyperloom.inference_optimizer.breakdown.recorder import stack_event +from hyperloom.inference_optimizer.breakdown.recorder.baseline_event import ( + PRODUCER as BASELINE_PRODUCER, + baseline_event_id, + make_baseline_recorder, +) +from hyperloom.inference_optimizer.breakdown.recorder.event_sink import make_sink +from hyperloom.inference_optimizer.breakdown.recorder.session_metadata import SECTION as METADATA_SECTION, _grading +from hyperloom.inference_optimizer.cli.bootstrap import seed_grading +from hyperloom.inference_optimizer.session.sbd_v6 import read_timeline_events +from hyperloom.inference_optimizer.session.session_binding import session_scope +from hyperloom.orchestrator.state.shared_state import SharedState, resolved_grading + +#: The AgentX axes a measured round carries, on the keys grading itself reads them from. +AGENTX_AXES: dict[str, Any] = { + GRADED_INTVTY: 41.2, + "total_throughput": 25978.0, + "input_throughput": 25795.0, + "tpot_p90_ms": 24.3, +} + + +@pytest.fixture(autouse=True) +def _bound_session(tmp_path): + """Bind the session the way startup does, so no call below takes a path.""" + with session_scope(tmp_path): + yield tmp_path + + +@pytest.fixture(autouse=True) +def _no_ambient_grading(monkeypatch): + """Clear the grading vars, so a test that does not set them is not reading the developer's shell.""" + monkeypatch.delenv("HYPERLOOM_PERF_METRIC", raising=False) + monkeypatch.delenv("HYPERLOOM_PERF_NOISE_PCT", raising=False) + monkeypatch.delenv("HYPERLOOM_AGENTX", raising=False) + + +def _state(**fields: Any) -> SharedState: + return SharedState(session_id="s-1", **fields) + + +# --------------------------------------------------------------------------- +# Resolving the axis at seed, where the run can still see its own configuration +# --------------------------------------------------------------------------- + + +def test_a_synthetic_session_is_seeded_on_the_output_axis(): + assert seed_grading("sglang", "synthetic")["objective"] == GRADED_OUTPUT + + +def test_an_agentx_session_is_seeded_on_the_interactivity_axis(): + assert seed_grading("sglang", "agentx")["objective"] == GRADED_INTVTY + + +def test_a_scriptable_framework_stays_on_output_even_under_agentx(): + # An image framework reports a quality gate, not an interactivity percentile, so the axis does not exist + # for it to be ranked on. + assert seed_grading("xdit", "agentx")["objective"] == GRADED_OUTPUT + + +def test_the_noise_band_is_captured_at_seed_rather_than_left_to_the_environment(monkeypatch): + monkeypatch.setenv("HYPERLOOM_PERF_NOISE_PCT", "3.5") + + assert seed_grading("sglang", "agentx")["noise_pct"] == 3.5 + + +def test_an_explicit_metric_override_is_resolved_at_seed(monkeypatch): + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "output_throughput") + + assert seed_grading("sglang", "agentx")["objective"] == GRADED_OUTPUT + + +# --------------------------------------------------------------------------- +# What was recorded beats what the current process happens to hold +# --------------------------------------------------------------------------- + + +def test_the_recorded_axis_wins_over_a_shell_that_lost_the_variable(monkeypatch): + # A resume is a new process. Deriving here would flip a session that graded on interactivity back to + # output halfway through, and the KEEP/REVERT rule has to be the one the session started with. + monkeypatch.setenv("HYPERLOOM_PERF_METRIC", "output_throughput") + state = _state(grading={"objective": GRADED_INTVTY, "noise_pct": 3.5}, benchmark_mode="synthetic") + + assert resolved_grading(state) == (True, 3.5) + + +def test_the_recorded_band_wins_over_the_ambient_default(monkeypatch): + monkeypatch.setenv("HYPERLOOM_PERF_NOISE_PCT", "5.0") + state = _state(grading={"objective": GRADED_INTVTY, "noise_pct": 3.5}) + + assert resolved_grading(state)[1] == 3.5 + + +def test_a_session_predating_the_field_derives_and_reports_no_band(): + # The band that session applied was never recorded, and today's default is not evidence of it. + state = _state(benchmark_mode="agentx", framework="sglang") + + assert resolved_grading(state) == (True, None) + + +# --------------------------------------------------------------------------- +# metadata.grading: the axis the session was configured for +# --------------------------------------------------------------------------- + + +def test_metadata_grading_declares_the_axis_and_the_guard_band(): + state = _state( + benchmark_mode="agentx", + grading={"objective": GRADED_INTVTY, "noise_pct": 3.5}, + ) + + assert _grading(state) == { + "benchmark_mode": "agentx", + "objective": GRADED_INTVTY, + "tput_guard": {"enabled": True, "noise_pct": 3.5}, + } + + +def test_metadata_grading_reports_no_guard_on_a_synthetic_session(): + state = _state(benchmark_mode="synthetic", grading=seed_grading("sglang", "synthetic")) + + block = _grading(state) + assert block["objective"] == GRADED_OUTPUT + assert block["tput_guard"]["enabled"] is False + + +def test_metadata_grading_names_the_mode_a_session_with_no_recorded_axis_ran(): + # ``benchmark_mode`` reaches ``reports/final.json`` but nothing else in the breakdown, so an AgentX + # session would otherwise be unidentifiable in this document. + assert _grading(_state(benchmark_mode="agentx", framework="sglang"))["benchmark_mode"] == "agentx" + + +def test_the_grading_block_reaches_the_exported_metadata(tmp_path): + # The whole point of recording it: the collector projects nothing for this block, so it lands purely from + # the spool and the export path reads no environment at all to produce it. + rec = recorder_for(tmp_path, producer="coordinator") + snapshot_metadata(rec, _state(benchmark_mode="agentx", grading={"objective": GRADED_INTVTY, "noise_pct": 3.5})) + + metadata = collect_v6_metadata( + exported_at_utc="2026-09-01T02:00:05+00:00", + session={"session_id": "s-1"}, + workload={"framework_name": "sglang"}, + model_info={}, + langfuse={"enabled": False}, + state={}, + warnings=[], + recorded=assemble_parts(tmp_path)[METADATA_SECTION], + ) + assert metadata["grading"] == { + "benchmark_mode": "agentx", + "objective": GRADED_INTVTY, + "tput_guard": {"enabled": True, "noise_pct": 3.5}, + } + + +def test_an_unrecorded_noise_band_survives_the_export_as_a_null(tmp_path): + # The singleton and the overlay both have to carry an explicit null through: today's default is not + # evidence of the band a session that predates the field applied. + rec = recorder_for(tmp_path, producer="coordinator") + snapshot_metadata(rec, _state(benchmark_mode="agentx", framework="sglang")) + + recorded = assemble_parts(tmp_path)[METADATA_SECTION] + assert recorded["grading"]["tput_guard"]["noise_pct"] is None + + +def test_metadata_grading_is_not_resolved_from_what_a_promotion_graded_on(): + # The lock on keeping the two facts separate. A session configured for interactivity whose comparisons + # all degraded still asked for interactivity; reporting output here would erase the request, and + # reporting interactivity in ``outcome`` would put that label on an output figure. + state = _state(benchmark_mode="agentx", grading={"objective": GRADED_INTVTY, "noise_pct": 5.0}) + + assert _grading(state)["objective"] == GRADED_INTVTY + + +# --------------------------------------------------------------------------- +# outcome: the axis a promotion was actually decided on, and the axes it measured +# --------------------------------------------------------------------------- + + +def _adopt(index: int, *, objective: str = GRADED_OUTPUT, degrade_reason: str = "") -> None: + """Record one adoption the way the lift does.""" + stack_event.record_adoption( + stack_index=index, + entry={"action": "explore"}, + throughput_before=100.0 + index * 10.0, + throughput_after=110.0 + index * 10.0, + baseline_tput=100.0, + objective=objective, + degrade_reason=degrade_reason, + ) + + +def _outcome(session_dir, *, close: dict[str, Any] | None = None) -> dict[str, Any]: + """The assembled ``outcome`` block over the recorded timeline.""" + stack_event.finish() + return collect_v6_outcome( + session={"stop_reason": "target_reached"}, + close=close or {}, + state={}, + timeline=read_timeline_events(session_dir), + ) + + +def test_outcome_reports_the_axis_the_settled_validation_was_measured_on(tmp_path): + _adopt(0, objective=GRADED_INTVTY) + stack_event.record_validation( + stack_len=1, + baseline_tput=38.0, + validated_tput=41.2, + validated_gain_pct=8.42, + graded_objective=GRADED_INTVTY, + measurement=AGENTX_AXES, + ) + + assert _outcome(tmp_path)["validation"]["graded_on"] == GRADED_INTVTY + + +def test_outcome_falls_back_to_the_adoption_axis_when_nothing_validated(tmp_path): + # A session that adopted but never measured the whole stack has no settled row to read, and the axis its + # adoptions were graded on is the only recorded answer. + _adopt(0, objective=GRADED_INTVTY) + + assert _outcome(tmp_path)["validation"]["graded_on"] == GRADED_INTVTY + + +def test_the_final_gain_carries_the_same_axis_as_the_reconciliation(tmp_path): + # One lock for both: the gain and the attribution are the same figure read twice, so a reader must never + # find two axis labels on them. + _adopt(0, objective=GRADED_INTVTY) + stack_event.record_validation( + stack_len=1, + baseline_tput=38.0, + validated_tput=41.2, + validated_gain_pct=8.42, + graded_objective=GRADED_INTVTY, + measurement=AGENTX_AXES, + ) + + outcome = _outcome(tmp_path, close={"final_recipe": {"throughput": 183.0}}) + assert outcome["final"]["graded_on"] == outcome["validation"]["graded_on"] == GRADED_INTVTY + + +def test_the_settled_axes_are_published_beside_the_gain_they_produced(tmp_path): + # Read off the validation row rather than ``current_best``: a revalidation moves the cumulative figure + # without re-promoting the recipe, so ``current_best`` can be a different measurement entirely. + stack_event.record_validation( + stack_len=1, + baseline_tput=38.0, + validated_tput=41.2, + validated_gain_pct=8.42, + graded_objective=GRADED_INTVTY, + measurement=AGENTX_AXES, + ) + + outcome = _outcome(tmp_path) + assert outcome["validation"]["perf"] == AGENTX_AXES + assert outcome["final"]["perf"] == AGENTX_AXES + + +def test_an_unmeasured_axis_is_an_explicit_null_rather_than_an_absent_key(tmp_path): + # Absent would be indistinguishable from an axis the framework failed to report, and zero reads as + # "measured, and it was zero". + stack_event.record_validation( + stack_len=1, + baseline_tput=100.0, + validated_tput=120.0, + validated_gain_pct=20.0, + graded_objective=GRADED_OUTPUT, + measurement={"output_throughput": 120.0}, + ) + + perf = _outcome(tmp_path)["validation"]["perf"] + assert set(perf) == set(GRADED_AXIS_KEYS) + assert all(value is None for value in perf.values()) + + +def test_a_session_with_no_ledger_publishes_no_axis(tmp_path): + outcome = collect_v6_outcome(session={"stop_reason": "signal"}, close={}, state={}, timeline=[]) + + assert outcome["validation"]["graded_on"] is None + assert all(value is None for value in outcome["validation"]["perf"].values()) + + +def test_the_notes_name_adoptions_that_fell_off_the_configured_axis(tmp_path): + # Their contributions sit in the same sum as the axis-graded ones, so the total is not single-axis and + # the reader has to be told. + _adopt(0, objective=GRADED_INTVTY) + _adopt(1, objective=GRADED_OUTPUT, degrade_reason="candidate_axes_missing") + _adopt(2, objective=GRADED_OUTPUT, degrade_reason="candidate_axes_missing") + + joined = " | ".join(_outcome(tmp_path)["validation"]["notes"]) + assert "2 adoption(s) were graded on the output axis" in joined + assert "candidate_axes_missing" in joined + + +def test_a_fully_graded_ledger_reports_no_degrade_finding(tmp_path): + # The whole and the parts agree here, so an empty list is the meaningful assertion: no finding at all, + # rather than a degrade finding drowned out by a reconciliation complaint. + _adopt(0, objective=GRADED_INTVTY) + stack_event.record_validation( + stack_len=1, + baseline_tput=100.0, + validated_tput=110.0, + validated_gain_pct=10.0, + graded_objective=GRADED_INTVTY, + measurement=AGENTX_AXES, + ) + + assert _outcome(tmp_path)["validation"]["notes"] == [] + + +# --------------------------------------------------------------------------- +# outcome.baseline: the axes the session was anchored on +# --------------------------------------------------------------------------- + + +def _record_baseline(**axes: Any) -> None: + """Record an anchoring baseline the way a dispatched measurement does.""" + recorder = make_baseline_recorder( + make_sink(baseline_event_id("prelude", 0), producer=BASELINE_PRODUCER), + task_id="t-1", + task_kind="baseline", + reason="", + framework="sglang", + establishes_quality_ref=True, + params={"config_path": "/cfg.yaml", "output_dir": "/w", "timeout_sec": 7800}, + ) + assert recorder is not None + recorder.finish({"status": "succeeded", "output_throughput": 183.0, **axes}) + + +def test_the_baseline_publishes_the_axes_the_session_was_anchored_on(tmp_path): + # Recorded on the baseline round rather than read off ``state.baseline_perf`` at export, because this + # block is already where ``outcome.baseline`` comes from and a second source is a second answer. + _record_baseline(**AGENTX_AXES) + + baseline = _outcome(tmp_path)["baseline"] + assert baseline["perf"] == AGENTX_AXES + # The output axis keeps its own meaning beside them: this addition takes nothing away. + assert baseline["throughput_tok_s_per_gpu"] == 183.0 + + +def test_a_synthetic_baseline_publishes_four_nulls(tmp_path): + _record_baseline() + + assert all(value is None for value in _outcome(tmp_path)["baseline"]["perf"].values()) + + +def test_a_session_with_no_anchoring_baseline_still_publishes_the_axis_shape(tmp_path): + assert set(_outcome(tmp_path)["baseline"]["perf"]) == set(GRADED_AXIS_KEYS) diff --git a/src/hyperloom/inference_optimizer/tests/test_sbd_v6_stages.py b/src/hyperloom/inference_optimizer/tests/test_sbd_v6_stages.py index 52a8d14a6e..0787d5f63a 100644 --- a/src/hyperloom/inference_optimizer/tests/test_sbd_v6_stages.py +++ b/src/hyperloom/inference_optimizer/tests/test_sbd_v6_stages.py @@ -225,6 +225,15 @@ def _baseline_event(*actions: dict) -> dict: return {"type": "baseline", "ext": {"actions": list(actions)}} +#: What ``outcome.baseline.perf`` reads on a round that measured no graded axis. +_UNMEASURED_AXES = { + "e2e_norm_intvty_p90": None, + "total_throughput": None, + "input_throughput": None, + "tpot_p90_ms": None, +} + + def test_outcome_baseline_reads_the_anchoring_measurement_off_the_timeline(): outcome = _v6_outcome([_baseline_event(_baseline_action(task_id="b-1", throughput=800.0))]) @@ -233,6 +242,9 @@ def test_outcome_baseline_reads_the_anchoring_measurement_off_the_timeline(): "accuracy": 0.81, "ttft_mean_ms": 120.0, "e2el_mean_ms": 900.0, + # A synthetic anchor measures none of the graded axes, and all four are still published; see + # test_sbd_v6_grading.py for the axes themselves. + "perf": _UNMEASURED_AXES, } @@ -275,6 +287,7 @@ def test_outcome_baseline_keeps_a_degraded_anchor_and_drops_a_failed_one(): "accuracy": None, "ttft_mean_ms": None, "e2el_mean_ms": None, + "perf": _UNMEASURED_AXES, } diff --git a/src/hyperloom/orchestrator/kernel/conc_sweep.py b/src/hyperloom/orchestrator/kernel/conc_sweep.py index 808971a912..014c270c63 100644 --- a/src/hyperloom/orchestrator/kernel/conc_sweep.py +++ b/src/hyperloom/orchestrator/kernel/conc_sweep.py @@ -17,7 +17,7 @@ from hyperloom.common import io as _common_io from hyperloom.common.gain_math import conc_pair_comparison from hyperloom.common.model_paths import resolve_session_model_path -from hyperloom.common.perf_metric import graded_metric_key, is_agentx_mode +from hyperloom.common.perf_metric import GRADED_INTVTY, GRADED_OUTPUT, is_agentx_mode from hyperloom.common.timeutil import now_iso, utc_now_compact from hyperloom.inference_optimizer.breakdown.recorder.conc_sweep_event import ( GRID_MODE_DEFAULT, @@ -50,12 +50,25 @@ load_model_meta, select_peak_and_bound, ) -from ..state.shared_state import SharedState +from ..state.shared_state import SharedState, resolved_grading log = logging.getLogger(__name__) +def _grading_of(state: Any) -> tuple[str, float | None]: + """The axis this sweep draws its speedups on, and the noise band its guard reads. + + Resolved through ``resolved_grading`` so the curve and the promotions in one session cannot end up on + different axes. The environment-derived ``graded_metric_key`` diverges two ways: it never sees the axis + recorded at seed, so a resume whose shell lost ``HYPERLOOM_PERF_METRIC`` redraws the curve on output; and it + has no scriptable carve-out, so an image framework -- which reports no interactivity axis at all -- would + compare every rung on a field it never measures and report the whole sweep as failed. + """ + on_intvty, noise_pct = resolved_grading(state) + return (GRADED_INTVTY if on_intvty else GRADED_OUTPUT), noise_pct + + SCHEMA_VERSION = "1.0" # Default ladders, one per workload (override via ``--conc-sweep-concs``). @@ -258,6 +271,9 @@ def _mbu_pct(measured: Any) -> float | None: return None return within_roofline_pct(peak=float(t_peak), achieved=float(measured)) + # Output throughput on both arms regardless of the graded axis, and not a bug to be aligned with it: the + # peak above is a memory-bandwidth-derived ceiling on output tokens per second, so MBU is only meaningful + # against the same quantity. bt = (by_conc_b.get(c) or {}).get("output_throughput") ot = (by_conc_o.get(c) or {}).get("output_throughput") rows.append( @@ -1158,9 +1174,8 @@ def _flush_partial_conc_sweep_report( # noqa: PLR0913 b_pts.sort(key=lambda p: p["conc"]) o_pts.sort(key=lambda p: p["conc"]) - comparison, summary = conc_pair_comparison( - b_pts, o_pts, metric_key=graded_metric_key(benchmark_mode=str(getattr(state, "benchmark_mode", "") or "")) - ) + metric_key, guard_noise_pct = _grading_of(state) + comparison, summary = conc_pair_comparison(b_pts, o_pts, metric_key=metric_key, guard_noise_pct=guard_noise_pct) if recorder is not None: recorder.record_progress(comparison=comparison, summary=summary) p: dict[str, Any] = { @@ -1543,10 +1558,12 @@ async def run_conc_sweep( baseline_points.sort(key=lambda p: p["conc"]) optimized_points.sort(key=lambda p: p["conc"]) + metric_key, guard_noise_pct = _grading_of(state) comparison, summary = conc_pair_comparison( baseline_points, optimized_points, - metric_key=graded_metric_key(benchmark_mode=str(getattr(state, "benchmark_mode", "") or "")), + metric_key=metric_key, + guard_noise_pct=guard_noise_pct, ) budget_limited_no_pair = _budget_limited_without_valid_pair( budget_exhausted=budget_exhausted, diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index 6311708556..7032741eef 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -850,9 +850,8 @@ def _update_cumulative_gain_validated( Returns: Whether a comparable measurement updated the validation watermark. """ - graded = resolve_graded_comparison( - self.shared_state, _graded_source(measurement, new_tput), against_baseline=True - ) + graded_source = _graded_source(measurement, new_tput) + graded = resolve_graded_comparison(self.shared_state, graded_source, against_baseline=True) if not graded.comparable: log.info("cumulative gain held: measurement not comparable (%s)", graded.degrade_reason) return False @@ -876,6 +875,9 @@ def _update_cumulative_gain_validated( source=source, measurement_basis=measurement_basis, graded_objective=graded.objective, + # The figures grading actually read, not the raw measurement: the caller's resolved output + # throughput is stamped into it, so the axes recorded here are the ones the verdict was reached on. + measurement=graded_source, ts=ts, ttft_mean_ms=measurement.get("ttft_mean_ms"), e2el_mean_ms=measurement.get("e2el_mean_ms"), diff --git a/src/hyperloom/orchestrator/state/shared_state.py b/src/hyperloom/orchestrator/state/shared_state.py index 165f75fb40..6f09c0a587 100644 --- a/src/hyperloom/orchestrator/state/shared_state.py +++ b/src/hyperloom/orchestrator/state/shared_state.py @@ -89,6 +89,31 @@ def resolve_grading_anchor_tput(state: Any) -> float: ANCHOR_DEGRADED: Any = object() +def resolved_grading(state: Any) -> tuple[bool, float | None]: + """Whether the interactivity objective applies to *state*, and the noise band it grades under. + + Prefers what the session recorded at seed over re-deriving it. The derivation reads the environment, and every + later reader of it is somewhere the environment is not evidence: a resumed process, a re-baseline subprocess, an + export driven from CLOSE. Sessions seeded before ``SharedState.grading`` existed carry nothing and only those + derive, reporting a null band because the band they actually applied was never recorded. + """ + from hyperloom.common.perf_metric import GRADED_INTVTY, intvty_serving_grading_enabled + + recorded = getattr(state, "grading", None) + recorded = recorded if isinstance(recorded, dict) else {} + objective = str(recorded.get("objective") or "").strip() + if objective: + noise_pct = recorded.get("noise_pct") + return objective == GRADED_INTVTY, (float(noise_pct) if isinstance(noise_pct, (int, float)) else None) + return ( + intvty_serving_grading_enabled( + scriptable=framework_is_scriptable(getattr(state, "framework", None)), + benchmark_mode=str(getattr(state, "benchmark_mode", "") or ""), + ), + None, + ) + + def resolve_graded_comparison( state: Any, measurement: Any, @@ -107,7 +132,8 @@ def resolve_graded_comparison( # ``keep_threshold_pct`` is floored at AGENTX_KEEP_THRESHOLD_FLOOR_PCT here because this is the one place every # lane's threshold passes through. ``anchor_perf``/``anchor_tput`` default to the session anchor; explore passes # its own because variants stack within a round, and ANCHOR_DEGRADED holds a round on the output axis rather than - # re-resolving the session anchor the way None does. + # re-resolving the session anchor the way None does. The objective and the band come from ``resolved_grading``, + # so both are the ones the session was seeded with rather than whatever the calling process's environment holds. from hyperloom.common.gain_math import gain_pct from hyperloom.common.perf_metric import ( AGENTX_KEEP_THRESHOLD_FLOOR_PCT, @@ -118,7 +144,6 @@ def resolve_graded_comparison( VERDICT_RECORDED, VERDICT_REVERT, intvty_of, - intvty_serving_grading_enabled, output_tput_of, passes_intvty_gate, passes_tput_guard, @@ -127,11 +152,9 @@ def resolve_graded_comparison( total_tput_of, ) + on_intvty, noise_pct = resolved_grading(state) degrade_reason = "" - if intvty_serving_grading_enabled( - scriptable=framework_is_scriptable(getattr(state, "framework", None)), - benchmark_mode=str(getattr(state, "benchmark_mode", "") or ""), - ): + if on_intvty: if anchor_perf is ANCHOR_DEGRADED: # Already on the output axis for this round. Re-resolving the # session anchor here would grade later variants on interactivity @@ -156,10 +179,10 @@ def resolve_graded_comparison( keep_threshold_pct, threshold, ) - tput_holds = passes_tput_guard(cand_perf, ref_perf) + tput_holds = passes_tput_guard(cand_perf, ref_perf, noise_pct=noise_pct) if gain is not None and gain >= threshold and tput_holds: verdict = VERDICT_KEEP - elif not passes_intvty_gate(cand_perf, ref_perf) and not tput_holds: + elif not passes_intvty_gate(cand_perf, ref_perf, noise_pct=noise_pct) and not tput_holds: verdict = VERDICT_REVERT else: verdict = VERDICT_RECORDED @@ -472,6 +495,12 @@ class SharedState(_RenderMixin, _ExploreStateMixin): benchmark_mode: str = "" # Generation counter for AgentX measurements. agentx_epoch: int = 0 + # The grading configuration this session was seeded with: {"objective": GRADED_INTVTY|GRADED_OUTPUT, + # "noise_pct": float}. Recorded rather than re-derived because the derivation reads HYPERLOOM_PERF_METRIC / + # HYPERLOOM_PERF_NOISE_PCT, and a resume is a new process: a shell that lost the variable would flip the axis + # mid-run, and a lost noise band would silently widen a 3.5% guard back to the 5% default. The KEEP/REVERT rule + # has to be the one the session started with. Empty on sessions predating the field, which fall back to deriving. + grading: dict[str, Any] = field(default_factory=dict) # Stamped once when the run objective is first met. target_reached_at: str = "" # CONC ladder for conc_sweep, seeded from the workload's own ladder by ``_parse_conc_sweep_concs``.