diff --git a/gate/make_manifest.py b/gate/make_manifest.py index 3cf23e3..f8e009c 100644 --- a/gate/make_manifest.py +++ b/gate/make_manifest.py @@ -40,15 +40,36 @@ if TYPE_CHECKING: from score import ScoringResult +# A round whose forecasts are mostly missing is an eval failure, not a bad +# forecaster: the lab forecaster never raises, so dead API credentials, +# exhausted credits, or a routing error silently default everything to 0.5 +# and score a plausible ~50 index (observed 2026-09-04: OpenAI credits out, +# 100% missing, four rounds "scored" 50.0). Healthy runs are <1% missing; +# total API failure is exactly 100%. Fail loudly between those. +MAX_MISSING_FRACTION = 0.10 + def load_rounds() -> list[str]: + """Load pinned gate rounds. + + If GATE_ROUNDS_OVERRIDE is set (comma-separated round names), it takes + precedence over gate_rounds.json. This lets the outer loop's OverfitDetector + evaluate holdout rounds without modifying the committed gate config. + """ + import os + + override = os.environ.get("GATE_ROUNDS_OVERRIDE") + if override: + rounds = [r.strip() for r in override.split(",") if r.strip()] + if rounds: + return rounds + if not ROUNDS.exists(): raise SystemExit( f"{ROUNDS} not found. Run: python gate/make_manifest.py " "--rounds 2026-03-01 2026-04-12 2026-05-10" ) - rounds: list[str] = json.loads(ROUNDS.read_text())["rounds"] - return rounds + return list[str](json.loads(ROUNDS.read_text())["rounds"]) def score_rounds( @@ -101,6 +122,17 @@ def score_rounds( f"Round {name} scored zero rows. Check that it has a published " "resolution set." ) + missing_fraction = scoring.n_missing / (rows + scoring.n_missing) + if missing_fraction > MAX_MISSING_FRACTION: + raise SystemExit( + f"Round {name}: {scoring.n_missing}/{rows + scoring.n_missing} " + f"forecasts missing ({missing_fraction:.0%}) exceeds the " + f"{MAX_MISSING_FRACTION:.0%} guard. This is almost certainly an " + "eval failure, not a bad forecaster — check API credentials, " + "credits, and model routing before trusting any score. " + "(Missing forecasts default to 0.5, so a dead API silently " + "scores ~50.)" + ) per_round.append((name, scoring)) if not per_round: @@ -119,7 +151,12 @@ def _report(mean_index: float, per_round: list[tuple[str, ScoringResult]]) -> No f"market={r.n_market:4d} ({r.market_index:6.2f}) " f"missing={r.n_missing}" ) - print(f"mean index across {len(per_round)} rounds: {mean_index:.3f}") + mean_dataset = sum(r.dataset_index for _, r in per_round) / len(per_round) + mean_market = sum(r.market_index for _, r in per_round) / len(per_round) + print( + f"mean across {len(per_round)} rounds: " + f"overall={mean_index:.3f} dataset={mean_dataset:.3f} market={mean_market:.3f}" + ) def main() -> None: @@ -143,9 +180,24 @@ def main() -> None: if args.set_baseline: mean_index, per_round = score_rounds(load_rounds()) _report(mean_index, per_round) - BASELINE.write_text(json.dumps({"brier_index": round(mean_index, 3)}, indent=2)) - print(f"\nwrote {BASELINE.name}: brier_index = {mean_index:.3f}") - print(f"ladder spans {mean_index - 4:.1f} to {mean_index + 5:.1f}") + + mean_dataset = sum(r.dataset_index for _, r in per_round) / len(per_round) + mean_market = sum(r.market_index for _, r in per_round) / len(per_round) + BASELINE.write_text( + json.dumps( + { + "overall_index": round(mean_index, 3), + "dataset_index": round(mean_dataset, 3), + "market_index": round(mean_market, 3), + }, + indent=2, + ) + ) + print(f"\nwrote {BASELINE.name}:") + print(f" overall_index = {mean_index:.3f} (ladder spans {mean_index - 4:.1f} to {mean_index + 5:.1f})") + print(f" dataset_index = {mean_dataset:.3f} (ladder spans {mean_dataset - 4:.1f} to {mean_dataset + 5:.1f})") + print(f" market_index = {mean_market:.3f} (ladder spans {mean_market - 4:.1f} to {mean_market + 5:.1f})") + print("two 10-rung ladders (20 total) — equal weight per category") if not (args.rounds or args.dry_run or args.set_baseline): p.error("nothing to do: pass --rounds, --dry-run, and/or --set-baseline") diff --git a/gate/test_brier_gate.py b/gate/test_brier_gate.py index 9a36fd1..7cda502 100644 --- a/gate/test_brier_gate.py +++ b/gate/test_brier_gate.py @@ -1,15 +1,20 @@ """Graded Brier Index gate. This is the outer-loop fitness function. -Scores FULL rounds, one at a time, and averages the per-round Brier Index. +Scores FULL rounds, one at a time, and averages per-round Brier Index. See gate/make_manifest.py::score_rounds for why per-round rather than pooled. +Two independent ladders — dataset and market — so the outer loop gets a +gradient on each category separately. This matches the competition's +equal-weight scoring: overall = (dataset_brier + market_brier) / 2. +A candidate that improves one category while hurting the other will show +a split signal instead of a masked overall gain — the failure mode that +made gen-0 calibration look like an improvement when it was a regression. + remote-factory scores a candidate by piping this command's stdout through -parse_pytest_stdout, which regexes "(\d+) passed" / "(\d+) failed" / -"(\d+) error", then computes `passed / total - 0.01 * node_count`. A binary -pass/fail gives that search no gradient, and the contrastive reflector returns -an empty report when top-K and bottom-K don't differ. Hence a 10-rung ladder -centered on a committed baseline: at baseline the score is 0.5, with room to -move both ways. +parse_pytest_stdout, which regexes "(\\d+) passed" / "(\\d+) failed" / +"(\\d+) error", then computes `passed / total - 0.01 * node_count`. With +two 10-rung ladders the total is 20, giving finer resolution (0.05) than +the original single 10-rung ladder (0.10). Run: uv run pytest gate/ -q @@ -40,56 +45,95 @@ _OFFSETS = list(range(-LADDER_BELOW, LADDER_ABOVE + 1)) -def _load_baseline() -> float | None: +def _load_baselines() -> dict[str, float] | None: + """Load per-category baselines from gate_baseline.json. + + Returns None (→ skip) if the file is missing or uses the old + single-value format. The caller should re-run --set-baseline. + """ if not BASELINE.exists(): return None try: - return float(json.loads(BASELINE.read_text())["brier_index"]) + data = json.loads(BASELINE.read_text()) + if "dataset_index" not in data or "market_index" not in data: + return None # Old format — needs re-baselining + return { + "dataset": float(data["dataset_index"]), + "market": float(data["market_index"]), + "overall": float(data.get("overall_index", (data["dataset_index"] + data["market_index"]) / 2)), + } except (json.JSONDecodeError, KeyError, ValueError, TypeError): return None @pytest.fixture(scope="session") -def baseline() -> float: - """Brier Index the ladder is centered on.""" - base = _load_baseline() - if base is None: - pytest.skip("gate_baseline.json missing; run gate/make_manifest.py --set-baseline") - return base +def baselines() -> dict[str, float]: + """Per-category Brier Index baselines the ladders are centered on.""" + bl = _load_baselines() + if bl is None: + pytest.skip( + "gate_baseline.json missing per-category baselines; " + "run: python gate/make_manifest.py --set-baseline" + ) + return bl @pytest.fixture(scope="session") -def brier_index(baseline: float) -> float: +def scores(baselines: dict[str, float]) -> dict[str, float]: """Score every pinned round once per session, not once per rung. - Depends on `baseline` so a missing baseline skips before any API calls. + Depends on `baselines` so a missing baseline skips before any API calls. + Returns mean per-category Brier Index across all pinned rounds. + Respects GATE_ROUNDS_OVERRIDE env var for holdout evaluation. """ - if not ROUNDS.exists(): - pytest.skip("gate_rounds.json missing; run gate/make_manifest.py --rounds ...") + from gate.make_manifest import load_rounds, score_rounds - from gate.make_manifest import score_rounds + rounds = load_rounds() + _, per_round = score_rounds(rounds) - rounds = json.loads(ROUNDS.read_text())["rounds"] - index, per_round = score_rounds(rounds) + mean_dataset = sum(r.dataset_index for _, r in per_round) / len(per_round) + mean_market = sum(r.market_index for _, r in per_round) / len(per_round) + mean_overall = sum(r.overall_index for _, r in per_round) / len(per_round) print() for name, res in per_round: + total = res.n_dataset + res.n_market print( - f"gate: {name} index={res.overall_index:.3f} " - f"rows={res.n_dataset + res.n_market} " - f"(dataset={res.n_dataset} market={res.n_market} missing={res.n_missing})" + f"gate: {name} overall={res.overall_index:.3f} " + f"dataset={res.dataset_index:.3f} (n={res.n_dataset}) " + f"market={res.market_index:.3f} (n={res.n_market}) " + f"missing={res.n_missing} rows={total}" ) - print(f"gate: mean index={index:.3f} across {len(per_round)} rounds") - return index + print( + f"gate: mean across {len(per_round)} rounds — " + f"overall={mean_overall:.3f} dataset={mean_dataset:.3f} market={mean_market:.3f}" + ) + return {"dataset": mean_dataset, "market": mean_market, "overall": mean_overall} + +# --------------------------------------------------------------------------- +# Two independent ladders: dataset and market. +# Equal rung count per category = equal weight, matching competition scoring. +# Total rungs = 2 * len(_OFFSETS) = 20. +# --------------------------------------------------------------------------- @pytest.mark.parametrize("offset", _OFFSETS) -def test_brier_index_at_or_above(brier_index: float, baseline: float, offset: int) -> None: - """One rung of the ladder. Pass count is the fitness signal.""" - threshold = baseline + offset - assert brier_index >= threshold, ( - f"Brier Index {brier_index:.3f} below rung {threshold:.1f} " - f"(baseline {baseline:.3f}{offset:+d})" +def test_dataset_index_at_or_above(scores: dict[str, float], baselines: dict[str, float], offset: int) -> None: + """Dataset Brier Index ladder. Pass count is half the fitness signal.""" + threshold = baselines["dataset"] + offset + assert scores["dataset"] >= threshold, ( + f"Dataset Brier Index {scores['dataset']:.3f} below rung {threshold:.1f} " + f"(baseline {baselines['dataset']:.3f}{offset:+d})" + ) + + +@pytest.mark.parametrize("offset", _OFFSETS) +def test_market_index_at_or_above(scores: dict[str, float], baselines: dict[str, float], offset: int) -> None: + """Market Brier Index ladder. Pass count is half the fitness signal.""" + threshold = baselines["market"] + offset + assert scores["market"] >= threshold, ( + f"Market Brier Index {scores['market']:.3f} below rung {threshold:.1f} " + f"(baseline {baselines['market']:.3f}{offset:+d})" ) @@ -100,8 +144,8 @@ def test_pinned_rounds_well_formed() -> None: silently drops rounds produces a plausible number rather than an obvious failure. score_rounds() raises on zero rows; this checks the pinned set. """ - if not ROUNDS.exists(): - pytest.skip("gate_rounds.json missing") - rounds = json.loads(ROUNDS.read_text())["rounds"] + from gate.make_manifest import load_rounds + + rounds = load_rounds() assert len(rounds) >= 1 assert len(set(rounds)) == len(rounds), "duplicate rounds pinned" diff --git a/gate_baseline.json b/gate_baseline.json index ecc49ca..fcddf80 100644 --- a/gate_baseline.json +++ b/gate_baseline.json @@ -1,3 +1,5 @@ { - "brier_index": 61.285 + "overall_index": 50.0, + "dataset_index": 50.0, + "market_index": 50.0 } \ No newline at end of file diff --git a/gate_rounds.json b/gate_rounds.json index 815877a..992a066 100644 --- a/gate_rounds.json +++ b/gate_rounds.json @@ -1,6 +1,8 @@ { "rounds": [ "2026-03-01", + "2026-03-15", + "2026-03-29", "2026-04-12" ] -} \ No newline at end of file +} diff --git a/tests/test_gate_missing_guard.py b/tests/test_gate_missing_guard.py new file mode 100644 index 0000000..4a16b16 --- /dev/null +++ b/tests/test_gate_missing_guard.py @@ -0,0 +1,80 @@ +"""Tests for the gate's missing-forecast guard (gate/make_manifest.py). + +The lab forecaster never raises, so a dead API (expired credentials, exhausted +credits, wrong routing) silently defaults every forecast to 0.5 and the round +scores a plausible ~50 Brier Index. On 2026-09-04 four rounds "passed" this +way with 100% missing forecasts. score_rounds() now fails loudly instead. +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any + +import pytest + +REPO = Path(__file__).resolve().parent.parent +if str(REPO) not in sys.path: + sys.path.insert(0, str(REPO)) + +from gate.make_manifest import MAX_MISSING_FRACTION # noqa: E402 + + +class _EvalResult: + """Minimal stand-in for eval.run_eval's return: only .scoring is used.""" + + def __init__(self, scoring: object) -> None: + self.scoring = scoring + + +def _scoring(n_dataset: int, n_market: int, n_missing: int) -> Any: + from score import ScoringResult + + return ScoringResult( + dataset_brier=0.25, + dataset_index=50.0, + market_brier=0.25, + market_index=50.0, + overall_brier=0.25, + overall_index=50.0, + n_dataset=n_dataset, + n_market=n_market, + n_missing=n_missing, + ) + + +def _score_rounds_with(monkeypatch: pytest.MonkeyPatch, result: _EvalResult) -> Any: + """Run score_rounds() on one fake round whose eval result is `result`.""" + import gate.make_manifest as mm + + async def fake_run_eval(**kwargs: object) -> _EvalResult: # noqa: ANN003, ARG001 + return result + + import eval as ev + + monkeypatch.setattr(ev, "run_eval", fake_run_eval) + return mm.score_rounds(["2026-03-01"]) + + +def test_total_api_failure_fails_loudly(monkeypatch: pytest.MonkeyPatch) -> None: + """100% missing (dead API) must raise, not score 50.0.""" + # n_dataset/n_market are scored rows; n_missing is everything that fell + # through to the 0.5 default. + result = _EvalResult(_scoring(n_dataset=684, n_market=123, n_missing=807)) + with pytest.raises(SystemExit, match="eval failure"): + _score_rounds_with(monkeypatch, result) + + +def test_healthy_missing_fraction_passes(monkeypatch: pytest.MonkeyPatch) -> None: + """<1% missing is normal for healthy runs and must not raise.""" + result = _EvalResult(_scoring(n_dataset=684, n_market=123, n_missing=3)) + mean_index, per_round = _score_rounds_with(monkeypatch, result) + assert len(per_round) == 1 + assert mean_index == pytest.approx(50.0) + + +def test_threshold_sits_between_healthy_and_dead() -> None: + """Healthy runs are <1% missing; total failure is 100%. The guard must + sit well above healthy and well below total failure.""" + assert 0.02 < MAX_MISSING_FRACTION < 0.9