Per-category gate ladders and 4-round training split

gate/make_manifest.py

@@ -40,15 +40,36 @@
 if TYPE_CHECKING:
     from score import ScoringResult
 
+# A round whose forecasts are mostly missing is an eval failure, not a bad
+# forecaster: the lab forecaster never raises, so dead API credentials,
+# exhausted credits, or a routing error silently default everything to 0.5
+# and score a plausible ~50 index (observed 2026-09-04: OpenAI credits out,
+# 100% missing, four rounds "scored" 50.0). Healthy runs are <1% missing;
+# total API failure is exactly 100%. Fail loudly between those.
+MAX_MISSING_FRACTION = 0.10
+
 
 def load_rounds() -> list[str]:
+    """Load pinned gate rounds.
+
+    If GATE_ROUNDS_OVERRIDE is set (comma-separated round names), it takes
+    precedence over gate_rounds.json. This lets the outer loop's OverfitDetector
+    evaluate holdout rounds without modifying the committed gate config.
+    """
+    import os
+
+    override = os.environ.get("GATE_ROUNDS_OVERRIDE")
+    if override:
+        rounds = [r.strip() for r in override.split(",") if r.strip()]
+        if rounds:
+            return rounds
+
     if not ROUNDS.exists():
         raise SystemExit(
             f"{ROUNDS} not found. Run: python gate/make_manifest.py "
             "--rounds 2026-03-01 2026-04-12 2026-05-10"
         )
-    rounds: list[str] = json.loads(ROUNDS.read_text())["rounds"]
-    return rounds
+    return list[str](json.loads(ROUNDS.read_text())["rounds"])
 
 
 def score_rounds(
@@ -101,6 +122,17 @@ def score_rounds(
                 f"Round {name} scored zero rows. Check that it has a published "
                 "resolution set."
             )
+        missing_fraction = scoring.n_missing / (rows + scoring.n_missing)
+        if missing_fraction > MAX_MISSING_FRACTION:
+            raise SystemExit(
+                f"Round {name}: {scoring.n_missing}/{rows + scoring.n_missing} "
+                f"forecasts missing ({missing_fraction:.0%}) exceeds the "
+                f"{MAX_MISSING_FRACTION:.0%} guard. This is almost certainly an "
+                "eval failure, not a bad forecaster — check API credentials, "
+                "credits, and model routing before trusting any score. "
+                "(Missing forecasts default to 0.5, so a dead API silently "
+                "scores ~50.)"
+            )
         per_round.append((name, scoring))
 
     if not per_round:
@@ -119,7 +151,12 @@ def _report(mean_index: float, per_round: list[tuple[str, ScoringResult]]) -> No
             f"market={r.n_market:4d} ({r.market_index:6.2f})  "
             f"missing={r.n_missing}"
         )
-    print(f"mean index across {len(per_round)} rounds: {mean_index:.3f}")
+    mean_dataset = sum(r.dataset_index for _, r in per_round) / len(per_round)
+    mean_market = sum(r.market_index for _, r in per_round) / len(per_round)
+    print(
+        f"mean across {len(per_round)} rounds: "
+        f"overall={mean_index:.3f}  dataset={mean_dataset:.3f}  market={mean_market:.3f}"
+    )
 
 
 def main() -> None:
@@ -143,9 +180,24 @@ def main() -> None:
     if args.set_baseline:
         mean_index, per_round = score_rounds(load_rounds())
         _report(mean_index, per_round)
-        BASELINE.write_text(json.dumps({"brier_index": round(mean_index, 3)}, indent=2))
-        print(f"\nwrote {BASELINE.name}: brier_index = {mean_index:.3f}")
-        print(f"ladder spans {mean_index - 4:.1f} to {mean_index + 5:.1f}")
+
+        mean_dataset = sum(r.dataset_index for _, r in per_round) / len(per_round)
+        mean_market = sum(r.market_index for _, r in per_round) / len(per_round)
+        BASELINE.write_text(
+            json.dumps(
+                {
+                    "overall_index": round(mean_index, 3),
+                    "dataset_index": round(mean_dataset, 3),
+                    "market_index": round(mean_market, 3),
+                },
+                indent=2,
+            )
+        )
+        print(f"\nwrote {BASELINE.name}:")
+        print(f"  overall_index = {mean_index:.3f}  (ladder spans {mean_index - 4:.1f} to {mean_index + 5:.1f})")
+        print(f"  dataset_index = {mean_dataset:.3f}  (ladder spans {mean_dataset - 4:.1f} to {mean_dataset + 5:.1f})")
+        print(f"  market_index  = {mean_market:.3f}  (ladder spans {mean_market - 4:.1f} to {mean_market + 5:.1f})")
+        print("two 10-rung ladders (20 total) — equal weight per category")
 
     if not (args.rounds or args.dry_run or args.set_baseline):
         p.error("nothing to do: pass --rounds, --dry-run, and/or --set-baseline")

gate/test_brier_gate.py

@@ -1,15 +1,20 @@
 """Graded Brier Index gate. This is the outer-loop fitness function.
 
-Scores FULL rounds, one at a time, and averages the per-round Brier Index.
+Scores FULL rounds, one at a time, and averages per-round Brier Index.
 See gate/make_manifest.py::score_rounds for why per-round rather than pooled.
 
+Two independent ladders — dataset and market — so the outer loop gets a
+gradient on each category separately. This matches the competition's
+equal-weight scoring: overall = (dataset_brier + market_brier) / 2.
+A candidate that improves one category while hurting the other will show
+a split signal instead of a masked overall gain — the failure mode that
+made gen-0 calibration look like an improvement when it was a regression.
+
 remote-factory scores a candidate by piping this command's stdout through
-parse_pytest_stdout, which regexes "(\d+) passed" / "(\d+) failed" /
-"(\d+) error", then computes `passed / total - 0.01 * node_count`. A binary
-pass/fail gives that search no gradient, and the contrastive reflector returns
-an empty report when top-K and bottom-K don't differ. Hence a 10-rung ladder
-centered on a committed baseline: at baseline the score is 0.5, with room to
-move both ways.
+parse_pytest_stdout, which regexes "(\\d+) passed" / "(\\d+) failed" /
+"(\\d+) error", then computes `passed / total - 0.01 * node_count`. With
+two 10-rung ladders the total is 20, giving finer resolution (0.05) than
+the original single 10-rung ladder (0.10).
 
 Run:  uv run pytest gate/ -q
 
@@ -40,56 +45,95 @@
 _OFFSETS = list(range(-LADDER_BELOW, LADDER_ABOVE + 1))
 
 
-def _load_baseline() -> float | None:
+def _load_baselines() -> dict[str, float] | None:
+    """Load per-category baselines from gate_baseline.json.
+
+    Returns None (→ skip) if the file is missing or uses the old
+    single-value format. The caller should re-run --set-baseline.
+    """
     if not BASELINE.exists():
         return None
     try:
-        return float(json.loads(BASELINE.read_text())["brier_index"])
+        data = json.loads(BASELINE.read_text())
+        if "dataset_index" not in data or "market_index" not in data:
+            return None  # Old format — needs re-baselining
+        return {
+            "dataset": float(data["dataset_index"]),
+            "market": float(data["market_index"]),
+            "overall": float(data.get("overall_index", (data["dataset_index"] + data["market_index"]) / 2)),
+        }
     except (json.JSONDecodeError, KeyError, ValueError, TypeError):
         return None
 
 
 @pytest.fixture(scope="session")
-def baseline() -> float:
-    """Brier Index the ladder is centered on."""
-    base = _load_baseline()
-    if base is None:
-        pytest.skip("gate_baseline.json missing; run gate/make_manifest.py --set-baseline")
-    return base
+def baselines() -> dict[str, float]:
+    """Per-category Brier Index baselines the ladders are centered on."""
+    bl = _load_baselines()
+    if bl is None:
+        pytest.skip(
+            "gate_baseline.json missing per-category baselines; "
+            "run: python gate/make_manifest.py --set-baseline"
+        )
+    return bl
 
 
 @pytest.fixture(scope="session")
-def brier_index(baseline: float) -> float:
+def scores(baselines: dict[str, float]) -> dict[str, float]:
     """Score every pinned round once per session, not once per rung.
 
-    Depends on `baseline` so a missing baseline skips before any API calls.
+    Depends on `baselines` so a missing baseline skips before any API calls.
+    Returns mean per-category Brier Index across all pinned rounds.
+    Respects GATE_ROUNDS_OVERRIDE env var for holdout evaluation.
     """
-    if not ROUNDS.exists():
-        pytest.skip("gate_rounds.json missing; run gate/make_manifest.py --rounds ...")
+    from gate.make_manifest import load_rounds, score_rounds
 
-    from gate.make_manifest import score_rounds
+    rounds = load_rounds()
+    _, per_round = score_rounds(rounds)
 
-    rounds = json.loads(ROUNDS.read_text())["rounds"]
-    index, per_round = score_rounds(rounds)
+    mean_dataset = sum(r.dataset_index for _, r in per_round) / len(per_round)
+    mean_market = sum(r.market_index for _, r in per_round) / len(per_round)
+    mean_overall = sum(r.overall_index for _, r in per_round) / len(per_round)
 
     print()
     for name, res in per_round:
+        total = res.n_dataset + res.n_market
         print(
-            f"gate: {name} index={res.overall_index:.3f} "
-            f"rows={res.n_dataset + res.n_market} "
-            f"(dataset={res.n_dataset} market={res.n_market} missing={res.n_missing})"
+            f"gate: {name} overall={res.overall_index:.3f} "
+            f"dataset={res.dataset_index:.3f} (n={res.n_dataset}) "
+            f"market={res.market_index:.3f} (n={res.n_market}) "
+            f"missing={res.n_missing} rows={total}"
         )
-    print(f"gate: mean index={index:.3f} across {len(per_round)} rounds")
-    return index
+    print(
+        f"gate: mean across {len(per_round)} rounds — "
+        f"overall={mean_overall:.3f} dataset={mean_dataset:.3f} market={mean_market:.3f}"
+    )
+    return {"dataset": mean_dataset, "market": mean_market, "overall": mean_overall}
+
 
+# ---------------------------------------------------------------------------
+# Two independent ladders: dataset and market.
+# Equal rung count per category = equal weight, matching competition scoring.
+# Total rungs = 2 * len(_OFFSETS) = 20.
+# ---------------------------------------------------------------------------
 
 @pytest.mark.parametrize("offset", _OFFSETS)
-def test_brier_index_at_or_above(brier_index: float, baseline: float, offset: int) -> None:
-    """One rung of the ladder. Pass count is the fitness signal."""
-    threshold = baseline + offset
-    assert brier_index >= threshold, (
-        f"Brier Index {brier_index:.3f} below rung {threshold:.1f} "
-        f"(baseline {baseline:.3f}{offset:+d})"
+def test_dataset_index_at_or_above(scores: dict[str, float], baselines: dict[str, float], offset: int) -> None:
+    """Dataset Brier Index ladder. Pass count is half the fitness signal."""
+    threshold = baselines["dataset"] + offset
+    assert scores["dataset"] >= threshold, (
+        f"Dataset Brier Index {scores['dataset']:.3f} below rung {threshold:.1f} "
+        f"(baseline {baselines['dataset']:.3f}{offset:+d})"
+    )
+
+
+@pytest.mark.parametrize("offset", _OFFSETS)
+def test_market_index_at_or_above(scores: dict[str, float], baselines: dict[str, float], offset: int) -> None:
+    """Market Brier Index ladder. Pass count is half the fitness signal."""
+    threshold = baselines["market"] + offset
+    assert scores["market"] >= threshold, (
+        f"Market Brier Index {scores['market']:.3f} below rung {threshold:.1f} "
+        f"(baseline {baselines['market']:.3f}{offset:+d})"
     )
 
 
@@ -100,8 +144,8 @@ def test_pinned_rounds_well_formed() -> None:
     silently drops rounds produces a plausible number rather than an obvious
     failure. score_rounds() raises on zero rows; this checks the pinned set.
     """
-    if not ROUNDS.exists():
-        pytest.skip("gate_rounds.json missing")
-    rounds = json.loads(ROUNDS.read_text())["rounds"]
+    from gate.make_manifest import load_rounds
+
+    rounds = load_rounds()
     assert len(rounds) >= 1
     assert len(set(rounds)) == len(rounds), "duplicate rounds pinned"

gate_baseline.json

@@ -1,3 +1,5 @@
 {
-  "brier_index": 61.285
+  "overall_index": 50.0,
+  "dataset_index": 50.0,
+  "market_index": 50.0
 }

gate_rounds.json

@@ -1,6 +1,8 @@
 {
   "rounds": [
     "2026-03-01",
+    "2026-03-15",
+    "2026-03-29",
     "2026-04-12"
   ]
-}
+}

tests/test_gate_missing_guard.py

@@ -0,0 +1,80 @@
+"""Tests for the gate's missing-forecast guard (gate/make_manifest.py).
+
+The lab forecaster never raises, so a dead API (expired credentials, exhausted
+credits, wrong routing) silently defaults every forecast to 0.5 and the round
+scores a plausible ~50 Brier Index. On 2026-09-04 four rounds "passed" this
+way with 100% missing forecasts. score_rounds() now fails loudly instead.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+REPO = Path(__file__).resolve().parent.parent
+if str(REPO) not in sys.path:
+    sys.path.insert(0, str(REPO))
+
+from gate.make_manifest import MAX_MISSING_FRACTION  # noqa: E402
+
+
+class _EvalResult:
+    """Minimal stand-in for eval.run_eval's return: only .scoring is used."""
+
+    def __init__(self, scoring: object) -> None:
+        self.scoring = scoring
+
+
+def _scoring(n_dataset: int, n_market: int, n_missing: int) -> Any:
+    from score import ScoringResult
+
+    return ScoringResult(
+        dataset_brier=0.25,
+        dataset_index=50.0,
+        market_brier=0.25,
+        market_index=50.0,
+        overall_brier=0.25,
+        overall_index=50.0,
+        n_dataset=n_dataset,
+        n_market=n_market,
+        n_missing=n_missing,
+    )
+
+
+def _score_rounds_with(monkeypatch: pytest.MonkeyPatch, result: _EvalResult) -> Any:
+    """Run score_rounds() on one fake round whose eval result is `result`."""
+    import gate.make_manifest as mm
+
+    async def fake_run_eval(**kwargs: object) -> _EvalResult:  # noqa: ANN003, ARG001
+        return result
+
+    import eval as ev
+
+    monkeypatch.setattr(ev, "run_eval", fake_run_eval)
+    return mm.score_rounds(["2026-03-01"])
+
+
+def test_total_api_failure_fails_loudly(monkeypatch: pytest.MonkeyPatch) -> None:
+    """100% missing (dead API) must raise, not score 50.0."""
+    # n_dataset/n_market are scored rows; n_missing is everything that fell
+    # through to the 0.5 default.
+    result = _EvalResult(_scoring(n_dataset=684, n_market=123, n_missing=807))
+    with pytest.raises(SystemExit, match="eval failure"):
+        _score_rounds_with(monkeypatch, result)
+
+
+def test_healthy_missing_fraction_passes(monkeypatch: pytest.MonkeyPatch) -> None:
+    """<1% missing is normal for healthy runs and must not raise."""
+    result = _EvalResult(_scoring(n_dataset=684, n_market=123, n_missing=3))
+    mean_index, per_round = _score_rounds_with(monkeypatch, result)
+    assert len(per_round) == 1
+    assert mean_index == pytest.approx(50.0)
+
+
+def test_threshold_sits_between_healthy_and_dead() -> None:
+    """Healthy runs are <1% missing; total failure is 100%. The guard must
+    sit well above healthy and well below total failure."""
+    assert 0.02 < MAX_MISSING_FRACTION < 0.9