diff --git a/bench/surface_bench/metrics.py b/bench/surface_bench/metrics.py index 38856a0..8e4d3b3 100644 --- a/bench/surface_bench/metrics.py +++ b/bench/surface_bench/metrics.py @@ -58,7 +58,9 @@ def _bootstrap_mean(xs: list[float], n_boot: int = 10000, seed: int = 0): def _bootstrap_delta(a: list[float], b: list[float], n_boot: int = 10000, seed: int = 0): - """Bootstrap CI for mean(a) - mean(b).""" + """Bootstrap CI for mean(a) - mean(b), plus an approximate two-sided p-value (the share of the + resampled difference distribution on the far side of zero, doubled). The p-value feeds the + Holm-Bonferroni correction across the comparison family; the CI is still the primary readout.""" if not a or not b: return None rng = random.Random(seed) @@ -70,7 +72,25 @@ def _bootstrap_delta(a: list[float], b: list[float], n_boot: int = 10000, seed: diffs.sort() lo = diffs[int(0.025 * n_boot)] hi = diffs[int(0.975 * n_boot) - 1] - return (sum(a) / len(a) - sum(b) / len(b), lo, hi) + n_le = sum(1 for d in diffs if d <= 0) + n_ge = sum(1 for d in diffs if d >= 0) + p = min(1.0, 2 * min(n_le, n_ge) / n_boot) + return (sum(a) / len(a) - sum(b) / len(b), lo, hi, p) + + +def _holm(entries: list[dict], alpha: float = 0.05) -> None: + """Holm-Bonferroni step-down over delta dicts (each carrying a "p"); sets "significant_holm" on + each in place. Once a hypothesis fails to clear its threshold, it and all weaker ones fail.""" + ranked = sorted(entries, key=lambda d: d.get("p", 1.0)) + m = len(ranked) + still_rejecting = True + for i, d in enumerate(ranked): + thresh = alpha / (m - i) + if still_rejecting and d.get("p", 1.0) <= thresh: + d["significant_holm"] = True + else: + still_rejecting = False + d["significant_holm"] = False def summarize(rows: list[dict]) -> dict: @@ -91,8 +111,9 @@ def summarize(rows: list[dict]) -> dict: "misled_ci": mr.wilson(), } - # Headline deltas (success rate) per model. - pairs = [("C2", "C1"), ("C0", "C1"), ("C3", "C1"), ("C2", "C0")] + # Headline deltas (success rate) per model. C3-Cw / Cw-C1 isolate whether Surface's *corrected + # code* (C3) drives recovery, or merely the suspicion a generic warning (Cw) would also raise. + pairs = [("C2", "C1"), ("C0", "C1"), ("C3", "C1"), ("C2", "C0"), ("C3", "Cw"), ("Cw", "C1")] def delta_block(subset: list[dict], model: str) -> dict: out: dict[str, dict] = {} @@ -101,15 +122,20 @@ def delta_block(subset: list[dict], model: str) -> dict: _samples(subset, model, hi, "ok"), _samples(subset, model, lo, "ok") ) if bs is not None: - point, ci_lo, ci_hi = bs + point, ci_lo, ci_hi, p = bs out[f"{hi}-{lo}"] = { "delta": point, "ci": [ci_lo, ci_hi], + "p": p, "significant": ci_lo > 0 or ci_hi < 0, } return out deltas = {model: delta_block(rows, model) for model in models} + # Holm-Bonferroni across the whole success-delta family (every model x pair), so a handful of + # comparisons don't manufacture significance. CI-significance is kept as the headline; the Holm + # flag is the conservative cross-check for the write-up. + _holm([d for model in models for d in deltas[model].values()]) # The gradient: the same deltas sliced by complexity tier. The headline of the experiment is # that the Surface effect (C2-C1) *grows* as re-deriving truth from code gets more expensive. @@ -151,7 +177,7 @@ def out_tokens(model, cond, *, only=None): for hi, lo in [("C1", "C2"), ("C1", "C0"), ("C1", "C3")]: bs = _bootstrap_delta(out_tokens(model, hi), out_tokens(model, lo)) if bs is not None: - point, ci_lo, ci_hi = bs + point, ci_lo, ci_hi, _p = bs token_deltas[model][f"{hi}-{lo}"] = { "delta": point, "ci": [ci_lo, ci_hi], @@ -172,7 +198,83 @@ def spend(rows_subset: list[dict]) -> float: }, } - return { + # Verification (multi-turn only): the headline of the agentic track. Among rows where the agent + # *could* read the hidden dependency, did it? A confident stale doc (C1) should suppress that + # check relative to no doc (C0) — H4 — and within C1 the verifiers should be right while the + # non-verifiers are misled (H5, the mediation). + multi = [r for r in rows if "verified_hidden" in r] + verification: dict[str, dict[str, dict]] = defaultdict(dict) + verification_deltas: dict[str, dict[str, dict]] = defaultdict(dict) + mediation: dict[str, dict] = {} + if multi: + for model in models: + for cond in conditions: + ver = _samples(multi, model, cond, "verified_hidden") + vr = Rate(len(ver), sum(ver)) + # success among rows that actually verified — verifying should rescue you + vt = [ + r + for r in multi + if r["model"] == model and r["condition"] == cond and r.get("verified_hidden") + ] + vt_ok = sum(1 for r in vt if r.get("ok")) / len(vt) if vt else None + verification[model][cond] = { + "n": vr.n, + "verification_rate": vr.rate, + "verification_ci": vr.wilson(), + "n_verified": vr.k, + "verified_then_correct": vt_ok, + } + # H4: does the stale doc suppress verification vs no doc? (C0 − C1, positive = suppressed) + for hi, lo in [("C0", "C1"), ("C2", "C1")]: + bs = _bootstrap_delta( + _samples(multi, model, hi, "verified_hidden"), + _samples(multi, model, lo, "verified_hidden"), + ) + if bs is not None: + point, ci_lo, ci_hi, p = bs + verification_deltas[model][f"{hi}-{lo}"] = { + "delta": point, + "ci": [ci_lo, ci_hi], + "p": p, + "significant": ci_lo > 0 or ci_hi < 0, + } + # H5 mediation: within C1, success for verifiers vs non-verifiers. + c1 = [r for r in multi if r["model"] == model and r["condition"] == "C1"] + ver_rows = [r for r in c1 if r.get("verified_hidden")] + unver_rows = [r for r in c1 if not r.get("verified_hidden")] + mediation[model] = { + "n_verified": len(ver_rows), + "verified_success": (sum(bool(r.get("ok")) for r in ver_rows) / len(ver_rows)) if ver_rows else None, + "n_unverified": len(unver_rows), + "unverified_success": (sum(bool(r.get("ok")) for r in unver_rows) / len(unver_rows)) if unver_rows else None, + } + + # Per-scenario breakdown — catches a single broken fixture hiding inside a family average (and is + # what the C2-fresh oracle reads). Verification rate is included where the run was multi-turn. + scenarios = sorted({r["scenario"] for r in rows}) + by_scenario: dict[str, dict[str, dict]] = {} + for sc in scenarios: + sub = [r for r in rows if r["scenario"] == sc] + by_scenario[sc] = {} + for model in models: + cells = {} + for cond in conditions: + ok = _samples(sub, model, cond, "ok") + if not ok: + continue + cell = {"n": len(ok), "success": sum(ok) / len(ok)} + vh = [ + r + for r in sub + if r["model"] == model and r["condition"] == cond and "verified_hidden" in r + ] + if vh: + cell["verification_rate"] = sum(bool(r["verified_hidden"]) for r in vh) / len(vh) + cells[cond] = cell + by_scenario[sc][model] = cells + + out = { "models": models, "conditions": conditions, "tiers": tiers, @@ -182,4 +284,10 @@ def spend(rows_subset: list[dict]) -> float: "by_tier": by_tier, "tokens": tokens, "token_deltas": token_deltas, + "by_scenario": by_scenario, } + if multi: + out["verification"] = verification + out["verification_deltas"] = verification_deltas + out["mediation"] = mediation + return out diff --git a/bench/surface_bench/oracle.py b/bench/surface_bench/oracle.py new file mode 100644 index 0000000..fe751ac --- /dev/null +++ b/bench/surface_bench/oracle.py @@ -0,0 +1,68 @@ +"""Post-run sanity tripwires for a results dir — cheap checks that catch authoring/harness bugs +before they reach the write-up (the failure mode recorded in issue #113). + + python -m surface_bench.oracle results/ + +Checks (per scenario x model): + * C2-fresh ≈ 100% — with a *fresh* doc and the code readable, the task must be solvable; a low + cell means the scenario is mis-authored (a leaked stale value, a broken grader, an impossible + task), not a real effect. + * cascade C1 misleads at all — if a stale doc never produces the wrong answer, the drift isn't + load-bearing and the scenario measures nothing. + +The surf_report `changed` divergence is sealed at authoring time by tools/author.py, so it isn't +re-checked here. Exit code is non-zero if any tripwire fires, so this can gate a run in CI. +""" + +from __future__ import annotations + +import sys + +from .metrics import load_rows + +C2_FRESH_THRESHOLD = 0.9 + + +def check(rows: list[dict], *, c2_threshold: float = C2_FRESH_THRESHOLD) -> list[str]: + warnings: list[str] = [] + scenarios = sorted({r["scenario"] for r in rows}) + models = sorted({r["model"] for r in rows}) + for sc in scenarios: + for m in models: + def cell(cond: str) -> list[dict]: + return [ + r for r in rows if r["scenario"] == sc and r["model"] == m and r["condition"] == cond + ] + + c2 = cell("C2") + if c2: + sr = sum(bool(r.get("ok")) for r in c2) / len(c2) + if sr < c2_threshold: + warnings.append( + f"C2-fresh low: {sc} / {m} = {sr:.0%} (< {c2_threshold:.0%}) " + "— likely an authoring bug, not a real effect" + ) + if sc.startswith("cascade-"): + c1 = cell("C1") + if c1 and not any(r.get("misled") for r in c1): + warnings.append( + f"C1 never misleads: {sc} / {m} — the drift may not be load-bearing" + ) + return warnings + + +def main() -> None: + if len(sys.argv) != 2: + sys.exit("usage: python -m surface_bench.oracle results/") + warnings = check(load_rows(sys.argv[1])) + if not warnings: + print("oracle: PASS — all tripwires clear") + return + print(f"oracle: {len(warnings)} warning(s):") + for w in warnings: + print(f" - {w}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/bench/surface_bench/prompts.py b/bench/surface_bench/prompts.py index aaa2fd2..35face9 100644 --- a/bench/surface_bench/prompts.py +++ b/bench/surface_bench/prompts.py @@ -11,13 +11,14 @@ from .scenarios import Scenario -CONDITIONS = ("C0", "C1", "C2", "C3") +CONDITIONS = ("C0", "C1", "C2", "C3", "Cw") CONDITION_LABEL = { "C0": "code only (no documentation)", "C1": "code + stale documentation", "C2": "code + fresh documentation", "C3": "code + stale documentation + surf divergence report", + "Cw": "code + stale documentation + generic staleness warning", } # Deliberately minimal and persona-free: no "you are an expert…" framing (which primes diligent, @@ -58,6 +59,16 @@ def _doc_block(scenario: Scenario, condition: str) -> str: "claim was last confirmed:\n\n" f"```json\n{scenario.surf_report.rstrip()}\n```" ) + if condition == "Cw": + # Control isolating Surface's *specific* contribution: a generic staleness warning with no + # corrected code, no file, no line, no value. If Cw recovers like C3, the value was just + # "any skepticism prompt"; if Cw stays at C1, only Surface's concrete fix helps. Keep this + # deliberately content-free — leaking the truth here would collapse the C3-vs-Cw contrast. + block += ( + "\n\n## Note\n\n" + "This documentation may be out of date. It was last reviewed some time ago and might " + "no longer reflect the current behaviour of the code." + ) return block diff --git a/bench/surface_bench/report.py b/bench/surface_bench/report.py index 20741e7..34dcea1 100644 --- a/bench/surface_bench/report.py +++ b/bench/surface_bench/report.py @@ -17,6 +17,8 @@ "C0-C1": "no-docs vs stale (rotted worse than nothing?)", "C3-C1": "surf-report vs stale (does surfacing drift recover it?)", "C2-C0": "fresh vs no-docs (value of accurate prose)", + "C3-Cw": "surf-report vs bare warning (is the value the *fix*, not just suspicion?)", + "Cw-C1": "bare warning vs stale (does a generic warning alone help?)", } TOKEN_DELTA_GLOSS = { @@ -91,10 +93,70 @@ def _spend_section(summary: dict) -> list[str]: return lines +def _verification_section(summary: dict) -> list[str]: + """The headline of the multi-turn track: a confident stale doc should suppress the file-read the + agent would otherwise do (H4), and within C1 verifiers should be right, non-verifiers misled (H5).""" + ver = summary.get("verification") + if not ver: + return [] + lines = [ + "## Verification — did the agent check the hidden dependency? (multi-turn)\n", + "With no doc the agent should go read the hidden code; a confident **stale** doc should " + "suppress that check (H4). Within C1, those who verified should be correct (H5).\n", + ] + for model in summary["models"]: + lines.append(f"### {model}\n") + lines.append("| Condition | n | Verified hidden dep | When verified, correct |") + lines.append("|---|---|---|---|") + for cond in summary["conditions"]: + v = ver.get(model, {}).get(cond) + if not v: + continue + lines.append( + f"| {cond} | {v['n']} | {_pct(v['verification_rate'])}{_ci(v['verification_ci'])} " + f"| {_pct(v['verified_then_correct'])} |" + ) + for key, d in summary.get("verification_deltas", {}).get(model, {}).items(): + sig = " ✓ significant" if d["significant"] else "" + lines.append( + f"\n- verification `{key}`: {100 * d['delta']:+.0f} pp " + f"[{100 * d['ci'][0]:+.0f}, {100 * d['ci'][1]:+.0f}]{sig}" + ) + med = summary.get("mediation", {}).get(model) + if med and (med["n_verified"] or med["n_unverified"]): + lines.append( + f"- C1 mediation (H5): verifiers {_pct(med['verified_success'])} correct " + f"(n={med['n_verified']}) vs non-verifiers {_pct(med['unverified_success'])} " + f"(n={med['n_unverified']})" + ) + lines.append("") + return lines + + +def _per_scenario_section(summary: dict) -> list[str]: + """Per-scenario success, so one broken fixture can't hide inside a family average.""" + bs = summary.get("by_scenario") + if not bs: + return [] + conds = summary["conditions"] + lines = ["## Per-scenario success rate\n"] + for model in summary["models"]: + lines.append(f"### {model}\n") + lines.append("| Scenario | " + " | ".join(conds) + " |") + lines.append("|" + "---|" * (len(conds) + 1)) + for sc in sorted(bs): + cells = bs[sc].get(model, {}) + row = [sc] + [_pct(cells[c]["success"]) if c in cells else "—" for c in conds] + lines.append("| " + " | ".join(row) + " |") + lines.append("") + return lines + + def render_markdown(summary: dict) -> str: lines = ["# Surface agent-impact benchmark\n"] lines += _spend_section(summary) lines += _gradient_table(summary) + lines += _verification_section(summary) for model in summary["models"]: lines.append(f"## {model}\n") lines.append("| Condition | n | Success | Misled |") @@ -109,10 +171,11 @@ def render_markdown(summary: dict) -> str: lines.append("\n**Deltas (success rate, 95% bootstrap CI):**\n") for key, d in summary["deltas"][model].items(): sig = " ✓ significant" if d["significant"] else "" + holm = " (Holm ✓)" if d.get("significant_holm") else (" (Holm ✗)" if "significant_holm" in d else "") gloss = DELTA_GLOSS.get(key, "") lines.append( f"- `{key}` {gloss}: {100 * d['delta']:+.0f} pp " - f"[{100 * d['ci'][0]:+.0f}, {100 * d['ci'][1]:+.0f}]{sig}" + f"[{100 * d['ci'][0]:+.0f}, {100 * d['ci'][1]:+.0f}]{sig}{holm}" ) toks = summary.get("tokens", {}).get(model, {}) @@ -135,6 +198,7 @@ def render_markdown(summary: dict) -> str: f"[{d['ci'][0]:+.0f}, {d['ci'][1]:+.0f}]{sig}" ) lines.append("") + lines += _per_scenario_section(summary) return "\n".join(lines) @@ -164,12 +228,13 @@ def maybe_plot(summary: dict, out_dir: Path) -> None: # unknown names fall back to alphabetical after the known ones. rank = {"haiku": 0, "sonnet": 1, "opus": 2} models = sorted(summary["models"], key=lambda m: (rank.get(m, 99), m)) - conds = [c for c in ("C0", "C1", "C2", "C3") if c in summary["conditions"]] + conds = [c for c in ("C0", "C1", "C2", "C3", "Cw") if c in summary["conditions"]] label = { "C0": "No docs", "C1": "Stale docs", "C2": "Fresh docs\n(Surface)", "C3": "Stale docs +\nSurface report", + "Cw": "Stale docs +\nstaleness warning", } palette = ["#4C72B0", "#DD8452", "#55A868", "#C44E52", "#8172B3"] color = {m: palette[i % len(palette)] for i, m in enumerate(models)} @@ -250,6 +315,25 @@ def grouped(ax, sub, valfn, ylabel, *, ymax=None, fmt="{:.0f}"): fig.savefig(out_dir / "cascade_success.png", dpi=150, bbox_inches="tight") plt.close(fig) + # Verification hero (multi-turn only): does a confident stale doc stop the agent checking? + if casc and any("verified_hidden" in r for r in rows): + + def ver_pct(sub, m, c): + x = [r for r in cell(sub, m, c) if "verified_hidden" in r] + return 100 * sum(bool(r["verified_hidden"]) for r in x) / len(x) if x else 0.0 + + fig, ax = plt.subplots(figsize=(6.6, 4.3)) + grouped(ax, casc, ver_pct, "Agent read the hidden dependency (%)", ymax=108, fmt="{:.0f}%") + ax.legend(title="model", fontsize=8, title_fontsize=8, frameon=False, loc="upper right") + ax.set_title( + "Does a confident (stale) doc stop the agent verifying?\n" + "With no doc the agent reads the hidden code; a stale doc suppresses the check", + fontsize=9.5, + ) + fig.tight_layout() + fig.savefig(out_dir / "verification_rate.png", dpi=150, bbox_inches="tight") + plt.close(fig) + def main() -> None: if len(sys.argv) != 2: diff --git a/bench/tests/test_phase6.py b/bench/tests/test_phase6.py new file mode 100644 index 0000000..29ea1ac --- /dev/null +++ b/bench/tests/test_phase6.py @@ -0,0 +1,109 @@ +"""Offline tests for the Cw control (phase 5) and the verification metrics / Holm / per-scenario / +oracle layer (phase 6).""" + +from __future__ import annotations + +from pathlib import Path + +from surface_bench import oracle +from surface_bench.metrics import summarize +from surface_bench.prompts import CONDITIONS, build_prompt +from surface_bench.scenarios import load_scenario + +BENCH_ROOT = Path(__file__).resolve().parents[1] + + +# ---- Phase 5: Cw control -------------------------------------------------------------------- + + +def test_cw_condition_is_stale_plus_generic_warning_only() -> None: + assert "Cw" in CONDITIONS + scenario = load_scenario(BENCH_ROOT / "scenarios" / "cascade-quota-batcher-code") + _, cw = build_prompt(scenario, "Cw") + _, c1 = build_prompt(scenario, "C1") + _, c3 = build_prompt(scenario, "C3") + + # Cw = the stale hub (like C1) + a generic warning, but WITHOUT surf's corrected code (unlike C3). + assert scenario.hub_stale.strip()[:40] in cw # carries the stale doc + assert "may be out of date" in cw + assert "Automated documentation check" not in cw # no surf report + assert "new_code" not in cw # no corrected value leaked + assert cw != c1 and cw != c3 + + +# ---- Phase 6: verification metrics ---------------------------------------------------------- + + +def _row(sc, cond, ok, misled, ver=None, tok=100): + r = { + "scenario": sc, + "model": "m", + "condition": cond, + "tier": "T1", + "ok": ok, + "misled": misled, + "output_tokens": tok, + } + if ver is not None: + r["verified_hidden"] = ver + return r + + +def _multi_rows(): + rows = [] + sc = "cascade-x" + rows += [_row(sc, "C0", True, False, ver=True) for _ in range(10)] # no doc -> verifies, correct + # stale doc -> mostly skips verification; verifiers right, non-verifiers misled (H5) + rows += [_row(sc, "C1", True, False, ver=True) for _ in range(2)] + rows += [_row(sc, "C1", False, True, ver=False) for _ in range(8)] + rows += [_row(sc, "C2", True, False, ver=False) for _ in range(10)] # fresh -> correct + rows += [_row(sc, "C3", True, False, ver=False) for _ in range(10)] # surf report -> correct + rows += [_row(sc, "Cw", False, True, ver=False) for _ in range(10)] # bare warning -> no help + return rows + + +def test_verification_block_and_mediation() -> None: + s = summarize(_multi_rows()) + assert "verification" in s + ver = s["verification"]["m"] + assert ver["C0"]["verification_rate"] == 1.0 + assert ver["C1"]["verification_rate"] == 0.2 + # H4: a stale doc suppresses verification vs no doc + assert s["verification_deltas"]["m"]["C0-C1"]["delta"] > 0 + # H5 mediation: within C1 verifiers are correct, non-verifiers are not + med = s["mediation"]["m"] + assert med["verified_success"] == 1.0 and med["n_verified"] == 2 + assert med["unverified_success"] == 0.0 and med["n_unverified"] == 8 + + +def test_cw_pairs_and_holm_on_success_deltas() -> None: + s = summarize(_multi_rows()) + d = s["deltas"]["m"] + assert "C3-Cw" in d and "Cw-C1" in d # the control contrasts + assert d["C3-Cw"]["delta"] > 0 # surf's fix beats a bare warning + for entry in d.values(): + assert "p" in entry and "significant_holm" in entry + + +def test_per_scenario_breakdown_carries_verification() -> None: + s = summarize(_multi_rows()) + cell = s["by_scenario"]["cascade-x"]["m"]["C0"] + assert cell["success"] == 1.0 + assert cell["verification_rate"] == 1.0 + + +# ---- Phase 6: oracle tripwires -------------------------------------------------------------- + + +def test_oracle_passes_clean_run() -> None: + assert oracle.check(_multi_rows()) == [] + + +def test_oracle_flags_low_c2_and_non_misleading_cascade() -> None: + rows = [] + rows += [_row("cascade-broken", "C2", False, False, ver=False) for _ in range(10)] # C2 fail + rows += [_row("cascade-inert", "C1", False, False, ver=False) for _ in range(10)] # never misleads + rows += [_row("cascade-inert", "C2", True, False, ver=False) for _ in range(10)] + warnings = oracle.check(rows) + assert any("C2-fresh low" in w and "cascade-broken" in w for w in warnings) + assert any("never misleads" in w and "cascade-inert" in w for w in warnings)