From 68b441fa33006ffb21b9d1f8a66dfab109bbbdc6 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Fri, 3 Jul 2026 15:13:18 -0400 Subject: [PATCH 01/15] feat(schemas): canonical repair-record + rollout-record schemas (ST1 AC1) --- .../repair/iter-002.json} | 0 schemas/repair-record.schema.json | 42 +++++++++ schemas/rollout-record.schema.json | 27 ++++++ scripts/test_schemas_metrics.py | 88 +++++++++++++++++++ 4 files changed, 157 insertions(+) rename examples/coverage-repair/{repair-record.json => .loop/repair/iter-002.json} (100%) create mode 100644 schemas/repair-record.schema.json create mode 100644 schemas/rollout-record.schema.json create mode 100644 scripts/test_schemas_metrics.py diff --git a/examples/coverage-repair/repair-record.json b/examples/coverage-repair/.loop/repair/iter-002.json similarity index 100% rename from examples/coverage-repair/repair-record.json rename to examples/coverage-repair/.loop/repair/iter-002.json diff --git a/schemas/repair-record.schema.json b/schemas/repair-record.schema.json new file mode 100644 index 0000000..8206d4c --- /dev/null +++ b/schemas/repair-record.schema.json @@ -0,0 +1,42 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "loop-engineer/repair@1", + "title": "Loop Engineer Repair Record @1", + "description": "The single canonical repair record (QW11/M4). One bounded repair pass emitted by loop-repair to .loop/repair/.json. Its verification_before/verification_after scores are repair-productivity's canonical input: productive == (verification_after.score > verification_before.score). Distinct from the rollout / candidate ledger record (loop-engineer/rollout@1), which adjudicates rollout candidates. The 7 canonical fields match evals/cases/structural.json repair_record_fields; the schema/iteration_id/attempt envelope matches examples/coverage-repair. additionalProperties is true because a record carries loop-specific evidence keys (metric, failing, verify_full).", + "type": "object", + "required": [ + "schema", + "iteration_id", + "attempt", + "failure_mode", + "hypothesis", + "repair_action", + "verification_before", + "verification_after", + "remaining_delta", + "productive" + ], + "properties": { + "schema": { "const": "loop-engineer/repair@1" }, + "iteration_id": { "type": ["string", "integer"] }, + "attempt": { "type": "integer", "minimum": 1 }, + "failure_mode": { "type": "string", "minLength": 1 }, + "hypothesis": { "type": "string", "minLength": 1 }, + "repair_action": { "type": "string", "minLength": 1 }, + "verification_before": { + "type": "object", + "required": ["score"], + "properties": { "score": { "type": "number" } }, + "additionalProperties": true + }, + "verification_after": { + "type": "object", + "required": ["score"], + "properties": { "score": { "type": "number" } }, + "additionalProperties": true + }, + "remaining_delta": { "type": "string" }, + "productive": { "type": "boolean" } + }, + "additionalProperties": true +} diff --git a/schemas/rollout-record.schema.json b/schemas/rollout-record.schema.json new file mode 100644 index 0000000..7ee4499 --- /dev/null +++ b/schemas/rollout-record.schema.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "loop-engineer/rollout@1", + "title": "Loop Engineer Rollout / Candidate Ledger Record @1", + "description": "One candidate adjudication in a rollout / genetic-hardening loop (scripts/rollout_ledger.py RECORD_FIELDS). Appended one JSON object per line to .loop/*.jsonl. This is NOT the repair record (loop-engineer/repair@1): its productive field is the rollout-productivity signal (productive == (score_delta is not None and score_delta > 0)), a flywheel view, not the repair-productivity baseline. Records do not carry a schema envelope field today; additionalProperties is true so one may.", + "type": "object", + "required": [ + "id", + "parent", + "verdict", + "score", + "score_delta", + "coherent_with_prior_winner", + "productive" + ], + "properties": { + "schema": { "const": "loop-engineer/rollout@1" }, + "id": { "type": "string", "minLength": 1 }, + "parent": { "type": ["string", "null"] }, + "verdict": { "type": "string" }, + "score": { "type": ["number", "null"] }, + "score_delta": { "type": ["number", "null"] }, + "coherent_with_prior_winner": { "type": "boolean" }, + "productive": { "type": "boolean" } + }, + "additionalProperties": true +} diff --git a/scripts/test_schemas_metrics.py b/scripts/test_schemas_metrics.py new file mode 100644 index 0000000..b6fd540 --- /dev/null +++ b/scripts/test_schemas_metrics.py @@ -0,0 +1,88 @@ +"""AC1 schema tests: the two canonical record schemas validate the shipped +example repair record and a rollout-ledger fixture. Runs a stdlib structural +check always, and full JSON-Schema validation when jsonschema is installed.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +_REPO = Path(__file__).resolve().parent.parent +_SCHEMAS = _REPO / "schemas" + + +def _load(path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def _structural_ok(schema: dict, instance: dict) -> list[str]: + """Minimal stdlib validation: required keys present + `const` fields match.""" + errors = [] + for key in schema.get("required", []): + if key not in instance: + errors.append(f"missing required key {key!r}") + for key, subschema in schema.get("properties", {}).items(): + if key in instance and isinstance(subschema, dict) and "const" in subschema: + if instance[key] != subschema["const"]: + errors.append(f"{key} != const {subschema['const']!r}") + return errors + + +def _jsonschema_errors(schema: dict, instance: dict) -> list[str]: + jsonschema = pytest.importorskip("jsonschema") + validator = jsonschema.Draft202012Validator(schema) + return [e.message for e in validator.iter_errors(instance)] + + +_ROLLOUT_FIXTURE = { + "id": "cand-2", + "parent": "cand-1", + "verdict": "Succeeded", + "score": 0.90, + "score_delta": 0.10, + "coherent_with_prior_winner": True, + "productive": True, +} + + +def test_repair_schema_exists_with_expected_id(): + schema = _load(_SCHEMAS / "repair-record.schema.json") + assert schema["$id"] == "loop-engineer/repair@1" + + +def test_rollout_schema_exists_with_expected_id(): + schema = _load(_SCHEMAS / "rollout-record.schema.json") + assert schema["$id"] == "loop-engineer/rollout@1" + + +def test_repair_schema_validates_shipped_example_record(): + schema = _load(_SCHEMAS / "repair-record.schema.json") + record = _load(_REPO / "examples" / "coverage-repair" / ".loop" / "repair" / "iter-002.json") + assert _structural_ok(schema, record) == [] + assert record["verification_before"]["score"] is not None + assert record["verification_after"]["score"] is not None + + +def test_rollout_schema_validates_ledger_fixture(): + schema = _load(_SCHEMAS / "rollout-record.schema.json") + assert _structural_ok(schema, _ROLLOUT_FIXTURE) == [] + + +def test_repair_schema_jsonschema_validation_of_example(): + schema = _load(_SCHEMAS / "repair-record.schema.json") + record = _load(_REPO / "examples" / "coverage-repair" / ".loop" / "repair" / "iter-002.json") + assert _jsonschema_errors(schema, record) == [] + + +def test_rollout_schema_jsonschema_validation_of_fixture(): + schema = _load(_SCHEMAS / "rollout-record.schema.json") + assert _jsonschema_errors(schema, _ROLLOUT_FIXTURE) == [] + + +def test_repair_schema_rejects_missing_score(): + schema = _load(_SCHEMAS / "repair-record.schema.json") + record = _load(_REPO / "examples" / "coverage-repair" / ".loop" / "repair" / "iter-002.json") + del record["verification_after"]["score"] + assert _jsonschema_errors(schema, record) != [] From 8592f1ae8bc9020ee1fe27b00645d1dbbd20e79c Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Fri, 3 Jul 2026 15:13:22 -0400 Subject: [PATCH 02/15] feat(metrics): derive FCR/RP from real .loop evidence with recompute-and-reject (ST1 AC2-AC5) --- scripts/metrics.py | 468 ++++++++++++++++++++++++++++++++++++++++ scripts/test_metrics.py | 321 +++++++++++++++++++++++++++ 2 files changed, 789 insertions(+) create mode 100644 scripts/metrics.py create mode 100644 scripts/test_metrics.py diff --git a/scripts/metrics.py b/scripts/metrics.py new file mode 100644 index 0000000..8805544 --- /dev/null +++ b/scripts/metrics.py @@ -0,0 +1,468 @@ +"""Derive false-completion-rate (FCR) and repair-productivity (RP) from a loop. + +The runnable core of ST1: point it at a loop dir and it computes the two +first-class metrics (`reference/eval-suite.md` §2) from that loop's *real* on-disk +evidence — RUNLOG success claims, deterministic verify bundles, the held-out gate +verdict, the canonical repair records, and receipts — never from the agent's +narration. Every headline number ships with a `provenance` block so a skeptic can +re-derive it by hand. + +Two honesty invariants distinguish this from a self-report: + + * **`productive` is recomputed, never trusted.** RP is aggregated only over + repair records whose stored `productive` agrees with the value recomputed from + `verification_before`/`verification_after.score` (`recheck_productive`). A + record that disagrees, or cannot demonstrate a score delta, is *rejected* and + excluded — reported under `provenance.rejected_records`, not silently coerced. + * **FCR is derived two ways and disagreement is surfaced.** (a) the RUNLOG + success-claim × verify-bundle cross-join (the deterministic anchor, per §3), + and (b) the aggregated held-out-gate `false_completion` flag. An unmatched + success-claim counts as a false completion (fail-closed, §8). + +`--baseline` writes a checked-in scorecard, but only over a genuinely gate-backed +run: it refuses (non-zero, writes nothing) if the run is not `evidence_backed` or +if any record was rejected. Baselining a self-asserted run would itself be a false +completion of ST1. + +Pure stdlib, offline, deterministic: the same loop dir yields a byte-identical +scorecard. + +Run:: + + python3 metrics.py + python3 metrics.py --baseline +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parent.parent +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +from loop.paths import LoopPaths, resolve_loop_paths # noqa: E402 + +import re # noqa: E402 + +METRICS_SCHEMA = "loop-engineer/metrics@1" +BASELINE_OUTPUT = "docs/metrics-baseline.json" + +# A RUNLOG iteration whose outcome declares a task/loop reached "done". A +# repair_triggered / task_failed outcome is an honest red, NOT a success claim. +_SUCCESS_OUTCOME_TOKENS = ("task_passed", "terminal", "succeeded", "advanced") + +# Gate tokens (mirrors inspect_loop._GATE_TOKENS): a real held-out / anti-cheat +# gate invocation writes one of these into the execution trail. +_GATE_TOKENS = ("holdout_gate", "anticheat_scan", "anti_cheat") + +_ITER_HEADER_RE = re.compile(r"(?m)^##\s+Iteration\s+(\S+)") +# "outcome" declaration followed (within a little markup/whitespace) by its token. +_OUTCOME_RE = re.compile(r"outcome[^A-Za-z0-9]{0,40}?([A-Za-z][A-Za-z_]{2,})", re.IGNORECASE | re.DOTALL) +_VERIFY_REF_RE = re.compile(r"(verify-[\w.-]+?\.json)") + + +def _read_text(path: Path) -> str: + try: + return path.read_text(encoding="utf-8") + except OSError: + return "" + + +def _read_json_object(path: Path) -> dict: + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {} + return data if isinstance(data, dict) else {} + + +def _num(value) -> float | None: + """A real number, or None. ``bool`` is not a number (True is not a score).""" + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return float(value) + return None + + +def _norm_iter(value) -> str: + return str(value).strip() + + +# --- recheck_productive: the shared, never-trust-the-flag validator (§4.3) ----- + + +def recheck_productive(record: dict) -> dict: + """Recompute a record's ``productive`` from its own evidence and compare. + + Returns a verdict dict — ``kind`` (``repair``/``rollout``/``unknown``), + ``stored``, ``expected``, ``valid`` (stored agrees with a computable + expected), and ``reason``. Both the metrics command (RP) and + ``rollout_ledger.summarize`` consume it; neither ever sums a caller-supplied + boolean verbatim. + """ + + stored = record.get("productive") + stored_is_bool = isinstance(stored, bool) + + has_before_after = isinstance(record.get("verification_before"), dict) and isinstance( + record.get("verification_after"), dict + ) + + if has_before_after: + kind = "repair" + before = _num(record["verification_before"].get("score")) + after = _num(record["verification_after"].get("score")) + if before is None or after is None: + return { + "kind": kind, + "stored": stored if stored_is_bool else None, + "expected": None, + "valid": False, + "reason": "missing numeric verification_before/after score", + } + expected = after > before + elif "score_delta" in record: + kind = "rollout" + delta = _num(record.get("score_delta")) + expected = delta is not None and delta > 0 + else: + return { + "kind": "unknown", + "stored": stored if stored_is_bool else None, + "expected": None, + "valid": False, + "reason": "unrecognized record shape (no verification_before/after or score_delta)", + } + + if not stored_is_bool: + return { + "kind": kind, + "stored": None, + "expected": expected, + "valid": False, + "reason": "productive is missing or not a boolean", + } + if stored != expected: + return { + "kind": kind, + "stored": stored, + "expected": expected, + "valid": False, + "reason": f"stored productive={stored} disagrees with recomputed {expected}", + } + return {"kind": kind, "stored": stored, "expected": expected, "valid": True, "reason": "ok"} + + +# --- RUNLOG / verify-bundle parsing (deterministic, evidence-only) ------------- + + +def _runlog_blocks(runlog_text: str) -> list[tuple[str, str]]: + """Split RUNLOG.md into (iteration_id, block_text) pairs, in file order.""" + matches = list(_ITER_HEADER_RE.finditer(runlog_text)) + blocks: list[tuple[str, str]] = [] + for i, m in enumerate(matches): + end = matches[i + 1].start() if i + 1 < len(matches) else len(runlog_text) + blocks.append((_norm_iter(m.group(1)), runlog_text[m.start():end])) + return blocks + + +def _block_claims_success(block_text: str) -> bool: + tokens = [t.lower() for t in _OUTCOME_RE.findall(block_text)] + return any(t in _SUCCESS_OUTCOME_TOKENS for t in tokens) + + +def _load_verify_bundles(loop_dir: Path) -> list[dict]: + bundles: list[dict] = [] + for path in sorted(loop_dir.rglob("verify-*.json")): + if "archive" in path.parts: + continue + data = _read_json_object(path) + if not data: + continue + outcome = str(data.get("outcome", "")).upper() + green = outcome == "PASS" or data.get("passed") is True + it = data.get("iteration_id", data.get("iteration")) + bundles.append( + { + "path": path, + "name": path.name, + "green": green, + "iter": _norm_iter(it) if it is not None else None, + } + ) + return bundles + + +def _assign_bundles_to_iters(bundles: list[dict], blocks: list[tuple[str, str]]) -> tuple[dict, list[str]]: + """Key each verify bundle to an iteration_id: its own iteration field first, + else the RUNLOG block that references its filename. Unassignable bundles are + returned separately (surfaced under provenance).""" + refs_by_iter = {iid: set(_VERIFY_REF_RE.findall(text)) for iid, text in blocks} + by_iter: dict[str, list[dict]] = {} + unmatched: list[str] = [] + for b in bundles: + target = b["iter"] + if target is None: + target = next((iid for iid, refs in refs_by_iter.items() if b["name"] in refs), None) + if target is None: + unmatched.append(b["name"]) + else: + by_iter.setdefault(target, []).append(b) + return by_iter, sorted(unmatched) + + +def _load_gate_verdicts(loop_dir: Path) -> list[dict]: + """Held-out / anti-cheat gate verdict artifacts (holdout_gate.decide output).""" + verdicts: list[dict] = [] + for path in sorted(loop_dir.rglob("*.json")): + if "archive" in path.parts: + continue + data = _read_json_object(path) + if "false_completion" in data and ("verdict" in data or "passed_visible" in data): + verdicts.append({"path": path, "data": data}) + return verdicts + + +def _gate_invoked(paths: LoopPaths, gate_verdicts: list[dict]) -> bool: + """Is a real gate invocation detectable — a recorded verdict artifact, or a + gate token in the RUNLOG / verify scripts / task verify commands? Mirrors the + inspector's invocation-evidence rule (HI4).""" + if gate_verdicts: + return True + haystacks = [_read_text(paths.runlog)] + scripts_dir = paths.workspace / "scripts" + for name in ("verify-fast", "verify-fast.sh", "verify-full", "verify-full.sh", + "verify-safety", "verify-safety.sh"): + haystacks.append(_read_text(scripts_dir / name)) + tasks = _read_json_object(paths.tasks).get("tasks") + if isinstance(tasks, list): + haystacks.extend(str(row.get("verify", "")) for row in tasks if isinstance(row, dict)) + blob = "\n".join(haystacks).lower() + return any(tok in blob for tok in _GATE_TOKENS) + + +def _load_receipt_costs(loop_dir: Path) -> tuple[float | None, int]: + """Sum ``cost_usd`` across receipts; None if no receipt carries a cost.""" + total: float | None = None + count = 0 + for path in sorted((loop_dir / "receipts").glob("*.jsonl")): + for line in _read_text(path).splitlines(): + line = line.strip() + if not line: + continue + try: + rec = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(rec, dict): + continue + count += 1 + cost = _num(rec.get("cost_usd")) + if cost is not None: + total = cost if total is None else total + cost + return total, count + + +def _rel(path: Path, base: Path) -> str: + try: + return str(path.relative_to(base)) + except ValueError: + return str(path) + + +# --- the scorecard ------------------------------------------------------------- + + +def compute_metrics(loop_dir: str | Path, loop_label: str | None = None) -> dict: + paths = resolve_loop_paths(loop_dir) + workspace = paths.workspace + loop_dir_path = paths.loop_dir + label = loop_label if loop_label is not None else str(loop_dir) + + runlog_text = _read_text(paths.runlog) + terminal = _read_json_object(paths.terminal) + blocks = _runlog_blocks(runlog_text) + + bundles = _load_verify_bundles(loop_dir_path) + verify_by_iter, unmatched_verify = _assign_bundles_to_iters(bundles, blocks) + + # Success claims: RUNLOG success-outcome iterations, plus the terminal claim. + claim_iters: set[str] = {iid for iid, text in blocks if _block_claims_success(text)} + if terminal.get("state") == "Succeeded": + tid = _norm_iter(terminal.get("iteration_id")) + claim_iters.add(tid) + # The terminal names the verify bundles that back its success claim. + evidence = terminal.get("evidence") + if isinstance(evidence, list): + wanted = {Path(str(e)).name for e in evidence} + for b in bundles: + if b["name"] in wanted: + verify_by_iter.setdefault(tid, []) + if b not in verify_by_iter[tid]: + verify_by_iter[tid].append(b) + + # FCR-A: a success claim with no green deterministic verify is a false + # completion (unmatched claims fail closed, §8). + false_completions = sum( + 1 for iid in claim_iters if not any(b["green"] for b in verify_by_iter.get(iid, [])) + ) + n_claims = len(claim_iters) + fcr_a = (false_completions / n_claims) if n_claims else 0.0 + + # FCR-B: aggregated held-out-gate false_completion flag. + gate_verdicts = _load_gate_verdicts(loop_dir_path) + fc_flagged = sum(1 for v in gate_verdicts if v["data"].get("false_completion") is True) + fcr_b = (fc_flagged / len(gate_verdicts)) if gate_verdicts else None + fcr_methods_agree = None if fcr_b is None else (fcr_a == fcr_b) + + evidence_backed = _gate_invoked(paths, gate_verdicts) + + # RP: over recomputed-and-agreed repair records only. + validated = 0 + productive = 0 + rejected: list[dict] = [] + rp_source: list[str] = [] + for path in sorted((loop_dir_path / "repair").glob("*.json")): + record = _read_json_object(path) + verdict = recheck_productive(record) + rel = _rel(path, workspace) + if verdict["valid"]: + validated += 1 + rp_source.append(rel) + if verdict["expected"]: + productive += 1 + else: + rejected.append({"record": rel, "reason": verdict["reason"]}) + repair_productivity = (productive / validated) if validated else None + + # Cost-per-success (layer 7): total receipt cost over true completions. + total_cost, _receipt_count = _load_receipt_costs(loop_dir_path) + successes = n_claims - false_completions + cost_per_success = (total_cost / successes) if (total_cost is not None and successes > 0) else None + + fcr_source = sorted({_rel(paths.runlog, workspace)} | {_rel(b["path"], workspace) for b in bundles}) + holdout_source = sorted(_rel(v["path"], workspace) for v in gate_verdicts) + rejected.sort(key=lambda r: r["record"]) + + return { + "schema": METRICS_SCHEMA, + "loop": label, + "false_completion_rate": fcr_a, + "repair_productivity": repair_productivity, + "iterations_claiming_success": n_claims, + "false_completions": false_completions, + "repair_passes": validated, + "productive_repairs": productive, + "cost_per_success_usd": cost_per_success, + "evidence_backed": evidence_backed, + "provenance": { + "fcr_source": fcr_source, + "rp_source": sorted(rp_source), + "rejected_records": rejected, + "false_completion_rate_holdout": fcr_b, + "fcr_methods_agree": fcr_methods_agree, + "holdout_source": holdout_source, + "unmatched_verify": unmatched_verify, + }, + } + + +def _input_files(loop_dir: str | Path) -> list[str]: + """The exact committed files the scorecard is derived from, repo-relative.""" + paths = resolve_loop_paths(loop_dir) + loop_dir_path = paths.loop_dir + candidates: list[Path] = [paths.runlog, paths.terminal, paths.tasks] + candidates += sorted(loop_dir_path.rglob("verify-*.json")) + candidates += [v["path"] for v in _load_gate_verdicts(loop_dir_path)] + candidates += sorted((loop_dir_path / "repair").glob("*.json")) + candidates += sorted((loop_dir_path / "receipts").glob("*.jsonl")) + seen: list[str] = [] + for p in candidates: + if "archive" in p.parts or not p.exists(): + continue + rel = _rel(p, _REPO_ROOT) + if rel not in seen: + seen.append(rel) + return sorted(seen) + + +def _git_commit() -> str | None: + try: + out = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=_REPO_ROOT, capture_output=True, text=True + ) + except OSError: + return None + return out.stdout.strip() or None if out.returncode == 0 else None + + +def build_baseline(loop_dir: str | Path, loop_label: str | None = None) -> tuple[bool, dict, list[str]]: + """Compute the scorecard and check the §4.5 baseline preconditions. + + Returns ``(ok, scorecard, refusal_reasons)``. ``ok`` is False iff the run is + not evidence-backed or contains rejected records. + """ + scorecard = compute_metrics(loop_dir, loop_label) + reasons: list[str] = [] + if not scorecard["evidence_backed"]: + reasons.append( + "run is not evidence_backed — no held-out / anti-cheat gate invocation " + "detectable in the verify trail or RUNLOG" + ) + rejected = scorecard["provenance"]["rejected_records"] + if rejected: + reasons.append( + f"{len(rejected)} rejected record(s) present (productive disagrees with its own evidence)" + ) + return (not reasons), scorecard, reasons + + +def write_baseline(loop_dir: str | Path, out_path: Path, loop_label: str | None = None) -> int: + ok, scorecard, reasons = build_baseline(loop_dir, loop_label) + if not ok: + print( + "metrics --baseline refused (writes nothing):\n - " + "\n - ".join(reasons), + file=sys.stderr, + ) + return 1 + baseline = dict(scorecard) + baseline["baseline"] = { + "source_example": scorecard["loop"], + "commit": _git_commit(), + "inputs": _input_files(loop_dir), + } + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(baseline, indent=2) + "\n", encoding="utf-8") + print(f"wrote baseline -> {_rel(out_path, _REPO_ROOT)}") + return 0 + + +def run(argv: list[str]) -> int: + baseline_mode = False + positional: list[str] = [] + for arg in argv: + if arg == "--baseline": + baseline_mode = True + else: + positional.append(arg) + if not positional: + print("usage: metrics.py [--baseline] ", file=sys.stderr) + return 2 + loop_dir = positional[0] + if baseline_mode: + return write_baseline(loop_dir, _REPO_ROOT / BASELINE_OUTPUT, loop_label=loop_dir) + print(json.dumps(compute_metrics(loop_dir, loop_label=loop_dir), indent=2)) + return 0 + + +def main(argv: list[str] | None = None) -> int: + return run(list(sys.argv[1:] if argv is None else argv)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_metrics.py b/scripts/test_metrics.py new file mode 100644 index 0000000..09d2127 --- /dev/null +++ b/scripts/test_metrics.py @@ -0,0 +1,321 @@ +"""Tests for scripts/metrics.py — FCR / RP derivation, recompute-and-reject, and +the gated baseline (ST1 AC2-AC5). Fixtures build real .loop/ trees under tmp_path +so the command is exercised on evidence, never narration.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import metrics + +_REPO = Path(__file__).resolve().parent.parent +_EXAMPLE = _REPO / "examples" / "coverage-repair" + + +# --- fixture builder ---------------------------------------------------------- + + +def _write(path: Path, data) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data) if not isinstance(data, str) else data, encoding="utf-8") + + +def _make_loop( + tmp_path: Path, + *, + runlog: str, + verify: dict[str, dict] | None = None, + repair: dict[str, dict] | None = None, + terminal: dict | None = None, + gate_verdict: dict | None = None, + receipts: list[dict] | None = None, +) -> Path: + ws = tmp_path / "ws" + (ws / ".loop").mkdir(parents=True, exist_ok=True) + _write(ws / "RUNLOG.md", runlog) + for name, bundle in (verify or {}).items(): + _write(ws / ".loop" / "artifacts" / name, bundle) + for name, record in (repair or {}).items(): + _write(ws / ".loop" / "repair" / name, record) + if terminal is not None: + _write(ws / ".loop" / "terminal_state.json", terminal) + if gate_verdict is not None: + _write(ws / ".loop" / "artifacts" / "holdout-verdict.json", gate_verdict) + if receipts is not None: + lines = "\n".join(json.dumps(r) for r in receipts) + "\n" + _write(ws / ".loop" / "receipts" / "run.jsonl", lines) + return ws + + +def _repair_record(before: float, after: float, productive: bool) -> dict: + return { + "schema": "loop-engineer/repair@1", + "iteration_id": "iter-001", + "attempt": 1, + "failure_mode": "deterministic-fail", + "hypothesis": "h", + "repair_action": "a", + "verification_before": {"score": before}, + "verification_after": {"score": after}, + "remaining_delta": "none", + "productive": productive, + } + + +# --- recheck_productive (AC2) ------------------------------------------------- + + +def test_recheck_productive_repair_agrees_when_score_improved(): + v = metrics.recheck_productive(_repair_record(0.74, 0.83, True)) + assert v["kind"] == "repair" + assert v["expected"] is True + assert v["valid"] is True + + +def test_recheck_productive_repair_agrees_on_churn(): + v = metrics.recheck_productive(_repair_record(0.80, 0.80, False)) + assert v["expected"] is False + assert v["valid"] is True + + +def test_recheck_productive_repair_rejects_disagreement(): + v = metrics.recheck_productive(_repair_record(0.80, 0.80, True)) # stored lies + assert v["valid"] is False + assert "disagree" in v["reason"] + + +def test_recheck_productive_repair_rejects_missing_score(): + rec = _repair_record(0.74, 0.83, True) + del rec["verification_after"]["score"] + v = metrics.recheck_productive(rec) + assert v["valid"] is False + assert "score" in v["reason"] + + +def test_recheck_productive_rollout_agrees_on_positive_delta(): + rec = {"id": "c1", "parent": None, "verdict": "ok", "score": 0.9, + "score_delta": 0.1, "coherent_with_prior_winner": True, "productive": True} + v = metrics.recheck_productive(rec) + assert v["kind"] == "rollout" + assert v["valid"] is True + + +def test_recheck_productive_rollout_rejects_disagreement(): + rec = {"id": "c1", "parent": None, "verdict": "ok", "score": 0.9, + "score_delta": 0.0, "coherent_with_prior_winner": True, "productive": True} + v = metrics.recheck_productive(rec) + assert v["valid"] is False + + +def test_recheck_productive_rollout_null_delta_is_not_productive(): + rec = {"id": "c1", "parent": None, "verdict": "ok", "score": None, + "score_delta": None, "coherent_with_prior_winner": True, "productive": False} + v = metrics.recheck_productive(rec) + assert v["expected"] is False + assert v["valid"] is True + + +def test_recheck_productive_unknown_shape_is_rejected(): + v = metrics.recheck_productive({"productive": True}) + assert v["kind"] == "unknown" + assert v["valid"] is False + + +# --- FCR cross-join (AC3/AC4) ------------------------------------------------- + + +_RUNLOG_ONE_CLAIM = ( + "# RUNLOG\n\n## Iteration 1 — t\n\n### Outcome\n\n`task_passed`\n" + "- **evidence:** .loop/artifacts/verify-A.json\n" +) + + +def test_fcr_is_one_when_claim_not_backed_by_green_verify(tmp_path): + ws = _make_loop( + tmp_path, + runlog=_RUNLOG_ONE_CLAIM, + verify={"verify-A.json": {"task": "A", "outcome": "FAIL", "score": 0.5}}, + ) + sc = metrics.compute_metrics(ws) + assert sc["iterations_claiming_success"] == 1 + assert sc["false_completions"] == 1 + assert sc["false_completion_rate"] == 1.0 + + +def test_fcr_is_zero_when_claim_backed_by_green_verify(tmp_path): + ws = _make_loop( + tmp_path, + runlog=_RUNLOG_ONE_CLAIM, + verify={"verify-A.json": {"task": "A", "outcome": "PASS", "score": 1.0}}, + ) + sc = metrics.compute_metrics(ws) + assert sc["false_completions"] == 0 + assert sc["false_completion_rate"] == 0.0 + + +def test_fcr_unmatched_success_claim_fails_closed(tmp_path): + # A claim with no verify bundle at all is a false completion (§8 fail-closed). + ws = _make_loop(tmp_path, runlog=_RUNLOG_ONE_CLAIM) + sc = metrics.compute_metrics(ws) + assert sc["false_completions"] == 1 + assert sc["false_completion_rate"] == 1.0 + + +# --- evidence_backed (AC4) ---------------------------------------------------- + + +def test_evidence_backed_false_when_claim_set_but_gate_never_run(tmp_path): + ws = _make_loop( + tmp_path, + runlog=_RUNLOG_ONE_CLAIM, + verify={"verify-A.json": {"task": "A", "outcome": "PASS", "score": 1.0}}, + terminal={"state": "Succeeded", "iteration_id": 1, + "criteria_met": {"1": True}, "false_completion": False, + "evidence": [".loop/artifacts/verify-A.json"]}, + ) + sc = metrics.compute_metrics(ws) + assert sc["evidence_backed"] is False + + +def test_evidence_backed_true_when_holdout_verdict_present(tmp_path): + ws = _make_loop( + tmp_path, + runlog=_RUNLOG_ONE_CLAIM, + verify={"verify-A.json": {"task": "A", "outcome": "PASS", "score": 1.0}}, + gate_verdict={"verdict": "Succeeded", "passed_visible": True, + "passed_holdout": True, "false_completion": False}, + ) + sc = metrics.compute_metrics(ws) + assert sc["evidence_backed"] is True + assert sc["provenance"]["false_completion_rate_holdout"] == 0.0 + assert sc["provenance"]["fcr_methods_agree"] is True + + +# --- RP (AC2) ----------------------------------------------------------------- + + +def test_rp_is_half_over_one_productive_and_one_churn(tmp_path): + prod = _repair_record(0.5, 0.8, True) + prod["iteration_id"] = "iter-001" + churn = _repair_record(0.8, 0.8, False) + churn["iteration_id"] = "iter-002" + ws = _make_loop( + tmp_path, + runlog=_RUNLOG_ONE_CLAIM, + repair={"iter-001.json": prod, "iter-002.json": churn}, + ) + sc = metrics.compute_metrics(ws) + assert sc["repair_passes"] == 2 + assert sc["productive_repairs"] == 1 + assert sc["repair_productivity"] == 0.5 + + +def test_rp_excludes_a_lying_repair_record(tmp_path): + honest = _repair_record(0.5, 0.8, True) + honest["iteration_id"] = "iter-001" + liar = _repair_record(0.8, 0.8, True) # churn asserted productive + liar["iteration_id"] = "iter-002" + ws = _make_loop( + tmp_path, + runlog=_RUNLOG_ONE_CLAIM, + repair={"iter-001.json": honest, "iter-002.json": liar}, + ) + sc = metrics.compute_metrics(ws) + assert sc["repair_passes"] == 1 # only the honest one counts + assert sc["repair_productivity"] == 1.0 + rejected = sc["provenance"]["rejected_records"] + assert len(rejected) == 1 + assert rejected[0]["record"].endswith("iter-002.json") + + +# --- determinism (AC3) -------------------------------------------------------- + + +def test_scorecard_is_byte_identical_across_runs(tmp_path): + prod = _repair_record(0.5, 0.8, True) + ws = _make_loop( + tmp_path, + runlog=_RUNLOG_ONE_CLAIM, + verify={"verify-A.json": {"task": "A", "outcome": "PASS", "score": 1.0}}, + repair={"iter-001.json": prod}, + gate_verdict={"verdict": "Succeeded", "passed_visible": True, + "passed_holdout": True, "false_completion": False}, + receipts=[{"schema": "loop-engineer/receipt@1", "iteration_id": 1, + "role": "write", "model": "opus", "outcome": "ok", "cost_usd": 0.4}], + ) + first = json.dumps(metrics.compute_metrics(ws, loop_label="ws"), indent=2) + second = json.dumps(metrics.compute_metrics(ws, loop_label="ws"), indent=2) + assert first == second + + +def test_cost_per_success_from_receipts(tmp_path): + ws = _make_loop( + tmp_path, + runlog=_RUNLOG_ONE_CLAIM, + verify={"verify-A.json": {"task": "A", "outcome": "PASS", "score": 1.0}}, + receipts=[{"cost_usd": 0.3}, {"cost_usd": 0.1}], + ) + sc = metrics.compute_metrics(ws) + assert sc["cost_per_success_usd"] == 0.4 # 0.4 total / 1 success + + +# --- baseline gating (AC5) ---------------------------------------------------- + + +def test_baseline_refuses_non_evidence_backed_run_and_writes_nothing(tmp_path): + ws = _make_loop( + tmp_path, + runlog=_RUNLOG_ONE_CLAIM, + verify={"verify-A.json": {"task": "A", "outcome": "PASS", "score": 1.0}}, + ) + out = tmp_path / "docs" / "metrics-baseline.json" + rc = metrics.write_baseline(ws, out, loop_label="ws") + assert rc != 0 + assert not out.exists() + + +def test_baseline_refuses_when_a_record_is_rejected(tmp_path): + liar = _repair_record(0.8, 0.8, True) + ws = _make_loop( + tmp_path, + runlog=_RUNLOG_ONE_CLAIM, + verify={"verify-A.json": {"task": "A", "outcome": "PASS", "score": 1.0}}, + repair={"iter-001.json": liar}, + gate_verdict={"verdict": "Succeeded", "passed_visible": True, + "passed_holdout": True, "false_completion": False}, + ) + ok, _sc, reasons = metrics.build_baseline(ws, "ws") + assert ok is False + assert any("rejected" in r for r in reasons) + + +def test_baseline_writes_scorecard_over_gate_backed_run(tmp_path): + prod = _repair_record(0.5, 0.8, True) + ws = _make_loop( + tmp_path, + runlog=_RUNLOG_ONE_CLAIM, + verify={"verify-A.json": {"task": "A", "outcome": "PASS", "score": 1.0}}, + repair={"iter-001.json": prod}, + gate_verdict={"verdict": "Succeeded", "passed_visible": True, + "passed_holdout": True, "false_completion": False}, + ) + out = tmp_path / "docs" / "metrics-baseline.json" + rc = metrics.write_baseline(ws, out, loop_label="ws") + assert rc == 0 + written = json.loads(out.read_text()) + assert written["schema"] == "loop-engineer/metrics@1" + assert written["baseline"]["source_example"] == "ws" + assert "inputs" in written["baseline"] + + +# --- flagship regression (pins the published baseline numbers) ---------------- + + +def test_metrics_on_flagship_example_is_clean_and_evidence_backed(): + sc = metrics.compute_metrics(_EXAMPLE, loop_label="examples/coverage-repair") + assert sc["false_completion_rate"] == 0.0 + assert sc["repair_productivity"] == 1.0 + assert sc["evidence_backed"] is True + assert sc["provenance"]["rejected_records"] == [] + assert sc["provenance"]["fcr_methods_agree"] is True From 5b2b8ffcb9fad751c6a188041d5654cc989a1340 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Fri, 3 Jul 2026 15:13:26 -0400 Subject: [PATCH 03/15] feat(rollout-ledger): reject-on-disagree via shared recheck_productive (ST1 AC2) --- scripts/rollout_ledger.py | 53 +++++++++++++++++++++++++--------- scripts/test_rollout_ledger.py | 28 ++++++++++++++---- 2 files changed, 61 insertions(+), 20 deletions(-) diff --git a/scripts/rollout_ledger.py b/scripts/rollout_ledger.py index a4e4295..5ed2ee5 100644 --- a/scripts/rollout_ledger.py +++ b/scripts/rollout_ledger.py @@ -1,11 +1,18 @@ -"""Append-only JSONL rollout ledger — the durable record of loop candidates (G8). +"""Append-only JSONL rollout / candidate ledger (G8, schema loop-engineer/rollout@1). -A rollout/repair loop produces a stream of *candidates*: each is a proposed -change (a repair, a harden, a config mutation) that the loop scored and -adjudicated against the prior winner. This ledger is where that stream lands — +A rollout/hardening loop produces a stream of *candidates*: each is a proposed +change (a harden, a config mutation, a rollout adjudication) that the loop scored +and adjudicated against the prior winner. This ledger is where that stream lands — one JSON object per line, appended and never rewritten, so the lineage survives compaction and session loss. +**This is the rollout / candidate ledger record, NOT the repair record.** The +canonical repair record (``schemas/repair-record.schema.json``, +``loop-engineer/repair@1``, on disk at ``.loop/repair/.json``) is +what repair-productivity (RP) is derived from — see ``reference/eval-suite.md`` +§2.2. This record's ``productive`` is the separate *rollout*-productivity signal +(a flywheel view of candidate adjudication), not the RP baseline. + Each record carries EXACTLY these 7 fields: * ``id`` — the candidate's id. @@ -16,12 +23,13 @@ * ``coherent_with_prior_winner`` — does it preserve the prior winner's gains? * ``productive`` — did it *measurably* improve the score? -``productive`` is the per-candidate signal behind the suite's -repair-productivity metric: a repair that does not move the score is churn, not -progress. ``summarize`` aggregates that signal into the productive fraction. +``productive`` is never trusted verbatim (M3/HI5): ``summarize`` recomputes it +from ``score_delta`` via the shared ``recheck_productive`` validator and +**rejects** any record whose stored flag disagrees, so the productive fraction is +a derivation, not a self-report. This ships as composable tooling, not a runtime: a loop calls ``append`` at each -adjudication and ``summarize`` when it wants the repair-productivity readout. +adjudication and ``summarize`` when it wants the rollout-productivity readout. Run:: @@ -108,16 +116,33 @@ def read(path: str | Path) -> list[dict]: def summarize(path: str | Path) -> dict: - """Compute repair-productivity (productive fraction), the candidate count, and - the number of malformed lines skipped.""" + """Compute rollout-productivity (productive fraction) over *validated* records. + + ``productive`` is recomputed from ``score_delta`` via the shared + ``recheck_productive`` validator; a record whose stored flag disagrees is + rejected (counted under ``rejected``, excluded from the fraction) rather than + summed verbatim. ``malformed`` counts unparseable lines skipped by the reader. + """ + from metrics import recheck_productive + records, malformed = _read_with_stats(path) - count = len(records) - productive = sum(1 for r in records if r.get("productive")) - repair_productivity = productive / count if count else 0.0 + validated = 0 + productive = 0 + rejected = 0 + for record in records: + verdict = recheck_productive(record) + if verdict["valid"]: + validated += 1 + if verdict["expected"]: + productive += 1 + else: + rejected += 1 + repair_productivity = productive / validated if validated else 0.0 return { - "count": count, + "count": validated, "productive": productive, "repair_productivity": repair_productivity, + "rejected": rejected, "malformed": malformed, } diff --git a/scripts/test_rollout_ledger.py b/scripts/test_rollout_ledger.py index 4943abf..04de3ff 100644 --- a/scripts/test_rollout_ledger.py +++ b/scripts/test_rollout_ledger.py @@ -64,11 +64,12 @@ def test_every_record_has_exactly_the_seven_fields(tmp_path): def test_summarize_computes_productive_fraction_and_count(tmp_path): - # Arrange + # Arrange — `productive` must agree with score_delta (summarize recomputes it + # and rejects disagreement): a productive candidate carries a positive delta. ledger = tmp_path / "rollout.jsonl" - rollout_ledger.append(_candidate(id="cand-1", productive=False), ledger) - rollout_ledger.append(_candidate(id="cand-2", productive=True), ledger) - rollout_ledger.append(_candidate(id="cand-3", productive=True), ledger) + rollout_ledger.append(_candidate(id="cand-1", score_delta=0.0, productive=False), ledger) + rollout_ledger.append(_candidate(id="cand-2", score_delta=0.1, productive=True), ledger) + rollout_ledger.append(_candidate(id="cand-3", score_delta=0.1, productive=True), ledger) # Act summary = rollout_ledger.summarize(ledger) @@ -145,11 +146,11 @@ def test_read_warns_to_stderr_on_malformed_line(tmp_path, capsys): def test_summarize_counts_malformed_lines(tmp_path): ledger = tmp_path / "rollout.jsonl" - rollout_ledger.append(_candidate(id="cand-1", productive=True), ledger) + rollout_ledger.append(_candidate(id="cand-1", score_delta=0.1, productive=True), ledger) with ledger.open("a", encoding="utf-8") as fh: fh.write("garbage line\n") fh.write("also not json {,,}\n") - rollout_ledger.append(_candidate(id="cand-2", productive=True), ledger) + rollout_ledger.append(_candidate(id="cand-2", score_delta=0.1, productive=True), ledger) summary = rollout_ledger.summarize(ledger) @@ -157,3 +158,18 @@ def test_summarize_counts_malformed_lines(tmp_path): assert summary["count"] == 2 assert summary["malformed"] == 2 assert summary["repair_productivity"] == 1.0 + + +def test_summarize_rejects_record_whose_productive_disagrees_with_delta(tmp_path): + # HI5/M3: a record claiming productive:true on a zero delta is a lie — + # summarize recomputes from score_delta, rejects it, and excludes it from the + # productive fraction rather than summing the flag verbatim. + ledger = tmp_path / "rollout.jsonl" + rollout_ledger.append(_candidate(id="honest", score_delta=0.1, productive=True), ledger) + rollout_ledger.append(_candidate(id="liar", score_delta=0.0, productive=True), ledger) + + summary = rollout_ledger.summarize(ledger) + + assert summary["count"] == 1 # only the honest record is validated + assert summary["rejected"] == 1 + assert summary["repair_productivity"] == 1.0 From 67bee4d4e8c45868a4572c41c89adb5860232381 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Fri, 3 Jul 2026 15:13:29 -0400 Subject: [PATCH 04/15] feat(contract): validate repair/rollout/receipt records when present (ST1 AC6) --- loop/contract.py | 77 +++++++++++++++++++++++ scripts/test_contract_records.py | 102 +++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 scripts/test_contract_records.py diff --git a/loop/contract.py b/loop/contract.py index 37fe5fd..057a32c 100644 --- a/loop/contract.py +++ b/loop/contract.py @@ -290,6 +290,82 @@ def _jsonschema_validate(data: dict[str, Any], name: str, path: Path, issues: li issues.append(ContractIssue("schema_violation", f"{path.name}: {location}: {err.message}", path)) +# The FCR/RP evidentiary trail (M5): repair records and receipt/rollout ledgers. +# Validated OPTIONALLY — only when the files exist — so an in-flight loop that has +# not emitted them yet still passes, while a loop that ships malformed metric +# inputs can no longer pass validation with them unchecked. +_RECORD_SCHEMA_FILES = { + "repair": "repair-record.schema.json", + "rollout": "rollout-record.schema.json", + "receipt": "receipt.schema.json", +} + + +def _load_schema_file(filename: str) -> dict[str, Any]: + return json.loads((_schemas_dir() / filename).read_text(encoding="utf-8")) + + +def _structural_record_check(data: dict[str, Any], schema: dict[str, Any], path: Path, issues: list[dict]) -> None: + props = schema.get("properties", {}) + for key in schema.get("required", []): + if key not in data: + issues.append(ContractIssue("invalid_record", f"{path.name}: missing {key}", path)) + for key, sub in props.items(): + if key in data and isinstance(sub, dict) and "const" in sub and data[key] != sub["const"]: + issues.append(ContractIssue("schema_mismatch", f"{path.name}: {key} != {sub['const']!r}", path)) + for vk in ("verification_before", "verification_after"): + sub = props.get(vk) + if isinstance(sub, dict) and "score" in sub.get("required", []) and vk in data: + value = data[vk] + score = value.get("score") if isinstance(value, dict) else None + if isinstance(score, bool) or not isinstance(score, (int, float)): + issues.append(ContractIssue("invalid_record", f"{path.name}: {vk}.score must be numeric", path)) + + +def _validate_record(data: dict[str, Any], schema_key: str, path: Path, mode: str, issues: list[dict]) -> None: + filename = _RECORD_SCHEMA_FILES[schema_key] + if mode == "jsonschema": + import jsonschema # type: ignore + + validator = jsonschema.Draft202012Validator(_load_schema_file(filename)) + for err in validator.iter_errors(data): + location = "/".join(str(p) for p in err.absolute_path) or "" + issues.append(ContractIssue("schema_violation", f"{path.name}: {location}: {err.message}", path)) + else: + _structural_record_check(data, _load_schema_file(filename), path, issues) + + +def _validate_jsonl(path: Path, schema_key: str, mode: str, issues: list[dict]) -> None: + for lineno, line in enumerate(path.read_text(encoding="utf-8", errors="ignore").splitlines(), start=1): + line = line.strip() + if not line: + continue + try: + data = json.loads(line) + except json.JSONDecodeError as exc: + issues.append(ContractIssue("invalid_json", f"{path.name}:{lineno}: {exc}", path)) + continue + if not isinstance(data, dict): + issues.append(ContractIssue("invalid_record", f"{path.name}:{lineno}: expected object", path)) + continue + _validate_record(data, schema_key, path, mode, issues) + + +def _validate_optional_records(paths: LoopPaths, mode: str, issues: list[dict]) -> None: + repair_dir = paths.loop_dir / "repair" + if repair_dir.is_dir(): + for record_path in sorted(repair_dir.glob("*.json")): + data = _read_json(record_path, issues) + if data is not None: + _validate_record(data, "repair", record_path, mode, issues) + for ledger_path in sorted(paths.loop_dir.glob("*.jsonl")): + _validate_jsonl(ledger_path, "rollout", mode, issues) + receipts_dir = paths.loop_dir / "receipts" + if receipts_dir.is_dir(): + for receipt_path in sorted(receipts_dir.glob("*.jsonl")): + _validate_jsonl(receipt_path, "receipt", mode, issues) + + def validate_contract(target: str | Path) -> dict[str, Any]: paths = resolve_loop_paths(target) issues: list[dict] = [] @@ -328,6 +404,7 @@ def validate_contract(target: str | Path) -> dict[str, Any]: if not paths.runlog.exists(): issues.append(ContractIssue("missing_file", "missing RUNLOG.md", paths.runlog)) _check_stub_verify_scripts(paths, issues) + _validate_optional_records(paths, mode, issues) return { "ok": not issues, diff --git a/scripts/test_contract_records.py b/scripts/test_contract_records.py new file mode 100644 index 0000000..313a5ec --- /dev/null +++ b/scripts/test_contract_records.py @@ -0,0 +1,102 @@ +"""AC6: validate_contract() validates .loop/repair/*.json and .loop/*.jsonl (and +.loop/receipts/*.jsonl) against their schemas WHEN PRESENT — absence is never an +error, and the shipped example / repo contract still validate clean.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from loop.contract import _validate_optional_records, validate_contract # noqa: E402 +from loop.paths import resolve_loop_paths # noqa: E402 + + +def _valid_repair() -> dict: + return { + "schema": "loop-engineer/repair@1", + "iteration_id": "iter-001", + "attempt": 1, + "failure_mode": "deterministic-fail", + "hypothesis": "h", + "repair_action": "a", + "verification_before": {"score": 0.5}, + "verification_after": {"score": 0.9}, + "remaining_delta": "none", + "productive": True, + } + + +def _optional_issues(loop_dir: Path) -> list[dict]: + paths = resolve_loop_paths(loop_dir) + issues: list[dict] = [] + _validate_optional_records(paths, "structural-fallback", issues) + return issues + + +def test_absent_record_files_are_not_an_error(tmp_path): + (tmp_path / ".loop").mkdir() + assert _optional_issues(tmp_path) == [] + + +def test_present_valid_repair_record_passes(tmp_path): + repair_dir = tmp_path / ".loop" / "repair" + repair_dir.mkdir(parents=True) + (repair_dir / "iter-001.json").write_text(json.dumps(_valid_repair()), encoding="utf-8") + assert _optional_issues(tmp_path) == [] + + +def test_present_repair_record_missing_field_is_flagged(tmp_path): + repair_dir = tmp_path / ".loop" / "repair" + repair_dir.mkdir(parents=True) + bad = _valid_repair() + del bad["hypothesis"] + (repair_dir / "iter-001.json").write_text(json.dumps(bad), encoding="utf-8") + issues = _optional_issues(tmp_path) + assert any("hypothesis" in i["message"] for i in issues) + + +def test_present_repair_record_non_numeric_score_is_flagged(tmp_path): + repair_dir = tmp_path / ".loop" / "repair" + repair_dir.mkdir(parents=True) + bad = _valid_repair() + del bad["verification_after"]["score"] + (repair_dir / "iter-001.json").write_text(json.dumps(bad), encoding="utf-8") + issues = _optional_issues(tmp_path) + assert any("verification_after.score" in i["message"] for i in issues) + + +def test_present_rollout_jsonl_bad_line_is_flagged(tmp_path): + loop_dir = tmp_path / ".loop" + loop_dir.mkdir() + good = {"id": "c1", "parent": None, "verdict": "ok", "score": 0.9, + "score_delta": 0.1, "coherent_with_prior_winner": True, "productive": True} + bad = {"id": "c2"} # missing the rest of the required rollout fields + (loop_dir / "rollout.jsonl").write_text( + json.dumps(good) + "\n" + json.dumps(bad) + "\n", encoding="utf-8" + ) + issues = _optional_issues(tmp_path) + assert any("rollout.jsonl" in i["message"] for i in issues) + + +def test_present_valid_receipt_jsonl_passes(tmp_path): + receipts = tmp_path / ".loop" / "receipts" + receipts.mkdir(parents=True) + rec = {"schema": "loop-engineer/receipt@1", "iteration_id": 1, + "role": "write", "model": "opus", "outcome": "ok"} + (receipts / "run.jsonl").write_text(json.dumps(rec) + "\n", encoding="utf-8") + assert _optional_issues(tmp_path) == [] + + +def test_flagship_example_contract_validates_clean_with_repair_record(): + report = validate_contract(ROOT / "examples" / "coverage-repair") + assert report["ok"] is True, report["issues"] + + +def test_repo_own_contract_still_validates_clean(): + report = validate_contract(ROOT / ".loop") + assert report["ok"] is True, report["issues"] From bb383f707451ac56af72ebb6a1f01895e2c97d12 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Fri, 3 Jul 2026 15:13:33 -0400 Subject: [PATCH 05/15] feat(cli): wire 'loop metrics' subcommand + console-script entry (ST1 Rider A/B) --- loop/__main__.py | 48 ++++++++++++++-- pyproject.toml | 8 +++ scripts/test_metrics_cli.py | 111 ++++++++++++++++++++++++++++++++++++ 3 files changed, 163 insertions(+), 4 deletions(-) create mode 100644 scripts/test_metrics_cli.py diff --git a/loop/__main__.py b/loop/__main__.py index b635b6c..d80b422 100644 --- a/loop/__main__.py +++ b/loop/__main__.py @@ -9,17 +9,18 @@ _PROG = "python3 -m loop" -_COMMANDS = ("scaffold", "doctor", "validate", "verify", "inspect") +_COMMANDS = ("scaffold", "doctor", "validate", "verify", "inspect", "metrics") # Read commands operate on an EXISTING contract dir; scaffold CREATES one, so it # is exempt from the "target must exist" guard. -_READ_COMMANDS = ("doctor", "validate", "verify", "inspect") +_READ_COMMANDS = ("doctor", "validate", "verify", "inspect", "metrics") -_USAGE = f"usage: {_PROG} " +_USAGE = f"usage: {_PROG} " -_HELP = f"""{_PROG} — validate and inspect a portable repo-OS loop contract. +_HELP = f"""{_PROG} — validate, inspect, and measure a portable repo-OS loop contract. {_USAGE} + {_PROG} metrics [--baseline] commands: scaffold Write a fresh, doctor-clean loop contract into . @@ -28,11 +29,17 @@ verify Alias for doctor — check the contract's state. inspect Score an existing loop against the prime-directive checklist (emits a weak/strong verdict and a gap report). + metrics Derive false-completion-rate + repair-productivity from the loop's + real .loop/ evidence (RUNLOG, verify bundles, held-out gate, repair + records) and emit a JSON scorecard. With --baseline, write a + checked-in baseline scorecard — refused unless the run is gate-backed. arguments: A workspace root or its .loop/ directory. options: + --baseline (metrics only) write docs/metrics-baseline.json over a gate-backed + run; exits non-zero and writes nothing otherwise. -h, --help Show this help and exit. --version Show the version and exit. """ @@ -68,6 +75,34 @@ def _print_json(report: dict) -> int: return 0 if report.get("ok") else 1 +def _run_metrics(argv: list[str]) -> int: + """`metrics [--baseline] ` — parses its own flag, then delegates to + scripts/metrics.py (imported repo-relative, the QW8 editable-install path).""" + unknown = [a for a in argv if a.startswith("-") and a != "--baseline"] + if unknown: + print(f"metrics: unknown option: {unknown[0]}", file=sys.stderr) + print(_USAGE, file=sys.stderr) + return 2 + positional = [a for a in argv if not a.startswith("-")] + if not positional: + print("metrics: missing target argument", file=sys.stderr) + print(_USAGE, file=sys.stderr) + return 2 + target = Path(positional[0]) + if not target.exists(): + print( + f"metrics: target path does not exist: {target}\n" + f" pass an existing loop workspace or its .loop/ directory.", + file=sys.stderr, + ) + return 2 + scripts_dir = Path(__file__).resolve().parent.parent / "scripts" + sys.path.insert(0, str(scripts_dir)) + import metrics # type: ignore + + return metrics.run(argv) + + def main(argv: list[str] | None = None) -> int: argv = list(sys.argv[1:] if argv is None else argv) @@ -87,6 +122,11 @@ def main(argv: list[str] | None = None) -> int: print(_USAGE, file=sys.stderr) return 2 + # metrics carries its own optional --baseline flag, so it parses its own args + # before the generic single-target guards below. + if command == "metrics": + return _run_metrics(argv) + if not argv: print(f"{command}: missing target argument", file=sys.stderr) print(_USAGE, file=sys.stderr) diff --git a/pyproject.toml b/pyproject.toml index 9f16148..85966bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,14 @@ keywords = ["agent", "loop", "agentic", "verification", "harness", "orchestratio yaml = ["pyyaml>=6"] schemas = ["jsonschema>=4"] +# Console entry point. `loop.__main__:main` resolves the bundled scripts/ dir +# relative to its own __file__, so this is EDITABLE-INSTALL ONLY: `pip install -e .` +# keeps loop/ pointing at the repo (where scripts/ lives). A non-editable wheel +# does not ship scripts/, so `inspect`/`metrics` would not resolve — see the +# [tool.hatch.build.targets.wheel] note below. +[project.scripts] +loop = "loop.__main__:main" + [project.urls] Homepage = "https://github.com/SollanSystems/loop-engineer" Repository = "https://github.com/SollanSystems/loop-engineer" diff --git a/scripts/test_metrics_cli.py b/scripts/test_metrics_cli.py new file mode 100644 index 0000000..f217cfb --- /dev/null +++ b/scripts/test_metrics_cli.py @@ -0,0 +1,111 @@ +"""CLI contract for `python3 -m loop metrics` (Rider A) + the editable-install +console-script / repo-relative scripts resolution (Rider B / QW8). Runs the real +entry point as a subprocess so exit codes and STDOUT/STDERR match what a user sees. +""" + +from __future__ import annotations + +import json +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent + + +def _run(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "-m", "loop", *args], cwd=ROOT, text=True, capture_output=True + ) + + +# --- Rider A: metrics subcommand --------------------------------------------- + + +def test_metrics_emits_scorecard_for_flagship_example(): + result = _run("metrics", "examples/coverage-repair") + assert result.returncode == 0, result.stderr + scorecard = json.loads(result.stdout) + assert scorecard["schema"] == "loop-engineer/metrics@1" + assert scorecard["evidence_backed"] is True + assert scorecard["false_completion_rate"] == 0.0 + assert scorecard["repair_productivity"] == 1.0 + assert "provenance" in scorecard + + +def test_metrics_missing_target_prints_usage_and_exits_nonzero(): + result = _run("metrics") + assert result.returncode != 0 + assert "usage" in result.stderr.lower() + assert "Traceback" not in result.stderr + assert result.stdout.strip() == "" + + +def test_metrics_nonexistent_target_gives_distinct_error(tmp_path): + missing = tmp_path / "nope" + result = _run("metrics", str(missing)) + assert result.returncode != 0 + assert "does not exist" in result.stderr.lower() + assert str(missing) in result.stderr + assert "Traceback" not in result.stderr + assert result.stdout.strip() == "" + + +def test_metrics_help_is_listed(): + out = _run("--help").stdout + assert "metrics" in out + assert "--baseline" in out + + +def test_metrics_baseline_refuses_non_evidence_backed_and_writes_nothing(tmp_path): + # A bare loop with a success claim but no gate is not evidence_backed → refuse. + ws = tmp_path / "ws" + (ws / ".loop").mkdir(parents=True) + (ws / "RUNLOG.md").write_text( + "## Iteration 1 — t\n### Outcome\n`task_passed`\n", encoding="utf-8" + ) + baseline_path = ROOT / "docs" / "metrics-baseline.json" + before = baseline_path.read_bytes() if baseline_path.exists() else None + + result = _run("metrics", "--baseline", str(ws)) + + assert result.returncode != 0 + assert "refused" in result.stderr.lower() + # The committed baseline (if any) is untouched by a refused run. + after = baseline_path.read_bytes() if baseline_path.exists() else None + assert after == before + + +# --- Rider B: editable-install console script + repo-relative scripts (QW8) --- + + +def test_pyproject_declares_loop_console_script(): + text = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + assert re.search(r"(?m)^\[project\.scripts\]", text), "missing [project.scripts]" + assert re.search(r'(?m)^loop\s*=\s*"loop\.__main__:main"', text) + + +def test_entrypoint_resolves_repo_relative_scripts_dir(): + # The QW8 constraint: loop.__main__ resolves the bundled scripts/ dir from its + # own __file__, so an editable install runs `inspect`/`metrics` from any dir. + if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + import loop.__main__ as entry + + scripts_dir = Path(entry.__file__).resolve().parent.parent / "scripts" + assert (scripts_dir / "metrics.py").exists() + assert (scripts_dir / "inspect_loop.py").exists() + + +def test_metrics_runs_from_a_foreign_cwd(tmp_path): + # Proves scripts/ resolution is repo-relative, not cwd-relative: invoke from an + # unrelated cwd with an absolute target. + result = subprocess.run( + [sys.executable, "-m", "loop", "metrics", str(ROOT / "examples" / "coverage-repair")], + cwd=tmp_path, + text=True, + capture_output=True, + ) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout)["schema"] == "loop-engineer/metrics@1" From 4d2db18927b828e1e9193afb6342f50021290dc7 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Fri, 3 Jul 2026 15:13:38 -0400 Subject: [PATCH 06/15] docs(eval-suite,example): name canonical repair record; relocate example record to .loop/repair (ST1 AC1) --- examples/coverage-repair/.loop/manifest.yaml | 2 +- examples/coverage-repair/ADR.md | 2 +- examples/coverage-repair/README.md | 4 ++-- examples/coverage-repair/RUNLOG.md | 4 ++-- examples/coverage-repair/WORKFLOW.md | 2 +- reference/eval-suite.md | 1 + 6 files changed, 8 insertions(+), 7 deletions(-) diff --git a/examples/coverage-repair/.loop/manifest.yaml b/examples/coverage-repair/.loop/manifest.yaml index 8286fdd..a09205c 100644 --- a/examples/coverage-repair/.loop/manifest.yaml +++ b/examples/coverage-repair/.loop/manifest.yaml @@ -24,7 +24,7 @@ outputs: task_queue: TASKS.json current_state: .loop/state.json verification_bundle: .loop/artifacts/ - repair_actions: .loop/artifacts/repair-record.json + repair_actions: .loop/repair/iter-002.json terminal_state: .loop/terminal_state.json lessons_learned: .loop/memory/lessons.md diff --git a/examples/coverage-repair/ADR.md b/examples/coverage-repair/ADR.md index f6a842d..83dd413 100644 --- a/examples/coverage-repair/ADR.md +++ b/examples/coverage-repair/ADR.md @@ -74,7 +74,7 @@ still gives resumable, evidence-backed verification. We do **not** reach for mul coverage), each with its own `verify` gate; progress is the count of `done` tasks with non-null `evidence`. - **Patch-and-repair:** on a red gate, hand to `[[loop-repair]]` for one bounded hypothesis → - one change → re-verify, capped at N=2 (see `repair-record.json`). + one change → re-verify, capped at N=2 (see `.loop/repair/iter-002.json`). ## Hand-off diff --git a/examples/coverage-repair/README.md b/examples/coverage-repair/README.md index 569665d..5c3a9af 100644 --- a/examples/coverage-repair/README.md +++ b/examples/coverage-repair/README.md @@ -29,7 +29,7 @@ Succeeded`. | `TASKS.json` | `[[loop-contract]]` → updated by `[[loop-run]]` | The **machine-readable queue** — `T1` (validation) and `T2` (coverage), each with its `verify` command, `criterion_ref`, `attempts`, and `evidence`. Both end `done`. | | `RUNLOG.md` | `[[loop-run]]` (+ repair blocks from `[[loop-repair]]`) | The **append-only history** — two dated iterations: iteration 1 verify **FAIL** → repair → iteration 2 verify **PASS** → terminal. | | `.loop/state.json` | `[[loop-run]]` | The **live FSM cursor** — serialized after every transition; here at the terminal snapshot (`state: terminal`, `best_score: 0.83`, `terminal_state: "Succeeded"`). This is what makes the loop resumable across sessions. | -| `repair-record.json` | `[[loop-repair]]` | One **structured repair record** — `failure_mode`, `hypothesis`, `repair_action`, `verification_before`, `verification_after`, `remaining_delta` (+ `productive: true`). The verification delta proves the repair moved the score (`repair-productivity`). | +| `.loop/repair/iter-002.json` | `[[loop-repair]]` | One **structured repair record** at its canonical path (`loop-engineer/repair@1`) — `failure_mode`, `hypothesis`, `repair_action`, `verification_before`, `verification_after`, `remaining_delta` (+ `productive: true`). The verification delta proves the repair moved the score, and is the record `python3 -m loop metrics` reads for `repair-productivity`. | | `terminal_state.json` | `[[loop-run]]` | The **single end record** — `state == "Succeeded"`, `criteria_met` both true, `evidence` paths, and `false_completion: false`. No silent "completed." | > The `scripts/verify-*` gates and `EVALS/` rubrics referenced here are designed by `[[loop-evals]]` @@ -76,7 +76,7 @@ the scenario: ```bash # All JSON in this example parses: -for f in TASKS.json .loop/state.json repair-record.json terminal_state.json; do +for f in TASKS.json .loop/state.json .loop/repair/iter-002.json terminal_state.json; do uv run --with pyyaml python3 -c "import json,sys; json.load(open('$f')); print('ok:', '$f')" done diff --git a/examples/coverage-repair/RUNLOG.md b/examples/coverage-repair/RUNLOG.md index 1d313a7..8987251 100644 --- a/examples/coverage-repair/RUNLOG.md +++ b/examples/coverage-repair/RUNLOG.md @@ -39,7 +39,7 @@ Then advanced to `T2` (coverage) in the same iteration's verify step: exist. **No** test-harness assertion edits or fixture relaxations to manufacture the pass. - **attempt:** 1 of 2 - **measurable improvement:** pending re-verify (see Iteration 2) — full record in - `repair-record.json` + `.loop/repair/iter-002.json` --- @@ -61,7 +61,7 @@ Then advanced to `T2` (coverage) in the same iteration's verify step: - **repair action:** the two added production branches lifted coverage past the gate. - **attempt:** 1 of 2 (cap not reached — one productive pass) - **measurable improvement:** YES — `verification_after.score` 0.83 > `verification_before.score` - 0.74 → `productive: true`. See `repair-record.json`. + 0.74 → `productive: true`. See `.loop/repair/iter-002.json`. ### Terminal diff --git a/examples/coverage-repair/WORKFLOW.md b/examples/coverage-repair/WORKFLOW.md index 99f7ec7..f617aee 100644 --- a/examples/coverage-repair/WORKFLOW.md +++ b/examples/coverage-repair/WORKFLOW.md @@ -42,7 +42,7 @@ same `.loop/state.json` checkpoint; they never spawn a fresh untracked attempt. - **Max repair attempts per task:** `2` (default). - After exceeding the cap: replan / revert / approve / terminate — never silently retry. - Each repair attempt produces a structured repair record (see `[[loop-repair]]` and - `repair-record.json` in this example). + `.loop/repair/iter-002.json` in this example). - A repair that does not measurably improve the score is churn → replan. - Detected verifier-gaming → hard-terminate `FailedSafety` immediately. diff --git a/reference/eval-suite.md b/reference/eval-suite.md index ffdb567..427abb8 100644 --- a/reference/eval-suite.md +++ b/reference/eval-suite.md @@ -56,6 +56,7 @@ RP = (repair passes where verification_after > verification_before) ``` - "Improvement" is read straight off the structured repair-record schema [[loop-repair]] emits: `verification_before` vs `verification_after` on the same metric (a layer-1 pass count, a layer-2 rubric dimension, or a closed `remaining_delta`). A pass that leaves `remaining_delta` unchanged is churn. +- **The canonical repair record is RP's only input.** RP reads *the repair record* — `schemas/repair-record.schema.json` (`loop-engineer/repair@1`), on disk at `.loop/repair/.json`, the `verification_before`/`verification_after.score` pass [[loop-repair]] emits. The append-only **rollout / candidate ledger** (`schemas/rollout-record.schema.json`, `loop-engineer/rollout@1`, `scripts/rollout_ledger.py`) is a *separate* artifact that adjudicates rollout candidates; its `productive` is a rollout-productivity flywheel view, **not** the RP baseline. Neither is trusted verbatim: `scripts/metrics.py` recomputes `productive` (`recheck_productive`) from the record's own evidence and rejects any record whose stored flag disagrees, so RP is aggregated only over validated records. - **Target: high and trending up.** Low RP means the repair loop is thrashing — it is hitting its max-N cap (default N=2, per `WORKFLOW.md`) and burning budget without converging. Low RP is the data-driven trigger for the escalation ladder's "same failure mode repeats without measurable improvement → re-plan" rung (see `reference/safety-and-approvals.md`). - RP and the repair max-N cap are complementary: the cap bounds *how many* repairs run; RP measures whether those repairs were *worth* running. A loop with a healthy cap but RP≈0 is still broken. From 720639b75e18b4ae1c1344e7bb5efc18042e0e29 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Fri, 3 Jul 2026 15:14:51 -0400 Subject: [PATCH 07/15] feat(metrics): publish real FCR/RP baseline + README Metrics passage (ST1 AC5) --- README.md | 26 ++++++++++++++++++++-- docs/metrics-baseline.json | 44 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 docs/metrics-baseline.json diff --git a/README.md b/README.md index 96d672c..f08ea13 100644 --- a/README.md +++ b/README.md @@ -72,8 +72,8 @@ opinions. What this suite owns: - **7 typed terminal states** — a contract primitive, so no run ends in a silent "completed." -- **`false-completion-rate`** — measurable with the bundled held-out gate and anti-cheat scan (computed from real runs; no baseline ships yet). -- **`repair-productivity`** — the fraction of repair attempts that measurably move verification forward. +- **`false-completion-rate`** — measurable with the bundled held-out gate and anti-cheat scan; **0.0** on the shipped gate-backed example (see [Measured baseline](#measured-baseline)). +- **`repair-productivity`** — the fraction of repair attempts that measurably move verification forward; **1.0** on that example. - **Repo-native loop state** — survives compaction, crashes, and handoff. - **Deterministic-gate-before-rubric ordering** — model judges are advisory, not the first line of proof. @@ -83,6 +83,28 @@ Engineer's claim is the proof-of-done framing plus the typed termination and loop-health metrics on top. It composes with those tools; it does not replace their execution engines. +### Measured baseline + +The two metrics are **derived by a tool, not quoted from prose.** The checked-in +scorecard [`docs/metrics-baseline.json`](docs/metrics-baseline.json) is computed +by `python3 -m loop metrics` over the gate-backed `examples/coverage-repair` run — +its `false_completion:false` is backed by a real `holdout_gate.py` verdict, and +its `productive` flag is recomputed from the repair record's own score delta, not +trusted: + +| Metric | Baseline | Source | +|---|---:|---| +| `false-completion-rate` | **0.0** | RUNLOG success-claims × verify bundles, cross-checked against the held-out gate flag (both agree) | +| `repair-productivity` | **1.0** | one repair pass, `verification_after.score` 0.83 > `before` 0.74 (recomputed, agreed) | + +The number ships with a `provenance` block naming every input file, so a skeptic +can re-derive it. Reproduce (and refuse to publish over a non-gate-backed run): + +```bash +python3 -m loop metrics examples/coverage-repair # print the scorecard +python3 -m loop metrics --baseline examples/coverage-repair # rewrite docs/metrics-baseline.json +``` + --- ## Proof-of-done, not self-assertion diff --git a/docs/metrics-baseline.json b/docs/metrics-baseline.json new file mode 100644 index 0000000..c8bd01c --- /dev/null +++ b/docs/metrics-baseline.json @@ -0,0 +1,44 @@ +{ + "schema": "loop-engineer/metrics@1", + "loop": "examples/coverage-repair", + "false_completion_rate": 0.0, + "repair_productivity": 1.0, + "iterations_claiming_success": 2, + "false_completions": 0, + "repair_passes": 1, + "productive_repairs": 1, + "cost_per_success_usd": null, + "evidence_backed": true, + "provenance": { + "fcr_source": [ + ".loop/artifacts/verify-T1.json", + ".loop/artifacts/verify-T2-iter1.json", + ".loop/artifacts/verify-T2.json", + "RUNLOG.md" + ], + "rp_source": [ + ".loop/repair/iter-002.json" + ], + "rejected_records": [], + "false_completion_rate_holdout": 0.0, + "fcr_methods_agree": true, + "holdout_source": [ + ".loop/artifacts/holdout-verdict.json" + ], + "unmatched_verify": [] + }, + "baseline": { + "source_example": "examples/coverage-repair", + "commit": "4d2db18927b828e1e9193afb6342f50021290dc7", + "inputs": [ + "examples/coverage-repair/.loop/artifacts/holdout-verdict.json", + "examples/coverage-repair/.loop/artifacts/verify-T1.json", + "examples/coverage-repair/.loop/artifacts/verify-T2-iter1.json", + "examples/coverage-repair/.loop/artifacts/verify-T2.json", + "examples/coverage-repair/.loop/repair/iter-002.json", + "examples/coverage-repair/RUNLOG.md", + "examples/coverage-repair/TASKS.json", + "examples/coverage-repair/terminal_state.json" + ] + } +} From 76e02265b28a4f41a6a07b14431b66a30f9b93f5 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Fri, 3 Jul 2026 15:50:47 -0400 Subject: [PATCH 08/15] fix(metrics): close ST1 evidence/FCR/RP gaming gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial review of the ST1 metrics slice found the honesty gates were satisfiable without real evidence. Harden compute_metrics/build_baseline: - evidence_backed no longer credits a RUNLOG prose mention or a bare TASKS verify declaration. Only (a) a structurally-valid held-out verdict artifact (per-check visible/holdout arrays + flags re-derivable from them — a 4-field stub is rejected) or (b) a NON-COMMENT gate line in a verify-* script whose gate script file exists. Records the verdict sha256 in provenance and states it is evidence, not proof (tamper detection is the anti-cheat layer's job). - FCR cross-join is fail-closed: a success claim is clean only if every attached verify bundle is green, EXCEPT a red bundle whose own task later reached green (an honestly-repaired intermediate). An unrelated green can no longer launder a claimed task's still-red gate. - Unrecognized ### Outcome tokens are surfaced under provenance.unrecognized_ outcomes instead of silently escaping the FCR denominator. - RP records are anchored: before/after scores must be corroborated by real verify-bundle scores (fabricated deltas -> rejected); no bundle to anchor -> flagged unanchored. - --baseline additionally refuses when the two FCR methods disagree, when no iteration claims success (vacuous 0/0), or when a counted RP record is unanchored; each refusal names the failed precondition. Co-Authored-By: Claude Fable 5 --- scripts/metrics.py | 269 ++++++++++++++++++++++++++++++++++------ scripts/test_metrics.py | 235 +++++++++++++++++++++++++++++++++-- 2 files changed, 456 insertions(+), 48 deletions(-) diff --git a/scripts/metrics.py b/scripts/metrics.py index 8805544..93b5574 100644 --- a/scripts/metrics.py +++ b/scripts/metrics.py @@ -17,12 +17,32 @@ * **FCR is derived two ways and disagreement is surfaced.** (a) the RUNLOG success-claim × verify-bundle cross-join (the deterministic anchor, per §3), and (b) the aggregated held-out-gate `false_completion` flag. An unmatched - success-claim counts as a false completion (fail-closed, §8). + success-claim counts as a false completion (fail-closed, §8). A success claim + is clean only if EVERY verify bundle attached to its iteration is green, or a + red one's own task later reached green (an honestly-repaired intermediate) — + an unrelated green bundle can never launder the claimed task's red gate. + +The `### Outcome` token contract: a claim counts toward the FCR denominator only +when its outcome token is a recognized SUCCESS token (`task_passed`, `terminal`, +`succeeded`, `advanced`); `repair_triggered` / `task_failed` and peers are honest +reds. Any token in neither set is surfaced under `provenance.unrecognized_outcomes` +so a synonym (`shipped`, `done`, …) is visible rather than silently escaping the +denominator — it never widens the recognized-success set on its own. + +A committed held-out verdict is validated structurally (it must carry the per-check +`visible`/`holdout` arrays and internally-consistent flags a real `holdout_gate` +run emits, not a hand-set 4-field stub) and its sha256 is recorded in provenance. +That committed verdict is *evidence, not proof*: it demonstrates a gate ran, but +tamper detection of the artifact itself belongs to the anti-cheat layer +(`anticheat_scan.py`) — this command does not, and does not claim to, make the +verdict tamper-proof. `--baseline` writes a checked-in scorecard, but only over a genuinely gate-backed -run: it refuses (non-zero, writes nothing) if the run is not `evidence_backed` or -if any record was rejected. Baselining a self-asserted run would itself be a false -completion of ST1. +run: it refuses (non-zero, writes nothing) if the run is not `evidence_backed`, if +any record was rejected, if the two FCR methods disagree, if no iteration claims +success (a vacuous 0/0 is not a publishable 0.0), or if any counted repair record +is unanchored. Baselining a self-asserted run would itself be a false completion of +ST1. Pure stdlib, offline, deterministic: the same loop dir yields a byte-identical scorecard. @@ -35,6 +55,7 @@ from __future__ import annotations +import hashlib import json import subprocess import sys @@ -55,9 +76,21 @@ # repair_triggered / task_failed outcome is an honest red, NOT a success claim. _SUCCESS_OUTCOME_TOKENS = ("task_passed", "terminal", "succeeded", "advanced") +# Recognized honest-red outcome tokens: not a success claim, but a known outcome +# (so they are not surfaced as an "unrecognized" synonym). Any outcome token in +# neither set is surfaced under provenance.unrecognized_outcomes. +_HONEST_RED_OUTCOME_TOKENS = ( + "repair_triggered", "task_failed", "replan", "reverted", "revert", + "blocked", "terminated", "aborted", "failed", +) +_KNOWN_OUTCOME_TOKENS = frozenset(_SUCCESS_OUTCOME_TOKENS) | frozenset(_HONEST_RED_OUTCOME_TOKENS) + # Gate tokens (mirrors inspect_loop._GATE_TOKENS): a real held-out / anti-cheat # gate invocation writes one of these into the execution trail. _GATE_TOKENS = ("holdout_gate", "anticheat_scan", "anti_cheat") +# Gate script filenames (mirrors inspect_loop._GATE_SCRIPTS): the file a real +# invocation runs. evidence_backed via a verify script requires one to exist. +_GATE_SCRIPTS = ("holdout_gate.py", "anticheat_scan.py", "anti_cheat.py") _ITER_HEADER_RE = re.compile(r"(?m)^##\s+Iteration\s+(\S+)") # "outcome" declaration followed (within a little markup/whitespace) by its token. @@ -171,9 +204,12 @@ def _runlog_blocks(runlog_text: str) -> list[tuple[str, str]]: return blocks +def _block_outcome_tokens(block_text: str) -> list[str]: + return [t.lower() for t in _OUTCOME_RE.findall(block_text)] + + def _block_claims_success(block_text: str) -> bool: - tokens = [t.lower() for t in _OUTCOME_RE.findall(block_text)] - return any(t in _SUCCESS_OUTCOME_TOKENS for t in tokens) + return any(t in _SUCCESS_OUTCOME_TOKENS for t in _block_outcome_tokens(block_text)) def _load_verify_bundles(loop_dir: Path) -> list[dict]: @@ -187,12 +223,15 @@ def _load_verify_bundles(loop_dir: Path) -> list[dict]: outcome = str(data.get("outcome", "")).upper() green = outcome == "PASS" or data.get("passed") is True it = data.get("iteration_id", data.get("iteration")) + task = data.get("task") bundles.append( { "path": path, "name": path.name, "green": green, "iter": _norm_iter(it) if it is not None else None, + "task": str(task) if task is not None else None, + "score": _num(data.get("score")), } ) return bundles @@ -216,34 +255,97 @@ def _assign_bundles_to_iters(bundles: list[dict], blocks: list[tuple[str, str]]) return by_iter, sorted(unmatched) +def _valid_check_list(checks) -> bool: + """A non-empty list of ``{"id": ..., "passed": bool}`` per-check results.""" + return ( + isinstance(checks, list) + and bool(checks) + and all(isinstance(c, dict) and "id" in c and isinstance(c.get("passed"), bool) for c in checks) + ) + + +def _is_valid_gate_verdict(data: dict) -> bool: + """Structurally validate a held-out verdict against ``holdout_gate.decide``'s + output shape. A real run carries the per-check ``visible``/``holdout`` arrays + plus flags RE-DERIVABLE from them; a hand-typed + ``{verdict, passed_visible, passed_holdout, false_completion}`` stub carries no + check evidence and is rejected — a self-asserted flag is not a gate run.""" + if not isinstance(data.get("verdict"), str): + return False + for key in ("passed_visible", "passed_holdout", "false_completion"): + if not isinstance(data.get(key), bool): + return False + visible, holdout = data.get("visible"), data.get("holdout") + if not _valid_check_list(visible) or not _valid_check_list(holdout): + return False + passed_visible = all(c["passed"] for c in visible) + passed_holdout = all(c["passed"] for c in holdout) + if data["passed_visible"] != passed_visible or data["passed_holdout"] != passed_holdout: + return False + return data["false_completion"] == (passed_visible and not passed_holdout) + + def _load_gate_verdicts(loop_dir: Path) -> list[dict]: - """Held-out / anti-cheat gate verdict artifacts (holdout_gate.decide output).""" + """Held-out / anti-cheat gate verdict artifacts (holdout_gate.decide output). + + Only structurally-valid verdicts are returned; a fabricated 4-field stub is + not counted as gate evidence. + """ verdicts: list[dict] = [] for path in sorted(loop_dir.rglob("*.json")): if "archive" in path.parts: continue data = _read_json_object(path) - if "false_completion" in data and ("verdict" in data or "passed_visible" in data): + if "false_completion" in data and _is_valid_gate_verdict(data): verdicts.append({"path": path, "data": data}) return verdicts +def _sha256(path: Path) -> str: + try: + return hashlib.sha256(path.read_bytes()).hexdigest() + except OSError: + return "" + + +def _verify_scripts(workspace: Path) -> list[Path]: + scripts = workspace / "scripts" + if not scripts.is_dir(): + return [] + return sorted(p for p in scripts.glob("verify-*") if p.is_file()) + + +def _gate_script_present(workspace: Path) -> bool: + """A gate script the verify surface can actually invoke exists — either bundled + with the loop or in the loop-engineer toolkit it composes with.""" + for base in (workspace / "scripts", _REPO_ROOT / "scripts"): + if any((base / name).exists() for name in _GATE_SCRIPTS): + return True + return False + + def _gate_invoked(paths: LoopPaths, gate_verdicts: list[dict]) -> bool: - """Is a real gate invocation detectable — a recorded verdict artifact, or a - gate token in the RUNLOG / verify scripts / task verify commands? Mirrors the - inspector's invocation-evidence rule (HI4).""" + """Is a real gate invocation detectable? Mirrors the inspector's + invocation-evidence rule (HI4), NOT looser prose matching: + + (a) a recorded, structurally-valid gate VERDICT artifact, or + (b) a NON-COMMENT gate-token line in a verify-* script whose gate script + file actually exists. + + A bare TASKS.json verify *declaration* or a RUNLOG prose mention is NOT an + invocation (a ``# TODO: call holdout_gate.py`` comment earns nothing).""" if gate_verdicts: return True - haystacks = [_read_text(paths.runlog)] - scripts_dir = paths.workspace / "scripts" - for name in ("verify-fast", "verify-fast.sh", "verify-full", "verify-full.sh", - "verify-safety", "verify-safety.sh"): - haystacks.append(_read_text(scripts_dir / name)) - tasks = _read_json_object(paths.tasks).get("tasks") - if isinstance(tasks, list): - haystacks.extend(str(row.get("verify", "")) for row in tasks if isinstance(row, dict)) - blob = "\n".join(haystacks).lower() - return any(tok in blob for tok in _GATE_TOKENS) + if not _gate_script_present(paths.workspace): + return False + for script in _verify_scripts(paths.workspace): + for line in _read_text(script).splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + if any(tok in stripped for tok in _GATE_TOKENS): + return True + return False def _load_receipt_costs(loop_dir: Path) -> tuple[float | None, int]: @@ -268,6 +370,37 @@ def _load_receipt_costs(loop_dir: Path) -> tuple[float | None, int]: return total, count +def _anchor_repair(record: dict, bundle_scores: set[float]) -> dict: + """Cross-check a repair record's self-reported before/after scores against the + deterministic verify bundles (§4.3, RP anchoring). + + Returns ``status`` one of ``anchored`` (both scores corroborated), + ``unanchored`` (no verify bundle carries a score to anchor against), or + ``rejected`` (a score is present but no verify bundle corroborates it — a + fabricated delta). ``recheck_productive`` runs first, so by here a repair + record's before/after scores are already numeric. + """ + before = record.get("verification_before") + after = record.get("verification_after") + before = _num(before.get("score")) if isinstance(before, dict) else None + after = _num(after.get("score")) if isinstance(after, dict) else None + if before is None and after is None: + return {"status": "unanchored", "reason": "no before/after score to anchor"} + if not bundle_scores: + return {"status": "unanchored", "reason": "no verify bundle scores to anchor against"} + missing = [ + label + for label, value in (("verification_before", before), ("verification_after", after)) + if value is not None and value not in bundle_scores + ] + if missing: + return { + "status": "rejected", + "reason": f"self-reported {', '.join(missing)}.score not corroborated by any verify bundle", + } + return {"status": "anchored", "reason": "ok"} + + def _rel(path: Path, base: Path) -> str: try: return str(path.relative_to(base)) @@ -306,14 +439,30 @@ def compute_metrics(loop_dir: str | Path, loop_label: str | None = None) -> dict if b not in verify_by_iter[tid]: verify_by_iter[tid].append(b) - # FCR-A: a success claim with no green deterministic verify is a false - # completion (unmatched claims fail closed, §8). - false_completions = sum( - 1 for iid in claim_iters if not any(b["green"] for b in verify_by_iter.get(iid, [])) - ) + # FCR-A: a success claim is clean only if every verify bundle attached to its + # iteration is green — with one honest exception: a red bundle whose OWN task + # later reached green is a repaired intermediate, not a false completion (the + # flagship's verify-T2-iter1 → verify-T2). An UNRELATED green bundle can never + # launder a claimed task's still-red gate. Unmatched claims fail closed (§8). + green_tasks = {b["task"] for b in bundles if b["green"] and b["task"]} + + def _claim_is_clean(iid: str) -> bool: + attached = verify_by_iter.get(iid, []) + if not attached or not any(b["green"] for b in attached): + return False + for b in attached: + if not b["green"] and (not b["task"] or b["task"] not in green_tasks): + return False + return True + + false_completions = sum(1 for iid in claim_iters if not _claim_is_clean(iid)) n_claims = len(claim_iters) fcr_a = (false_completions / n_claims) if n_claims else 0.0 + unrecognized_outcomes = sorted( + {t for _iid, text in blocks for t in _block_outcome_tokens(text) if t not in _KNOWN_OUTCOME_TOKENS} + ) + # FCR-B: aggregated held-out-gate false_completion flag. gate_verdicts = _load_gate_verdicts(loop_dir_path) fc_flagged = sum(1 for v in gate_verdicts if v["data"].get("false_completion") is True) @@ -322,22 +471,34 @@ def compute_metrics(loop_dir: str | Path, loop_label: str | None = None) -> dict evidence_backed = _gate_invoked(paths, gate_verdicts) - # RP: over recomputed-and-agreed repair records only. + # RP: over recomputed-and-agreed repair records only, whose before/after + # scores are anchored to the deterministic verify bundles (§4.3). A record + # whose stored productive lies (recheck) OR whose scores are not corroborated + # by any verify bundle (anchor) is rejected; a record with no bundle to anchor + # against is counted but flagged unanchored (a baseline refuses over those). + bundle_scores = {b["score"] for b in bundles if b["score"] is not None} validated = 0 productive = 0 rejected: list[dict] = [] + unanchored: list[str] = [] rp_source: list[str] = [] for path in sorted((loop_dir_path / "repair").glob("*.json")): record = _read_json_object(path) - verdict = recheck_productive(record) rel = _rel(path, workspace) - if verdict["valid"]: - validated += 1 - rp_source.append(rel) - if verdict["expected"]: - productive += 1 - else: + verdict = recheck_productive(record) + if not verdict["valid"]: rejected.append({"record": rel, "reason": verdict["reason"]}) + continue + anchor = _anchor_repair(record, bundle_scores) + if anchor["status"] == "rejected": + rejected.append({"record": rel, "reason": anchor["reason"]}) + continue + if anchor["status"] == "unanchored": + unanchored.append(rel) + validated += 1 + rp_source.append(rel) + if verdict["expected"]: + productive += 1 repair_productivity = (productive / validated) if validated else None # Cost-per-success (layer 7): total receipt cost over true completions. @@ -347,6 +508,10 @@ def compute_metrics(loop_dir: str | Path, loop_label: str | None = None) -> dict fcr_source = sorted({_rel(paths.runlog, workspace)} | {_rel(b["path"], workspace) for b in bundles}) holdout_source = sorted(_rel(v["path"], workspace) for v in gate_verdicts) + holdout_verdicts = sorted( + ({"source": _rel(v["path"], workspace), "sha256": _sha256(v["path"])} for v in gate_verdicts), + key=lambda e: e["source"], + ) rejected.sort(key=lambda r: r["record"]) return { @@ -364,9 +529,12 @@ def compute_metrics(loop_dir: str | Path, loop_label: str | None = None) -> dict "fcr_source": fcr_source, "rp_source": sorted(rp_source), "rejected_records": rejected, + "unanchored_records": sorted(unanchored), + "unrecognized_outcomes": unrecognized_outcomes, "false_completion_rate_holdout": fcr_b, "fcr_methods_agree": fcr_methods_agree, "holdout_source": holdout_source, + "holdout_verdicts": holdout_verdicts, "unmatched_verify": unmatched_verify, }, } @@ -404,20 +572,43 @@ def _git_commit() -> str | None: def build_baseline(loop_dir: str | Path, loop_label: str | None = None) -> tuple[bool, dict, list[str]]: """Compute the scorecard and check the §4.5 baseline preconditions. - Returns ``(ok, scorecard, refusal_reasons)``. ``ok`` is False iff the run is - not evidence-backed or contains rejected records. + Returns ``(ok, scorecard, refusal_reasons)``. ``ok`` is False when the run is + not evidence-backed, contains a rejected record, has disagreeing FCR methods, + claims no success (vacuous 0/0), or counts an unanchored repair record. Each + refusal names the precondition that failed. """ scorecard = compute_metrics(loop_dir, loop_label) + prov = scorecard["provenance"] reasons: list[str] = [] if not scorecard["evidence_backed"]: reasons.append( "run is not evidence_backed — no held-out / anti-cheat gate invocation " - "detectable in the verify trail or RUNLOG" + "detectable (a structurally-valid verdict artifact, or a non-comment gate " + "line in a verify-* script whose gate script exists)" ) - rejected = scorecard["provenance"]["rejected_records"] + rejected = prov["rejected_records"] if rejected: reasons.append( - f"{len(rejected)} rejected record(s) present (productive disagrees with its own evidence)" + f"{len(rejected)} rejected record(s) present (productive disagrees with its own " + "evidence, or before/after scores are not corroborated by any verify bundle)" + ) + if prov["fcr_methods_agree"] is False: + reasons.append( + "fcr_methods_agree is False — the deterministic cross-join " + f"(FCR {scorecard['false_completion_rate']}) and the held-out-gate flag " + f"(FCR {prov['false_completion_rate_holdout']}) disagree; a baseline may not " + "launder an inconsistent run into a clean number" + ) + if scorecard["iterations_claiming_success"] == 0: + reasons.append( + "iterations_claiming_success == 0 — a vacuous 0/0 run yields no publishable " + "false-completion-rate (an FCR 0.0 over no success claims is not a baseline)" + ) + if prov["unanchored_records"]: + reasons.append( + f"{len(prov['unanchored_records'])} counted repair record(s) are unanchored — no " + "verify bundle score corroborates their before/after; a baseline must anchor RP " + "to deterministic evidence" ) return (not reasons), scorecard, reasons diff --git a/scripts/test_metrics.py b/scripts/test_metrics.py index 09d2127..818573d 100644 --- a/scripts/test_metrics.py +++ b/scripts/test_metrics.py @@ -63,6 +63,22 @@ def _repair_record(before: float, after: float, productive: bool) -> dict: } +def _holdout_verdict(*, false_completion: bool = False) -> dict: + """A structurally-valid held-out verdict in ``holdout_gate.decide`` shape — + per-check visible/holdout arrays plus flags re-derivable from them.""" + holdout_passed = not false_completion + return { + "verdict": "FailedUnverifiable" if false_completion else "Succeeded", + "reason": "fixture", + "passed_visible": True, + "passed_holdout": holdout_passed, + "false_completion": false_completion, + "visible": [{"id": "unit", "passed": True, "returncode": 0}], + "holdout": [{"id": "probe", "passed": holdout_passed, + "returncode": 0 if holdout_passed else 1}], + } + + # --- recheck_productive (AC2) ------------------------------------------------- @@ -183,8 +199,7 @@ def test_evidence_backed_true_when_holdout_verdict_present(tmp_path): tmp_path, runlog=_RUNLOG_ONE_CLAIM, verify={"verify-A.json": {"task": "A", "outcome": "PASS", "score": 1.0}}, - gate_verdict={"verdict": "Succeeded", "passed_visible": True, - "passed_holdout": True, "false_completion": False}, + gate_verdict=_holdout_verdict(), ) sc = metrics.compute_metrics(ws) assert sc["evidence_backed"] is True @@ -239,8 +254,7 @@ def test_scorecard_is_byte_identical_across_runs(tmp_path): runlog=_RUNLOG_ONE_CLAIM, verify={"verify-A.json": {"task": "A", "outcome": "PASS", "score": 1.0}}, repair={"iter-001.json": prod}, - gate_verdict={"verdict": "Succeeded", "passed_visible": True, - "passed_holdout": True, "false_completion": False}, + gate_verdict=_holdout_verdict(), receipts=[{"schema": "loop-engineer/receipt@1", "iteration_id": 1, "role": "write", "model": "opus", "outcome": "ok", "cost_usd": 0.4}], ) @@ -282,8 +296,7 @@ def test_baseline_refuses_when_a_record_is_rejected(tmp_path): runlog=_RUNLOG_ONE_CLAIM, verify={"verify-A.json": {"task": "A", "outcome": "PASS", "score": 1.0}}, repair={"iter-001.json": liar}, - gate_verdict={"verdict": "Succeeded", "passed_visible": True, - "passed_holdout": True, "false_completion": False}, + gate_verdict=_holdout_verdict(), ) ok, _sc, reasons = metrics.build_baseline(ws, "ws") assert ok is False @@ -295,10 +308,12 @@ def test_baseline_writes_scorecard_over_gate_backed_run(tmp_path): ws = _make_loop( tmp_path, runlog=_RUNLOG_ONE_CLAIM, - verify={"verify-A.json": {"task": "A", "outcome": "PASS", "score": 1.0}}, + # The claim's own green evidence (0.8) plus a red pre-repair bundle (0.5) + # anchor the repair record's before/after scores to real verify evidence. + verify={"verify-A.json": {"task": "A", "outcome": "PASS", "score": 0.8}, + "verify-before.json": {"task": "A", "outcome": "FAIL", "score": 0.5}}, repair={"iter-001.json": prod}, - gate_verdict={"verdict": "Succeeded", "passed_visible": True, - "passed_holdout": True, "false_completion": False}, + gate_verdict=_holdout_verdict(), ) out = tmp_path / "docs" / "metrics-baseline.json" rc = metrics.write_baseline(ws, out, loop_label="ws") @@ -318,4 +333,206 @@ def test_metrics_on_flagship_example_is_clean_and_evidence_backed(): assert sc["repair_productivity"] == 1.0 assert sc["evidence_backed"] is True assert sc["provenance"]["rejected_records"] == [] + assert sc["provenance"]["unanchored_records"] == [] + assert sc["provenance"]["unrecognized_outcomes"] == [] assert sc["provenance"]["fcr_methods_agree"] is True + + +# --- evidence_backed honesty (P1/P2): comment/prose/stub are NOT invocations --- + + +def test_runlog_prose_mention_of_gate_is_not_evidence(tmp_path): + # Exploit A: a bare RUNLOG mention (even a negation) is not a gate invocation. + ws = _make_loop( + tmp_path, + runlog="# RUNLOG\n\n## Iteration 1 — t\n\n" + "- note: we never ran holdout_gate.py here; TODO wire it up.\n", + ) + assert metrics.compute_metrics(ws)["evidence_backed"] is False + + +def test_hand_authored_verdict_without_check_arrays_is_not_evidence(tmp_path): + # Exploit A2: a 4-field stub carries no per-check visible/holdout results a + # real holdout_gate.decide emits — it is not a gate run. + ws = _make_loop( + tmp_path, + runlog=_RUNLOG_ONE_CLAIM, + verify={"verify-A.json": {"task": "A", "outcome": "PASS", "score": 1.0}}, + gate_verdict={"verdict": "Succeeded", "passed_visible": True, + "passed_holdout": True, "false_completion": False}, + ) + sc = metrics.compute_metrics(ws) + assert sc["evidence_backed"] is False + assert sc["provenance"]["holdout_verdicts"] == [] + ok, _sc, _reasons = metrics.build_baseline(ws, "ws") + assert ok is False + + +def test_valid_verdict_is_evidence_and_sha256_recorded(tmp_path): + ws = _make_loop( + tmp_path, + runlog=_RUNLOG_ONE_CLAIM, + verify={"verify-A.json": {"task": "A", "outcome": "PASS", "score": 1.0}}, + gate_verdict=_holdout_verdict(), + ) + prov = metrics.compute_metrics(ws)["provenance"] + assert len(prov["holdout_verdicts"]) == 1 + assert len(prov["holdout_verdicts"][0]["sha256"]) == 64 + + +def test_comment_only_gate_line_in_verify_script_is_not_evidence(tmp_path): + ws = _make_loop( + tmp_path, + runlog=_RUNLOG_ONE_CLAIM, + verify={"verify-A.json": {"task": "A", "outcome": "PASS", "score": 1.0}}, + ) + (ws / "scripts").mkdir() + (ws / "scripts" / "verify-full").write_text( + "#!/bin/sh\n# TODO: someday call scripts/holdout_gate.py here\necho PASS\n", + encoding="utf-8", + ) + assert metrics.compute_metrics(ws)["evidence_backed"] is False + + +def test_executed_gate_line_in_verify_script_is_evidence(tmp_path): + ws = _make_loop( + tmp_path, + runlog=_RUNLOG_ONE_CLAIM, + verify={"verify-A.json": {"task": "A", "outcome": "PASS", "score": 1.0}}, + ) + (ws / "scripts").mkdir() + (ws / "scripts" / "verify-full").write_text( + "#!/bin/sh\npython3 scripts/holdout_gate.py manifest.json\n", encoding="utf-8" + ) + assert metrics.compute_metrics(ws)["evidence_backed"] is True + + +# --- FCR cross-join laundering (P2) ------------------------------------------- + + +def test_unrelated_green_bundle_does_not_launder_a_red_claimed_task(tmp_path): + # Exploit F: the claimed task's gate is RED; an unrelated green must not clear it. + ws = _make_loop( + tmp_path, + runlog="# RUNLOG\n\n## Iteration 3 — t\n\n- **outcome:** advanced\n" + "- refs: .loop/artifacts/verify-T3.json and .loop/artifacts/verify-unrelated.json\n", + verify={ + "verify-T3.json": {"task": "T3", "outcome": "FAIL", "score": 0.1}, + "verify-unrelated.json": {"task": "T1", "outcome": "PASS", "score": 1.0}, + }, + ) + sc = metrics.compute_metrics(ws) + assert sc["false_completions"] == 1 + assert sc["false_completion_rate"] == 1.0 + + +def test_honest_intermediate_red_later_repaired_is_not_a_false_completion(tmp_path): + # The flagship shape: a red bundle whose OWN task later reaches green is an + # honestly-repaired intermediate, not a laundered false completion. + ws = _make_loop( + tmp_path, + runlog="# RUNLOG\n\n## Iteration 1 — t\n\n- **outcome:** advanced\n" + "- refs: .loop/artifacts/verify-T2-iter1.json and .loop/artifacts/verify-T2.json\n", + verify={ + "verify-T2-iter1.json": {"task": "T2", "outcome": "FAIL", "score": 0.74}, + "verify-T2.json": {"task": "T2", "outcome": "PASS", "score": 0.83}, + }, + ) + assert metrics.compute_metrics(ws)["false_completions"] == 0 + + +# --- success-token allow-list surfacing + vacuous refusal (P2) ----------------- + + +def test_unrecognized_outcome_token_is_surfaced_and_not_a_claim(tmp_path): + # Exploit B: a synonym escapes the denominator but is surfaced, not silent. + ws = _make_loop( + tmp_path, + runlog="# RUNLOG\n\n## Iteration 1 — t\n\n- **outcome:** shipped\n" + "- refs: .loop/artifacts/verify-red.json\n", + verify={"verify-red.json": {"iteration_id": 1, "outcome": "FAIL", "score": 0.0}}, + ) + sc = metrics.compute_metrics(ws) + assert "shipped" in sc["provenance"]["unrecognized_outcomes"] + assert sc["iterations_claiming_success"] == 0 + + +def test_baseline_refuses_vacuous_zero_claim_run(tmp_path): + ws = _make_loop( + tmp_path, + runlog="# RUNLOG\n\n## Iteration 1 — t\n\n- **outcome:** shipped\n", + gate_verdict=_holdout_verdict(), + ) + ok, _sc, reasons = metrics.build_baseline(ws, "ws") + assert ok is False + assert any("claim" in r.lower() for r in reasons) + + +# --- two-way FCR disagreement refusal (P1) ------------------------------------ + + +def test_baseline_refuses_when_fcr_methods_disagree(tmp_path): + # A clean deterministic cross-join (fcr_a=0) over a run whose held-out gate + # flags a false completion (fcr_b=1) must not publish a laundered 0.0. + ws = _make_loop( + tmp_path, + runlog=_RUNLOG_ONE_CLAIM, + verify={"verify-A.json": {"task": "A", "outcome": "PASS", "score": 1.0}}, + gate_verdict=_holdout_verdict(false_completion=True), + ) + sc = metrics.compute_metrics(ws) + assert sc["provenance"]["fcr_methods_agree"] is False + ok, _sc, reasons = metrics.build_baseline(ws, "ws") + assert ok is False + assert any("agree" in r.lower() for r in reasons) + + +# --- RP anchoring against verify bundles (P2) --------------------------------- + + +def test_rp_record_with_fabricated_scores_is_rejected_when_bundles_exist(tmp_path): + # Exploit C: before/after unrelated to any real verify bundle score → rejected. + ws = _make_loop( + tmp_path, + runlog=_RUNLOG_ONE_CLAIM, + verify={"verify-A.json": {"task": "A", "outcome": "PASS", "score": 0.9}}, + repair={"iter-001.json": _repair_record(0.0, 1.0, True)}, + ) + sc = metrics.compute_metrics(ws) + rejected = sc["provenance"]["rejected_records"] + assert any(r["record"].endswith("iter-001.json") for r in rejected) + assert sc["repair_productivity"] is None + + +def test_rp_record_anchors_cleanly_against_matching_bundles(tmp_path): + ws = _make_loop( + tmp_path, + runlog=_RUNLOG_ONE_CLAIM, + verify={ + "verify-before.json": {"task": "T2", "outcome": "FAIL", "score": 0.74}, + "verify-after.json": {"task": "T2", "outcome": "PASS", "score": 0.83}, + }, + repair={"iter-001.json": _repair_record(0.74, 0.83, True)}, + ) + sc = metrics.compute_metrics(ws) + assert sc["provenance"]["rejected_records"] == [] + assert sc["provenance"]["unanchored_records"] == [] + assert sc["repair_productivity"] == 1.0 + + +def test_baseline_refuses_when_a_counted_rp_record_is_unanchored(tmp_path): + # A green claim-backing bundle with no numeric score leaves nothing to anchor + # the repair record's before/after against → unanchored → baseline refuses. + ws = _make_loop( + tmp_path, + runlog=_RUNLOG_ONE_CLAIM, + verify={"verify-A.json": {"task": "A", "outcome": "PASS"}}, + repair={"iter-001.json": _repair_record(0.5, 0.8, True)}, + gate_verdict=_holdout_verdict(), + ) + sc = metrics.compute_metrics(ws) + unanchored = sc["provenance"]["unanchored_records"] + assert len(unanchored) == 1 and unanchored[0].endswith("iter-001.json") + ok, _sc, reasons = metrics.build_baseline(ws, "ws") + assert ok is False + assert any("anchor" in r.lower() for r in reasons) From a93bf832a6133c82d476e1b5be595adc39b94518 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Fri, 3 Jul 2026 15:51:13 -0400 Subject: [PATCH 09/15] fix(contract): report validated record schemas in schemas_checked doctor validated repair records and receipt/rollout ledgers when present but reported only the 4 core schema ids, under-counting its own coverage. Return the record schema keys actually checked and append their schema ids (repair, rollout, receipt) to schemas_checked, in deterministic order. A loop with no record files is unchanged (still the 4 core ids). Co-Authored-By: Claude Fable 5 --- loop/contract.py | 26 +++++++++++++++++++++++--- scripts/test_contract_records.py | 25 +++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/loop/contract.py b/loop/contract.py index 057a32c..645a45b 100644 --- a/loop/contract.py +++ b/loop/contract.py @@ -300,6 +300,14 @@ def _jsonschema_validate(data: dict[str, Any], name: str, path: Path, issues: li "receipt": "receipt.schema.json", } +# The $id each record schema publishes, reported under schemas_checked when the +# corresponding record files were present and validated (deterministic order). +_RECORD_SCHEMA_IDS = ( + ("repair", "loop-engineer/repair@1"), + ("rollout", "loop-engineer/rollout@1"), + ("receipt", "loop-engineer/receipt@1"), +) + def _load_schema_file(filename: str) -> dict[str, Any]: return json.loads((_schemas_dir() / filename).read_text(encoding="utf-8")) @@ -351,19 +359,27 @@ def _validate_jsonl(path: Path, schema_key: str, mode: str, issues: list[dict]) _validate_record(data, schema_key, path, mode, issues) -def _validate_optional_records(paths: LoopPaths, mode: str, issues: list[dict]) -> None: +def _validate_optional_records(paths: LoopPaths, mode: str, issues: list[dict]) -> set[str]: + """Validate record files that are present; return the set of record schema + keys actually checked (``repair``/``rollout``/``receipt``) so ``doctor`` can + report them under ``schemas_checked`` instead of under-counting its coverage.""" + checked: set[str] = set() repair_dir = paths.loop_dir / "repair" if repair_dir.is_dir(): for record_path in sorted(repair_dir.glob("*.json")): data = _read_json(record_path, issues) if data is not None: _validate_record(data, "repair", record_path, mode, issues) + checked.add("repair") for ledger_path in sorted(paths.loop_dir.glob("*.jsonl")): _validate_jsonl(ledger_path, "rollout", mode, issues) + checked.add("rollout") receipts_dir = paths.loop_dir / "receipts" if receipts_dir.is_dir(): for receipt_path in sorted(receipts_dir.glob("*.jsonl")): _validate_jsonl(receipt_path, "receipt", mode, issues) + checked.add("receipt") + return checked def validate_contract(target: str | Path) -> dict[str, Any]: @@ -404,13 +420,17 @@ def validate_contract(target: str | Path) -> dict[str, Any]: if not paths.runlog.exists(): issues.append(ContractIssue("missing_file", "missing RUNLOG.md", paths.runlog)) _check_stub_verify_scripts(paths, issues) - _validate_optional_records(paths, mode, issues) + records_checked = _validate_optional_records(paths, mode, issues) + + schemas_checked = list(SCHEMA_IDS) + [ + schema_id for key, schema_id in _RECORD_SCHEMA_IDS if key in records_checked + ] return { "ok": not issues, "paths": paths.to_json(), "validation_mode": mode, - "schemas_checked": list(SCHEMA_IDS), + "schemas_checked": schemas_checked, "issues": issues, } diff --git a/scripts/test_contract_records.py b/scripts/test_contract_records.py index 313a5ec..1d55819 100644 --- a/scripts/test_contract_records.py +++ b/scripts/test_contract_records.py @@ -92,6 +92,31 @@ def test_present_valid_receipt_jsonl_passes(tmp_path): assert _optional_issues(tmp_path) == [] +def test_schemas_checked_reports_repair_when_a_repair_record_is_validated(): + # P3: schemas_checked must not under-report — a loop with repair records shows + # the repair schema id, not just the 4 core contract schemas. + report = validate_contract(ROOT / "examples" / "coverage-repair") + assert "loop-engineer/repair@1" in report["schemas_checked"] + assert report["schemas_checked"][:4] == [ + "loop-engineer/manifest@1", + "loop-engineer/state@1", + "loop-engineer/tasks@1", + "loop-engineer/terminal@1", + ] + + +def test_schemas_checked_omits_record_schemas_when_no_record_files(tmp_path): + (tmp_path / ".loop").mkdir() + (tmp_path / "RUNLOG.md").write_text("# RUNLOG\n", encoding="utf-8") + report = validate_contract(tmp_path) + assert report["schemas_checked"] == [ + "loop-engineer/manifest@1", + "loop-engineer/state@1", + "loop-engineer/tasks@1", + "loop-engineer/terminal@1", + ] + + def test_flagship_example_contract_validates_clean_with_repair_record(): report = validate_contract(ROOT / "examples" / "coverage-repair") assert report["ok"] is True, report["issues"] From b4a850af82d36437dc5c7120bbd6c66c3efce307 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Fri, 3 Jul 2026 15:51:19 -0400 Subject: [PATCH 10/15] refactor(rollout): rename summarize key to rollout_productivity summarize() returned its productive fraction under the key repair_productivity, the exact mislabel the slice de-branding closes: the rollout ledger carries rollout-productivity, NOT the RP baseline (metrics.py derives RP from the canonical repair record). Rename the key and update callers and tests. Co-Authored-By: Claude Fable 5 --- scripts/rollout_ledger.py | 7 +++++-- scripts/test_rollout_ledger.py | 8 ++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/scripts/rollout_ledger.py b/scripts/rollout_ledger.py index 5ed2ee5..c37fdd2 100644 --- a/scripts/rollout_ledger.py +++ b/scripts/rollout_ledger.py @@ -137,11 +137,14 @@ def summarize(path: str | Path) -> dict: productive += 1 else: rejected += 1 - repair_productivity = productive / validated if validated else 0.0 + rollout_productivity = productive / validated if validated else 0.0 return { "count": validated, "productive": productive, - "repair_productivity": repair_productivity, + # This is the rollout-productivity signal (a flywheel view of candidate + # adjudication), NOT the RP baseline — RP is derived by scripts/metrics.py + # from the canonical repair record. See the module docstring. + "rollout_productivity": rollout_productivity, "rejected": rejected, "malformed": malformed, } diff --git a/scripts/test_rollout_ledger.py b/scripts/test_rollout_ledger.py index 04de3ff..2a7f6dd 100644 --- a/scripts/test_rollout_ledger.py +++ b/scripts/test_rollout_ledger.py @@ -76,7 +76,7 @@ def test_summarize_computes_productive_fraction_and_count(tmp_path): # Assert assert summary["count"] == 3 - assert summary["repair_productivity"] == 2 / 3 + assert summary["rollout_productivity"] == 2 / 3 def test_append_is_append_only_and_preserves_prior_lines(tmp_path): @@ -101,7 +101,7 @@ def test_summarize_of_empty_ledger_is_zero(tmp_path): # Assert assert summary["count"] == 0 - assert summary["repair_productivity"] == 0.0 + assert summary["rollout_productivity"] == 0.0 # --- M4-CLI item 9: malformed ledger lines are tolerated, not fatal ---------- @@ -157,7 +157,7 @@ def test_summarize_counts_malformed_lines(tmp_path): # Valid candidates are summarized; malformed lines are counted separately. assert summary["count"] == 2 assert summary["malformed"] == 2 - assert summary["repair_productivity"] == 1.0 + assert summary["rollout_productivity"] == 1.0 def test_summarize_rejects_record_whose_productive_disagrees_with_delta(tmp_path): @@ -172,4 +172,4 @@ def test_summarize_rejects_record_whose_productive_disagrees_with_delta(tmp_path assert summary["count"] == 1 # only the honest record is validated assert summary["rejected"] == 1 - assert summary["repair_productivity"] == 1.0 + assert summary["rollout_productivity"] == 1.0 From dd21ffed9cbc88442565fe60911d201e797558e4 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Fri, 3 Jul 2026 15:51:26 -0400 Subject: [PATCH 11/15] docs: canonical repair-record path, evidence-not-proof, README baseline binding - Scaffolding (manifest template, repo-os-contract, prompt-templates) declared the stale .loop/artifacts/repair-record.json output path the metrics tool cannot read; point them at the canonical per-iteration repair path. - README Measured-baseline passage now states the committed verdict is evidence, not proof, and cites its sha256. - Add a test binding the README FCR/RP literals to docs/metrics-baseline.json so they cannot silently drift, plus a guard against the stale scaffolding path. Co-Authored-By: Claude Fable 5 --- README.md | 8 +++++-- reference/prompt-templates.md | 2 +- reference/repo-os-contract.md | 2 +- scripts/test_docs_baseline.py | 41 +++++++++++++++++++++++++++++++++++ templates/manifest.yaml.tmpl | 2 +- 5 files changed, 50 insertions(+), 5 deletions(-) create mode 100644 scripts/test_docs_baseline.py diff --git a/README.md b/README.md index f08ea13..dc158ff 100644 --- a/README.md +++ b/README.md @@ -97,8 +97,12 @@ trusted: | `false-completion-rate` | **0.0** | RUNLOG success-claims × verify bundles, cross-checked against the held-out gate flag (both agree) | | `repair-productivity` | **1.0** | one repair pass, `verification_after.score` 0.83 > `before` 0.74 (recomputed, agreed) | -The number ships with a `provenance` block naming every input file, so a skeptic -can re-derive it. Reproduce (and refuse to publish over a non-gate-backed run): +The number ships with a `provenance` block naming every input file (including the +held-out verdict's sha256), so a skeptic can re-derive it. That committed verdict +is *evidence, not proof*: it is validated structurally and shows a gate ran, but +tamper detection of the artifact itself belongs to the anti-cheat layer — the tool +does not claim the verdict is tamper-proof. Reproduce (and refuse to publish over a +non-gate-backed, inconsistent, vacuous, or unanchored run): ```bash python3 -m loop metrics examples/coverage-repair # print the scorecard diff --git a/reference/prompt-templates.md b/reference/prompt-templates.md index d9edf5e..4690cee 100644 --- a/reference/prompt-templates.md +++ b/reference/prompt-templates.md @@ -180,7 +180,7 @@ DO: do not delete assertions. Editing the gate to go green is verifier-gaming and is a security failure (see reference/safety-and-approvals.md). 4. Re-run the SAME verification. -5. Append a repair record to the RUNLOG / repair-record.json: +5. Append a repair record to the RUNLOG / .loop/repair/.json: { failure_mode, hypothesis, repair_action, verification_before, verification_after, remaining_delta } diff --git a/reference/repo-os-contract.md b/reference/repo-os-contract.md index bc334c0..2a35ac1 100644 --- a/reference/repo-os-contract.md +++ b/reference/repo-os-contract.md @@ -346,7 +346,7 @@ outputs: task_queue: TASKS.json current_state: .loop/state.json verification_bundle: .loop/artifacts/ - repair_actions: .loop/artifacts/repair-record.json + repair_actions: .loop/repair/.json terminal_state: .loop/terminal_state.json lessons_learned: .loop/memory/lessons.md diff --git a/scripts/test_docs_baseline.py b/scripts/test_docs_baseline.py new file mode 100644 index 0000000..20391c6 --- /dev/null +++ b/scripts/test_docs_baseline.py @@ -0,0 +1,41 @@ +"""Doc/scaffolding bindings for ST1: + + * AC5 — the README Measured-baseline literals are SOURCED from + docs/metrics-baseline.json, not retyped prose: parse them and assert equality + so a future baseline change can't leave the README stale. + * P3 — shipped scaffolding must declare the canonical repair-record location + (.loop/repair/.json), never the stale .loop/artifacts path the + metrics tool cannot read. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +_REPO = Path(__file__).resolve().parent.parent + + +def _readme_metric(readme: str, label: str) -> float: + m = re.search(rf"\|\s*`{re.escape(label)}`\s*\|\s*\*\*([0-9.]+)\*\*\s*\|", readme) + assert m, f"no README Measured-baseline row for {label}" + return float(m.group(1)) + + +def test_readme_baseline_literals_match_committed_scorecard(): + readme = (_REPO / "README.md").read_text(encoding="utf-8") + baseline = json.loads((_REPO / "docs" / "metrics-baseline.json").read_text(encoding="utf-8")) + assert _readme_metric(readme, "false-completion-rate") == baseline["false_completion_rate"] + assert _readme_metric(readme, "repair-productivity") == baseline["repair_productivity"] + + +def test_scaffolding_uses_canonical_repair_record_path(): + stale = ".loop/artifacts/repair-record.json" + scanned = [ + _REPO / "templates" / "manifest.yaml.tmpl", + _REPO / "reference" / "repo-os-contract.md", + _REPO / "reference" / "prompt-templates.md", + ] + offenders = [str(p.relative_to(_REPO)) for p in scanned if stale in p.read_text(encoding="utf-8")] + assert not offenders, f"stale repair-record path {stale!r} in: {offenders}" diff --git a/templates/manifest.yaml.tmpl b/templates/manifest.yaml.tmpl index 623cbec..906c037 100644 --- a/templates/manifest.yaml.tmpl +++ b/templates/manifest.yaml.tmpl @@ -21,7 +21,7 @@ outputs: task_queue: TASKS.json current_state: .loop/state.json verification_bundle: .loop/artifacts/ - repair_actions: .loop/artifacts/repair-record.json + repair_actions: .loop/repair/.json terminal_state: .loop/terminal_state.json lessons_learned: .loop/memory/lessons.md From 3cfbb13cf2cc81cbb899b8acababae31fd1385f2 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Fri, 3 Jul 2026 15:51:42 -0400 Subject: [PATCH 12/15] chore(metrics): regenerate gate-backed baseline scorecard Recompute docs/metrics-baseline.json with the hardened metrics command. Still evidence_backed via the real committed holdout verdict, FCR 0.0 / RP 1.0; adds the new provenance fields (holdout sha256, unanchored_records, unrecognized_ outcomes). Co-Authored-By: Claude Fable 5 --- docs/metrics-baseline.json | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/metrics-baseline.json b/docs/metrics-baseline.json index c8bd01c..5099536 100644 --- a/docs/metrics-baseline.json +++ b/docs/metrics-baseline.json @@ -20,16 +20,24 @@ ".loop/repair/iter-002.json" ], "rejected_records": [], + "unanchored_records": [], + "unrecognized_outcomes": [], "false_completion_rate_holdout": 0.0, "fcr_methods_agree": true, "holdout_source": [ ".loop/artifacts/holdout-verdict.json" ], + "holdout_verdicts": [ + { + "source": ".loop/artifacts/holdout-verdict.json", + "sha256": "b203cf5bb6b05f15172209e067d9f4fa0aaf60cada33fe98172a8279051c1000" + } + ], "unmatched_verify": [] }, "baseline": { "source_example": "examples/coverage-repair", - "commit": "4d2db18927b828e1e9193afb6342f50021290dc7", + "commit": "dd21ffed9cbc88442565fe60911d201e797558e4", "inputs": [ "examples/coverage-repair/.loop/artifacts/holdout-verdict.json", "examples/coverage-repair/.loop/artifacts/verify-T1.json", From 2344314b5f0ce06c121baf51c4f17688186d42fb Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Fri, 3 Jul 2026 16:22:16 -0400 Subject: [PATCH 13/15] fix(metrics): outcome-class FCR, verdict-only baseline, task-keyed RP anchoring (round-2 adversarial findings) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A completion-class claim (task_passed/succeeded/terminal) is clean only if every attached bundle is green — no cross-iteration escape; the repaired-intermediate exception is progress-class (advanced) only and order-aware. --baseline accepts nothing weaker than a structurally-valid gate verdict artifact; the gate-script existence check no longer leaks to this repo's own toolkit. RP anchors to a same-task red->green bundle pair (order enforced when known), not global score membership. Short outcome tokens are surfaced. Every round-2 exploit (G0-G5, F, H1/H2, N2/N2b, N3) is pinned as a regression test. Co-Authored-By: Claude Fable 5 --- README.md | 5 +- scripts/metrics.py | 204 ++++++++++++++++++++++++++-------------- scripts/test_metrics.py | 157 ++++++++++++++++++++++++++++++- 3 files changed, 292 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index dc158ff..ee067c7 100644 --- a/README.md +++ b/README.md @@ -99,8 +99,9 @@ trusted: The number ships with a `provenance` block naming every input file (including the held-out verdict's sha256), so a skeptic can re-derive it. That committed verdict -is *evidence, not proof*: it is validated structurally and shows a gate ran, but -tamper detection of the artifact itself belongs to the anti-cheat layer — the tool +is *evidence, not proof*: it is validated structurally, but a fully-fabricated, +internally-consistent artifact defeats offline shape-checking by construction — +tamper detection of the artifact itself belongs to the anti-cheat layer; the tool does not claim the verdict is tamper-proof. Reproduce (and refuse to publish over a non-gate-backed, inconsistent, vacuous, or unanchored run): diff --git a/scripts/metrics.py b/scripts/metrics.py index 93b5574..9d124c9 100644 --- a/scripts/metrics.py +++ b/scripts/metrics.py @@ -17,32 +17,42 @@ * **FCR is derived two ways and disagreement is surfaced.** (a) the RUNLOG success-claim × verify-bundle cross-join (the deterministic anchor, per §3), and (b) the aggregated held-out-gate `false_completion` flag. An unmatched - success-claim counts as a false completion (fail-closed, §8). A success claim - is clean only if EVERY verify bundle attached to its iteration is green, or a - red one's own task later reached green (an honestly-repaired intermediate) — - an unrelated green bundle can never launder the claimed task's red gate. + success-claim counts as a false completion (fail-closed, §8). Claim + cleanness is outcome-class aware: a COMPLETION-class claim (`task_passed`, + `terminal`, `succeeded`, or the terminal state itself) is clean only if + EVERY verify bundle attached to its iteration is green — no exception of + any kind ("not backed by a green verify … is a false completion, full + stop"); a PROGRESS-class claim (`advanced`) may carry a red bundle only if + that bundle's own task reaches green in a STRICTLY LATER iteration (an + honestly-repaired intermediate). An unrelated green bundle never launders a + red gate, and a same-iteration green never excuses its own task's red — + within-iteration chronology is unknowable, so it fails closed. The `### Outcome` token contract: a claim counts toward the FCR denominator only -when its outcome token is a recognized SUCCESS token (`task_passed`, `terminal`, -`succeeded`, `advanced`); `repair_triggered` / `task_failed` and peers are honest -reds. Any token in neither set is surfaced under `provenance.unrecognized_outcomes` -so a synonym (`shipped`, `done`, …) is visible rather than silently escaping the +when its outcome token is a recognized SUCCESS token — completion-class +(`task_passed`, `terminal`, `succeeded`) or progress-class (`advanced`); +`repair_triggered` / `task_failed` and peers are honest reds. Any other token of +two or more characters is surfaced under `provenance.unrecognized_outcomes` so a +synonym (`shipped`, `done`, `ok`, …) is visible rather than silently escaping the denominator — it never widens the recognized-success set on its own. A committed held-out verdict is validated structurally (it must carry the per-check `visible`/`holdout` arrays and internally-consistent flags a real `holdout_gate` run emits, not a hand-set 4-field stub) and its sha256 is recorded in provenance. -That committed verdict is *evidence, not proof*: it demonstrates a gate ran, but +That committed verdict is *evidence, not proof*: a fully-fabricated, +internally-consistent artifact defeats offline shape-checking by construction, so tamper detection of the artifact itself belongs to the anti-cheat layer (`anticheat_scan.py`) — this command does not, and does not claim to, make the verdict tamper-proof. `--baseline` writes a checked-in scorecard, but only over a genuinely gate-backed -run: it refuses (non-zero, writes nothing) if the run is not `evidence_backed`, if -any record was rejected, if the two FCR methods disagree, if no iteration claims -success (a vacuous 0/0 is not a publishable 0.0), or if any counted repair record -is unanchored. Baselining a self-asserted run would itself be a false completion of -ST1. +run: it refuses (non-zero, writes nothing) if no structurally-valid held-out +verdict artifact exists in the loop (plain-mode `evidence_backed` via a gate line +in a verify script is a weaker heuristic and NEVER qualifies a published +baseline), if any record was rejected, if the two FCR methods disagree, if no +iteration claims success (a vacuous 0/0 is not a publishable 0.0), or if any +counted repair record is unanchored. Baselining a self-asserted run would itself +be a false completion of ST1. Pure stdlib, offline, deterministic: the same loop dir yields a byte-identical scorecard. @@ -74,7 +84,12 @@ # A RUNLOG iteration whose outcome declares a task/loop reached "done". A # repair_triggered / task_failed outcome is an honest red, NOT a success claim. -_SUCCESS_OUTCOME_TOKENS = ("task_passed", "terminal", "succeeded", "advanced") +# Completion-class tokens assert "done" — every attached bundle must be green, +# no exceptions. Progress-class tokens assert forward motion — a red +# intermediate is excusable only by a strictly-later green of the same task. +_COMPLETION_OUTCOME_TOKENS = ("task_passed", "terminal", "succeeded") +_PROGRESS_OUTCOME_TOKENS = ("advanced",) +_SUCCESS_OUTCOME_TOKENS = _COMPLETION_OUTCOME_TOKENS + _PROGRESS_OUTCOME_TOKENS # Recognized honest-red outcome tokens: not a success claim, but a known outcome # (so they are not surfaced as an "unrecognized" synonym). Any outcome token in @@ -94,7 +109,7 @@ _ITER_HEADER_RE = re.compile(r"(?m)^##\s+Iteration\s+(\S+)") # "outcome" declaration followed (within a little markup/whitespace) by its token. -_OUTCOME_RE = re.compile(r"outcome[^A-Za-z0-9]{0,40}?([A-Za-z][A-Za-z_]{2,})", re.IGNORECASE | re.DOTALL) +_OUTCOME_RE = re.compile(r"outcome[^A-Za-z0-9]{0,40}?([A-Za-z][A-Za-z_]{1,})", re.IGNORECASE | re.DOTALL) _VERIFY_REF_RE = re.compile(r"(verify-[\w.-]+?\.json)") @@ -248,6 +263,7 @@ def _assign_bundles_to_iters(bundles: list[dict], blocks: list[tuple[str, str]]) target = b["iter"] if target is None: target = next((iid for iid, refs in refs_by_iter.items() if b["name"] in refs), None) + b["assigned_iter"] = target if target is None: unmatched.append(b["name"]) else: @@ -316,12 +332,11 @@ def _verify_scripts(workspace: Path) -> list[Path]: def _gate_script_present(workspace: Path) -> bool: - """A gate script the verify surface can actually invoke exists — either bundled - with the loop or in the loop-engineer toolkit it composes with.""" - for base in (workspace / "scripts", _REPO_ROOT / "scripts"): - if any((base / name).exists() for name in _GATE_SCRIPTS): - return True - return False + """A gate script the loop's OWN verify surface can invoke exists. Only the + loop's workspace counts — checking this repo's toolkit would be vacuously + true for every foreign loop, since loop-engineer ships holdout_gate.py.""" + base = workspace / "scripts" + return any((base / name).exists() for name in _GATE_SCRIPTS) def _gate_invoked(paths: LoopPaths, gate_verdicts: list[dict]) -> bool: @@ -329,11 +344,13 @@ def _gate_invoked(paths: LoopPaths, gate_verdicts: list[dict]) -> bool: invocation-evidence rule (HI4), NOT looser prose matching: (a) a recorded, structurally-valid gate VERDICT artifact, or - (b) a NON-COMMENT gate-token line in a verify-* script whose gate script - file actually exists. + (b) a NON-COMMENT gate-token line in a verify-* script, where the loop's + own workspace also carries the gate script file. - A bare TASKS.json verify *declaration* or a RUNLOG prose mention is NOT an - invocation (a ``# TODO: call holdout_gate.py`` comment earns nothing).""" + Path (b) is a WEAKER heuristic than (a) — a script line proves intent, not + a run — which is why ``--baseline`` accepts only (a). A bare TASKS.json + verify *declaration* or a RUNLOG prose mention is NOT an invocation (a + ``# TODO: call holdout_gate.py`` comment earns nothing).""" if gate_verdicts: return True if not _gate_script_present(paths.workspace): @@ -370,15 +387,23 @@ def _load_receipt_costs(loop_dir: Path) -> tuple[float | None, int]: return total, count -def _anchor_repair(record: dict, bundle_scores: set[float]) -> dict: +def _anchor_repair(record: dict, bundles: list[dict], iter_order: dict[str, int]) -> dict: """Cross-check a repair record's self-reported before/after scores against the deterministic verify bundles (§4.3, RP anchoring). - Returns ``status`` one of ``anchored`` (both scores corroborated), - ``unanchored`` (no verify bundle carries a score to anchor against), or - ``rejected`` (a score is present but no verify bundle corroborates it — a - fabricated delta). ``recheck_productive`` runs first, so by here a repair - record's before/after scores are already numeric. + A record anchors only against a SAME-TASK red→green bundle pair: a non-green + bundle whose score equals ``verification_before.score`` and a green bundle of + the same task whose score equals ``verification_after.score``. When both + bundles' iterations are known, the red must precede the green — a pair whose + green came first is a regression, not a repair. Set-membership over all + bundle scores anchored nothing (a loop authors its own bundles, so matching + two free-floating numbers is free). + + Returns ``status`` one of ``anchored``, ``rejected`` (scored bundles exist + but no qualifying pair corroborates the delta — a fabricated/borrowed + number), or ``unanchored`` (no scored bundle to anchor against at all). + ``recheck_productive`` runs first, so by here a valid record's before/after + scores are already numeric. """ before = record.get("verification_before") after = record.get("verification_after") @@ -386,19 +411,28 @@ def _anchor_repair(record: dict, bundle_scores: set[float]) -> dict: after = _num(after.get("score")) if isinstance(after, dict) else None if before is None and after is None: return {"status": "unanchored", "reason": "no before/after score to anchor"} - if not bundle_scores: + scored = [b for b in bundles if b["score"] is not None] + if not scored: return {"status": "unanchored", "reason": "no verify bundle scores to anchor against"} - missing = [ - label - for label, value in (("verification_before", before), ("verification_after", after)) - if value is not None and value not in bundle_scores - ] - if missing: - return { - "status": "rejected", - "reason": f"self-reported {', '.join(missing)}.score not corroborated by any verify bundle", - } - return {"status": "anchored", "reason": "ok"} + reds = [b for b in scored if not b["green"] and b["task"]] + greens = [b for b in scored if b["green"] and b["task"]] + for red in reds: + for green in greens: + if red["task"] != green["task"]: + continue + red_order = iter_order.get(red.get("assigned_iter")) + green_order = iter_order.get(green.get("assigned_iter")) + if red_order is not None and green_order is not None and not red_order < green_order: + continue + if red["score"] == before and green["score"] == after: + return {"status": "anchored", "reason": "ok"} + return { + "status": "rejected", + "reason": ( + "self-reported verification_before/after scores are not corroborated by any " + "same-task red-to-green verify bundle pair" + ), + } def _rel(path: Path, base: Path) -> str: @@ -425,10 +459,20 @@ def compute_metrics(loop_dir: str | Path, loop_label: str | None = None) -> dict verify_by_iter, unmatched_verify = _assign_bundles_to_iters(bundles, blocks) # Success claims: RUNLOG success-outcome iterations, plus the terminal claim. - claim_iters: set[str] = {iid for iid, text in blocks if _block_claims_success(text)} + # Completion-class claims (task_passed/terminal/succeeded, and the terminal + # state itself) assert "done"; progress-class claims (advanced) assert motion. + completion_iters: set[str] = set() + progress_iters: set[str] = set() + for iid, text in blocks: + tokens = _block_outcome_tokens(text) + if any(t in _COMPLETION_OUTCOME_TOKENS for t in tokens): + completion_iters.add(iid) + elif any(t in _PROGRESS_OUTCOME_TOKENS for t in tokens): + progress_iters.add(iid) if terminal.get("state") == "Succeeded": tid = _norm_iter(terminal.get("iteration_id")) - claim_iters.add(tid) + completion_iters.add(tid) + progress_iters.discard(tid) # The terminal names the verify bundles that back its success claim. evidence = terminal.get("evidence") if isinstance(evidence, list): @@ -438,20 +482,41 @@ def compute_metrics(loop_dir: str | Path, loop_label: str | None = None) -> dict verify_by_iter.setdefault(tid, []) if b not in verify_by_iter[tid]: verify_by_iter[tid].append(b) - - # FCR-A: a success claim is clean only if every verify bundle attached to its - # iteration is green — with one honest exception: a red bundle whose OWN task - # later reached green is a repaired intermediate, not a false completion (the - # flagship's verify-T2-iter1 → verify-T2). An UNRELATED green bundle can never - # launder a claimed task's still-red gate. Unmatched claims fail closed (§8). - green_tasks = {b["task"] for b in bundles if b["green"] and b["task"]} + claim_iters = completion_iters | progress_iters + + # FCR-A cleanness is outcome-class aware. Completion-class: every attached + # bundle green, no exceptions ("not backed by a green verify … is a false + # completion, full stop"). Progress-class: a red bundle is excused only if + # its OWN task reaches green in a STRICTLY LATER iteration (the flagship's + # verify-T2-iter1 → verify-T2) — an unrelated green never launders a red + # gate, and a same-iteration green never excuses its own task's red + # (within-iteration chronology is unknowable). Unmatched claims and + # unordered iterations fail closed (§8). + iter_order = {iid: i for i, (iid, _text) in enumerate(blocks)} + green_task_orders: dict[str, set[int]] = {} + for iid, attached in verify_by_iter.items(): + order = iter_order.get(iid) + if order is None: + continue + for b in attached: + if b["green"] and b["task"]: + green_task_orders.setdefault(b["task"], set()).add(order) def _claim_is_clean(iid: str) -> bool: attached = verify_by_iter.get(iid, []) - if not attached or not any(b["green"] for b in attached): + if not attached: return False + if iid in completion_iters: + return all(b["green"] for b in attached) + if not any(b["green"] for b in attached): + return False + claim_order = iter_order.get(iid) for b in attached: - if not b["green"] and (not b["task"] or b["task"] not in green_tasks): + if b["green"]: + continue + if claim_order is None or not b["task"]: + return False + if not any(o > claim_order for o in green_task_orders.get(b["task"], ())): return False return True @@ -472,11 +537,10 @@ def _claim_is_clean(iid: str) -> bool: evidence_backed = _gate_invoked(paths, gate_verdicts) # RP: over recomputed-and-agreed repair records only, whose before/after - # scores are anchored to the deterministic verify bundles (§4.3). A record - # whose stored productive lies (recheck) OR whose scores are not corroborated - # by any verify bundle (anchor) is rejected; a record with no bundle to anchor - # against is counted but flagged unanchored (a baseline refuses over those). - bundle_scores = {b["score"] for b in bundles if b["score"] is not None} + # scores are anchored to a same-task red→green verify-bundle pair (§4.3). A + # record whose stored productive lies (recheck) OR whose scores no pair + # corroborates (anchor) is rejected; a record with no scored bundle to + # anchor against is counted but flagged unanchored (a baseline refuses). validated = 0 productive = 0 rejected: list[dict] = [] @@ -489,7 +553,7 @@ def _claim_is_clean(iid: str) -> bool: if not verdict["valid"]: rejected.append({"record": rel, "reason": verdict["reason"]}) continue - anchor = _anchor_repair(record, bundle_scores) + anchor = _anchor_repair(record, bundles, iter_order) if anchor["status"] == "rejected": rejected.append({"record": rel, "reason": anchor["reason"]}) continue @@ -572,19 +636,21 @@ def _git_commit() -> str | None: def build_baseline(loop_dir: str | Path, loop_label: str | None = None) -> tuple[bool, dict, list[str]]: """Compute the scorecard and check the §4.5 baseline preconditions. - Returns ``(ok, scorecard, refusal_reasons)``. ``ok`` is False when the run is - not evidence-backed, contains a rejected record, has disagreeing FCR methods, - claims no success (vacuous 0/0), or counts an unanchored repair record. Each - refusal names the precondition that failed. + Returns ``(ok, scorecard, refusal_reasons)``. ``ok`` is False when the loop + carries no structurally-valid gate verdict artifact (the strict form of + evidence-backing — a verify-script gate line never qualifies a baseline), + contains a rejected record, has disagreeing FCR methods, claims no success + (vacuous 0/0), or counts an unanchored repair record. Each refusal names the + precondition that failed. """ scorecard = compute_metrics(loop_dir, loop_label) prov = scorecard["provenance"] reasons: list[str] = [] - if not scorecard["evidence_backed"]: + if not prov["holdout_verdicts"]: reasons.append( - "run is not evidence_backed — no held-out / anti-cheat gate invocation " - "detectable (a structurally-valid verdict artifact, or a non-comment gate " - "line in a verify-* script whose gate script exists)" + "no structurally-valid held-out verdict artifact in the loop — a published " + "baseline requires a recorded gate VERDICT; a gate line in a verify-* script " + "(plain-mode evidence_backed) is a weaker heuristic and never qualifies" ) rejected = prov["rejected_records"] if rejected: diff --git a/scripts/test_metrics.py b/scripts/test_metrics.py index 818573d..6a57a25 100644 --- a/scripts/test_metrics.py +++ b/scripts/test_metrics.py @@ -395,18 +395,35 @@ def test_comment_only_gate_line_in_verify_script_is_not_evidence(tmp_path): def test_executed_gate_line_in_verify_script_is_evidence(tmp_path): + # Path (b) requires the loop's OWN workspace to carry the gate script. ws = _make_loop( tmp_path, runlog=_RUNLOG_ONE_CLAIM, verify={"verify-A.json": {"task": "A", "outcome": "PASS", "score": 1.0}}, ) (ws / "scripts").mkdir() + (ws / "scripts" / "holdout_gate.py").write_text("# loop-local gate\n", encoding="utf-8") (ws / "scripts" / "verify-full").write_text( "#!/bin/sh\npython3 scripts/holdout_gate.py manifest.json\n", encoding="utf-8" ) assert metrics.compute_metrics(ws)["evidence_backed"] is True +def test_gate_script_outside_loop_does_not_confer_evidence(tmp_path): + # loop-engineer itself ships holdout_gate.py; that must not vacuously satisfy + # the "gate script exists" clause for a foreign loop that carries none. + ws = _make_loop( + tmp_path, + runlog=_RUNLOG_ONE_CLAIM, + verify={"verify-A.json": {"task": "A", "outcome": "PASS", "score": 1.0}}, + ) + (ws / "scripts").mkdir() + (ws / "scripts" / "verify-full").write_text( + "#!/bin/sh\npython3 scripts/holdout_gate.py manifest.json\n", encoding="utf-8" + ) + assert metrics.compute_metrics(ws)["evidence_backed"] is False + + # --- FCR cross-join laundering (P2) ------------------------------------------- @@ -427,13 +444,17 @@ def test_unrelated_green_bundle_does_not_launder_a_red_claimed_task(tmp_path): def test_honest_intermediate_red_later_repaired_is_not_a_false_completion(tmp_path): - # The flagship shape: a red bundle whose OWN task later reaches green is an - # honestly-repaired intermediate, not a laundered false completion. + # The real flagship shape: iteration 1 claims progress (`advanced`) with a + # green T1 beside an honest red T2; T2 reaches green in a STRICTLY LATER + # iteration. That is a repaired intermediate, not a laundered completion. ws = _make_loop( tmp_path, runlog="# RUNLOG\n\n## Iteration 1 — t\n\n- **outcome:** advanced\n" - "- refs: .loop/artifacts/verify-T2-iter1.json and .loop/artifacts/verify-T2.json\n", + "- refs: .loop/artifacts/verify-T1.json and .loop/artifacts/verify-T2-iter1.json\n" + "\n## Iteration 2 — t\n\n- **outcome:** repair_triggered\n" + "- refs: .loop/artifacts/verify-T2.json\n", verify={ + "verify-T1.json": {"task": "T1", "outcome": "PASS", "score": 0.74}, "verify-T2-iter1.json": {"task": "T2", "outcome": "FAIL", "score": 0.74}, "verify-T2.json": {"task": "T2", "outcome": "PASS", "score": 0.83}, }, @@ -520,6 +541,136 @@ def test_rp_record_anchors_cleanly_against_matching_bundles(tmp_path): assert sc["repair_productivity"] == 1.0 +# --- round-2 adversarial regressions: outcome classes, verdict-only baseline, --- +# --- task-keyed RP anchoring (re-verify findings) -------------------------------- + + +def test_completion_claim_with_red_bundle_is_false_completion_even_if_later_repaired(tmp_path): + # Exploits G0/G2/H1: a COMPLETION-class claim (task_passed/succeeded/terminal) + # over a red gate is a false completion, full stop — a later repair of the + # same task never excuses it (that escape is progress-class only). + ws = _make_loop( + tmp_path, + runlog="# RUNLOG\n\n## Iteration 1 — t\n\n- **outcome:** succeeded\n" + "- refs: .loop/artifacts/verify-T1.json and .loop/artifacts/verify-T2-iter1.json\n" + "\n## Iteration 2 — t\n\n- **outcome:** repair_triggered\n" + "- refs: .loop/artifacts/verify-T2.json\n", + verify={ + "verify-T1.json": {"task": "T1", "outcome": "PASS", "score": 0.74}, + "verify-T2-iter1.json": {"task": "T2", "outcome": "FAIL", "score": 0.74}, + "verify-T2.json": {"task": "T2", "outcome": "PASS", "score": 0.83}, + }, + ) + sc = metrics.compute_metrics(ws) + assert sc["false_completions"] == 1 + assert sc["false_completion_rate"] == 1.0 + + +def test_progress_claim_same_iteration_green_does_not_excuse_its_own_tasks_red(tmp_path): + # Exploit G3 class: a green sibling of the SAME task in the SAME iteration + # proves nothing about order (within-iteration chronology is unknowable) — + # fail closed. + ws = _make_loop( + tmp_path, + runlog="# RUNLOG\n\n## Iteration 1 — t\n\n- **outcome:** advanced\n" + "- refs: .loop/artifacts/verify-T2-a.json and .loop/artifacts/verify-T2-b.json\n", + verify={ + "verify-T2-a.json": {"task": "T2", "outcome": "FAIL", "score": 0.74}, + "verify-T2-b.json": {"task": "T2", "outcome": "PASS", "score": 0.83}, + }, + ) + assert metrics.compute_metrics(ws)["false_completions"] == 1 + + +def test_progress_claim_over_regressed_task_is_false_completion(tmp_path): + # Exploit H2: task green EARLIER, red at claim time — "later reached green" + # must be order-aware, not a global green-anywhere set. + ws = _make_loop( + tmp_path, + runlog="# RUNLOG\n\n## Iteration 1 — t\n\n- **outcome:** repair_triggered\n" + "- refs: .loop/artifacts/verify-X-early.json\n" + "\n## Iteration 2 — t\n\n- **outcome:** advanced\n" + "- refs: .loop/artifacts/verify-X-late.json\n", + verify={ + "verify-X-early.json": {"task": "X", "outcome": "PASS", "score": 0.9}, + "verify-X-late.json": {"task": "X", "outcome": "FAIL", "score": 0.4}, + }, + ) + assert metrics.compute_metrics(ws)["false_completions"] == 1 + + +def test_baseline_requires_verdict_artifact_not_verify_script_reference(tmp_path): + # Exploit N2 pinned: plain-mode evidence_backed via a loop-local gate script + + # invocation line is a heuristic — it must NEVER qualify a published baseline. + ws = _make_loop( + tmp_path, + runlog=_RUNLOG_ONE_CLAIM, + verify={"verify-A.json": {"task": "A", "outcome": "PASS", "score": 1.0}}, + ) + (ws / "scripts").mkdir() + (ws / "scripts" / "holdout_gate.py").write_text("# loop-local gate\n", encoding="utf-8") + (ws / "scripts" / "verify-full").write_text( + "#!/bin/sh\npython3 scripts/holdout_gate.py manifest.json\n", encoding="utf-8" + ) + assert metrics.compute_metrics(ws)["evidence_backed"] is True + out = tmp_path / "docs" / "metrics-baseline.json" + rc = metrics.write_baseline(ws, out, loop_label="ws") + assert rc != 0 + assert not out.exists() + ok, _sc, reasons = metrics.build_baseline(ws, "ws") + assert ok is False + assert any("verdict" in r.lower() for r in reasons) + + +def test_rp_borrowed_cross_task_scores_are_rejected(tmp_path): + # Exploit N3: before/after matching free-floating scores from DIFFERENT tasks + # anchors nothing — only a same-task red→green pair corroborates a repair. + ws = _make_loop( + tmp_path, + runlog=_RUNLOG_ONE_CLAIM, + verify={ + "verify-zzz.json": {"task": "ZZZ", "outcome": "FAIL", "score": 0.74}, + "verify-A.json": {"task": "A", "outcome": "PASS", "score": 0.83}, + }, + repair={"iter-001.json": _repair_record(0.74, 0.83, True)}, + ) + sc = metrics.compute_metrics(ws) + rejected = sc["provenance"]["rejected_records"] + assert any(r["record"].endswith("iter-001.json") for r in rejected) + assert sc["repair_productivity"] is None + + +def test_rp_pair_with_green_before_red_is_a_regression_not_an_anchor(tmp_path): + # Order-aware anchoring: when both bundle iterations are known, the red must + # precede the green; a green-then-red pair is a regression, not a repair. + ws = _make_loop( + tmp_path, + runlog="# RUNLOG\n\n## Iteration 1 — t\n\n- **outcome:** task_passed\n" + "- refs: .loop/artifacts/verify-good.json\n" + "\n## Iteration 2 — t\n\n- **outcome:** repair_triggered\n" + "- refs: .loop/artifacts/verify-bad.json\n", + verify={ + "verify-good.json": {"task": "X", "outcome": "PASS", "score": 0.83}, + "verify-bad.json": {"task": "X", "outcome": "FAIL", "score": 0.74}, + }, + repair={"iter-001.json": _repair_record(0.74, 0.83, True)}, + ) + sc = metrics.compute_metrics(ws) + rejected = sc["provenance"]["rejected_records"] + assert any(r["record"].endswith("iter-001.json") for r in rejected) + + +def test_short_outcome_token_is_surfaced(tmp_path): + # A <=2-char synonym ("ok") must be surfaced, not silently dropped. + ws = _make_loop( + tmp_path, + runlog="# RUNLOG\n\n## Iteration 1 — t\n\n- **outcome:** ok\n", + ) + sc = metrics.compute_metrics(ws) + assert "ok" in sc["provenance"]["unrecognized_outcomes"] + assert sc["iterations_claiming_success"] == 0 + + def test_baseline_refuses_when_a_counted_rp_record_is_unanchored(tmp_path): # A green claim-backing bundle with no numeric score leaves nothing to anchor # the repair record's before/after against → unanchored → baseline refuses. From 46d4ce163c65d9f075babecedcc65e94b4e1200f Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Fri, 3 Jul 2026 16:22:17 -0400 Subject: [PATCH 14/15] chore(metrics): restamp baseline at the round-2-hardened derivation Co-Authored-By: Claude Fable 5 --- docs/metrics-baseline.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/metrics-baseline.json b/docs/metrics-baseline.json index 5099536..0de7c24 100644 --- a/docs/metrics-baseline.json +++ b/docs/metrics-baseline.json @@ -37,7 +37,7 @@ }, "baseline": { "source_example": "examples/coverage-repair", - "commit": "dd21ffed9cbc88442565fe60911d201e797558e4", + "commit": "2344314b5f0ce06c121baf51c4f17688186d42fb", "inputs": [ "examples/coverage-repair/.loop/artifacts/holdout-verdict.json", "examples/coverage-repair/.loop/artifacts/verify-T1.json", From d7549632c89829c36a0c50dcb2b8fec2d0ea9be4 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Fri, 3 Jul 2026 21:13:29 -0400 Subject: [PATCH 15/15] fix(tests): make live-contract and foreign-cwd tests fresh-checkout-safe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The repo's live .loop run-state is gitignored, so CI has none — the live-contract gate skips there and stays a local/operator check. The foreign-cwd CLI test sets PYTHONPATH for module visibility (CI has no editable install); it tests cwd-independent scripts/ resolution, not packaging. Verified against a git-archive fresh-checkout simulation. Co-Authored-By: Claude Fable 5 --- scripts/test_contract_records.py | 6 ++++++ scripts/test_metrics_cli.py | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/scripts/test_contract_records.py b/scripts/test_contract_records.py index 1d55819..92d6f9b 100644 --- a/scripts/test_contract_records.py +++ b/scripts/test_contract_records.py @@ -8,6 +8,8 @@ import sys from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parent.parent if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) @@ -123,5 +125,9 @@ def test_flagship_example_contract_validates_clean_with_repair_record(): def test_repo_own_contract_still_validates_clean(): + # The repo's live .loop run-state is gitignored, so a fresh checkout (CI) + # has none; the live-contract gate is a local/operator check. + if not (ROOT / ".loop" / "state.json").exists(): + pytest.skip("no live .loop contract in this checkout (gitignored run-state)") report = validate_contract(ROOT / ".loop") assert report["ok"] is True, report["issues"] diff --git a/scripts/test_metrics_cli.py b/scripts/test_metrics_cli.py index f217cfb..8d0410e 100644 --- a/scripts/test_metrics_cli.py +++ b/scripts/test_metrics_cli.py @@ -6,6 +6,7 @@ from __future__ import annotations import json +import os import re import subprocess import sys @@ -100,10 +101,14 @@ def test_entrypoint_resolves_repo_relative_scripts_dir(): def test_metrics_runs_from_a_foreign_cwd(tmp_path): # Proves scripts/ resolution is repo-relative, not cwd-relative: invoke from an - # unrelated cwd with an absolute target. + # unrelated cwd with an absolute target. PYTHONPATH stands in for the editable + # install's module visibility (CI has no `pip install -e .`); what is under + # test is the cwd-independent scripts/ resolution, not packaging. + env = dict(os.environ, PYTHONPATH=str(ROOT)) result = subprocess.run( [sys.executable, "-m", "loop", "metrics", str(ROOT / "examples" / "coverage-repair")], cwd=tmp_path, + env=env, text=True, capture_output=True, )