From 025b60f6dd8240e485c60ca98e82e419ef3c97c7 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Sun, 5 Jul 2026 18:37:58 -0400 Subject: [PATCH 1/5] fix(inspect): grade false-completion defense on execution evidence, not tokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The inspector's false-completion-defense credit was fully gameable: a keyword-stuffed fake scored 100/100. Two holes: - _gate_invoked_in_verify credited any verify-* line containing a gate token, so echo "holdout_gate" (and printf / ":" / quoted-string variants) earned full "invoked" credit without executing anything. - _gate_run_recorded credited a bare gate token anywhere in RUNLOG.md prose, so stuffed narration sufficed — and because a token like anticheat_scan contains the run-word "scan", even a token+run-word rule would self-satisfy. Now credit requires genuine execution evidence: the gate script (holdout_gate.py / anticheat_scan.py / anti_cheat.py) invoked as a command (not printed by an inert emitter), or a RUNLOG line carrying a gate token AND an independent run-word (checked against the residue after tokens are stripped), or a parseable .loop/receipts/*.jsonl record. The genuinely gate-backed flagship (examples/coverage-repair) keeps its invoked credit — its double-quoted python3 "$REPO/scripts/holdout_gate.py" invocation is a real execution. Co-Authored-By: Claude Fable 5 --- scripts/inspect_loop.py | 88 ++++++++++++++++++++----- scripts/test_inspect_loop.py | 121 +++++++++++++++++++++++++++++++++++ 2 files changed, 192 insertions(+), 17 deletions(-) diff --git a/scripts/inspect_loop.py b/scripts/inspect_loop.py index 61374d0..ddc848f 100644 --- a/scripts/inspect_loop.py +++ b/scripts/inspect_loop.py @@ -16,10 +16,12 @@ key, or the phrase "false-completion" in prose earns *nothing* — those are assertions the loop makes about itself. The three grades are: - * **invoked** (full credit) — a ``scripts/verify-*`` gate invokes a holdout / - anti-cheat gate on an executable (non-comment) line, OR ``RUNLOG.md`` / - ``.loop/receipts/*.jsonl`` records an actual run (a ``holdout_gate`` verdict - / anti-cheat scan result). + * **invoked** (full credit) — a ``scripts/verify-*`` gate *executes* a holdout + / anti-cheat gate script (``python3 scripts/holdout_gate.py``, not an + ``echo "holdout_gate"`` that only prints the token), OR ``RUNLOG.md`` / + ``.loop/receipts/*.jsonl`` records an actual run (a gate token AND a run-word + on one line — a ``holdout_gate`` verdict / anti-cheat scan result — not a + bare token in stuffed prose). * **wired** (partial credit, half the weight) — a gate script file exists (``scripts/holdout_gate.py`` / ``anticheat_scan.py`` / ``anti_cheat.py``) and is referenced from the contract's verify surface (SPEC / WORKFLOW / @@ -41,6 +43,7 @@ from __future__ import annotations import json +import re import sys from pathlib import Path @@ -118,6 +121,17 @@ class _Paths: _GATE_RUN_WORDS = ("verdict", "scan", "result", "passed", "failed", "ran", "clean", "flagged") _FALSE_COMPLETION_PARTIAL_DIVISOR = 2 # wired-but-unrun earns half the weight +# A gate script referenced as a *.py path — the invocation shape a real verify +# gate uses (`python3 scripts/holdout_gate.py`, `./scripts/anticheat_scan.py`). +# The bare token ("holdout_gate") without .py is prose, not an invocation. +_GATE_SCRIPT_RE = re.compile(r"\b(?:holdout_gate|anticheat_scan|anti_cheat)\.py\b") +# Commands whose arguments are inert output, not executed gates: a token inside +# `echo`/`printf`/`:` (or the `true`/`false` no-ops) is printed, never run. +_INERT_EMITTERS = frozenset({"echo", "printf", ":", "true", "false"}) +# Split a shell line into command segments so a real invocation chained after an +# inert emitter (`echo x && python3 ...holdout_gate.py`) is still discovered. +_SEGMENT_SPLIT_RE = re.compile(r"&&|\|\||[;|()&]") + def _read_text(path: Path) -> str: try: @@ -175,33 +189,73 @@ def _verify_scripts(workspace: Path) -> list[Path]: return sorted(p for p in scripts.glob("verify-*") if p.is_file()) +def _leading_command(segment: str) -> str: + """The command word of a shell segment (past subshell/group openers).""" + stripped = segment.lstrip("({ \t") + parts = stripped.split(None, 1) + return parts[0] if parts else "" + + def _gate_invoked_in_verify(workspace: Path) -> bool: - """A verify-* script runs a holdout/anti-cheat gate on an executable line.""" + """A verify-* script genuinely *executes* a holdout/anti-cheat gate. + + Credit requires the gate script (`holdout_gate.py` / `anticheat_scan.py` / + `anti_cheat.py`) to be *invoked* as a command — not merely printed. A token + inside an `echo`/`printf`/`:` argument is inert output and earns nothing. + """ for script in _verify_scripts(workspace): for line in _read_text(script).splitlines(): stripped = line.strip() if not stripped or stripped.startswith("#"): continue - if any(token in stripped for token in _GATE_TOKENS): + for segment in _SEGMENT_SPLIT_RE.split(stripped): + segment = segment.strip() + if not segment or not _GATE_SCRIPT_RE.search(segment): + continue + if _leading_command(segment) in _INERT_EMITTERS: + continue return True return False +def _records_gate_run(low: str) -> bool: + """A gate token AND an independent run-word share one lowered line. + + The run-word is checked against the residue *after* the gate tokens are + removed, so a token that itself contains a run-word (``anticheat_scan`` ⊃ + ``scan``) cannot self-satisfy — a bare token earns nothing. + """ + if not any(token in low for token in _GATE_TOKENS): + return False + residue = low + for token in _GATE_TOKENS: + residue = residue.replace(token, " ") + return any(word in residue for word in _GATE_RUN_WORDS) + + +def _receipt_records_gate(line: str) -> bool: + """A receipt line is a real gate record: parseable JSON with gate+run fields.""" + line = line.strip() + if not line: + return False + try: + obj = json.loads(line) + except json.JSONDecodeError: + return False + return _records_gate_run(json.dumps(obj).lower()) + + def _gate_run_recorded(paths) -> bool: """RUNLOG.md / .loop/receipts/*.jsonl record an actual gate run.""" - texts = [_read_text(paths.runlog)] + for line in _read_text(paths.runlog).splitlines(): + if _records_gate_run(line.lower()): + return True receipts = paths.loop_dir / "receipts" if receipts.is_dir(): - texts.extend(_read_text(p) for p in sorted(receipts.glob("*.jsonl"))) - for text in texts: - for line in text.splitlines(): - low = line.lower() - if any(token in low for token in _GATE_TOKENS): - return True - if ("holdout" in low or "anticheat" in low or "anti-cheat" in low) and any( - word in low for word in _GATE_RUN_WORDS - ): - return True + for receipt in sorted(receipts.glob("*.jsonl")): + for line in _read_text(receipt).splitlines(): + if _receipt_records_gate(line): + return True return False diff --git a/scripts/test_inspect_loop.py b/scripts/test_inspect_loop.py index fffa433..bd4c7ae 100644 --- a/scripts/test_inspect_loop.py +++ b/scripts/test_inspect_loop.py @@ -440,3 +440,124 @@ def test_documented_cli_by_path_reads_dotloop_manifest(tmp_path): gap_text = " ".join(report["gaps"]).lower() assert "plan-then-execute" not in present_text, report assert "plan-then-execute" in gap_text, report + + +# --- M3: false-completion credit grades on execution evidence, not tokens ---- + + +def _verify_gate_loop(root: pathlib.Path, name: str, verify_body: str) -> pathlib.Path: + """A loop whose only signal is a single verify-safety script body.""" + d = root / name + (d / "scripts").mkdir(parents=True) + (d / "scripts" / "verify-safety").write_text( + "#!/bin/sh\n" + verify_body, encoding="utf-8" + ) + return d + + +def test_string_literal_gate_tokens_earn_no_invoked_credit(tmp_path): + # M3(a): a token inside echo/printf/':' args (or any quoted-string emit) is + # inert output, not an executed gate — it must earn ZERO invoked credit. + exploits = ( + 'echo "holdout_gate"\n', + "printf 'holdout_gate\\n'\n", + ": holdout_gate\n", + 'echo "run the holdout_gate please"\n', + 'echo "holdout_gate.py"\n', # even the script name, still just printed + ) + for i, body in enumerate(exploits): + loop = _verify_gate_loop(tmp_path, f"exploit{i}", body) + assert il._gate_invoked_in_verify(loop) is False, body + + +def test_real_gate_invocations_keep_invoked_credit(tmp_path): + # M3(a): the token as part of a genuinely invoked command keeps full credit. + real = ( + "python3 scripts/holdout_gate.py --strict\n", + "bash scripts/anticheat_scan.py\n", + "./scripts/holdout_gate.py\n", + "uv run python scripts/anticheat_scan.py\n", + # the flagship shape: the gate path is double-quoted, invoked by python3. + 'python3 "$REPO/scripts/holdout_gate.py" "$EX/target/manifest.json" --cwd "$EX/target"\n', + # chained after an echo — the real invocation must still count. + 'echo "gate:" && python3 scripts/holdout_gate.py\n', + ) + for i, body in enumerate(real): + loop = _verify_gate_loop(tmp_path, f"real{i}", body) + assert il._gate_invoked_in_verify(loop) is True, body + + +def test_bare_gate_token_in_runlog_earns_no_recorded_credit(tmp_path): + # M3(b): a bare gate token in RUNLOG prose is a self-narration, not a run — + # even when the token itself (anticheat_scan) contains a run-word ("scan"). + loop = tmp_path / "rl" + (loop / ".loop").mkdir(parents=True) + (loop / "RUNLOG.md").write_text( + "This loop uses holdout_gate and anticheat_scan for false-completion defense.\n", + encoding="utf-8", + ) + paths = il.resolve_loop_paths(loop) + assert il._gate_run_recorded(paths) is False + + +def test_recorded_gate_run_with_run_word_earns_credit(tmp_path): + # M3(b): a real recorded run — token AND an independent run-word on one line. + loop = tmp_path / "rl2" + (loop / ".loop").mkdir(parents=True) + (loop / "RUNLOG.md").write_text( + "gate: scripts/holdout_gate.py target/manifest.json -> verdict Succeeded\n", + encoding="utf-8", + ) + paths = il.resolve_loop_paths(loop) + assert il._gate_run_recorded(paths) is True + + +def test_receipts_jsonl_records_gate_run(tmp_path): + # M3(b): a structured receipt line is parsed as JSON and matched on fields. + loop = tmp_path / "rc" + (loop / ".loop" / "receipts").mkdir(parents=True) + (loop / ".loop" / "receipts" / "run.jsonl").write_text( + json.dumps({"event": "holdout_gate", "verdict": "Succeeded"}) + "\n", + encoding="utf-8", + ) + paths = il.resolve_loop_paths(loop) + assert il._gate_run_recorded(paths) is True + + +def test_prose_renamed_to_jsonl_earns_no_recorded_credit(tmp_path): + # M3(b): a non-JSON prose line stuffed into a .jsonl file is not a receipt. + loop = tmp_path / "rc2" + (loop / ".loop" / "receipts").mkdir(parents=True) + (loop / ".loop" / "receipts" / "run.jsonl").write_text( + "holdout_gate anticheat_scan verdict clean passed\n", encoding="utf-8" + ) + paths = il.resolve_loop_paths(loop) + assert il._gate_run_recorded(paths) is False + + +def test_stuffed_fake_without_gate_file_cannot_reach_wired(tmp_path): + # M3(c): file-exists + surface-reference is the wired bar; with NO gate script + # file, a stuffed contract that merely names the gate cannot reach "wired". + loop = tmp_path / "nofile" + (loop / "scripts").mkdir(parents=True) + (loop / "WORKFLOW.md").write_text( + "# WORKFLOW\nHigh-value tasks are gated by scripts/holdout_gate.py anti-cheat.\n", + encoding="utf-8", + ) + (loop / "SPEC.md").write_text( + "# SPEC\nholdout_gate anticheat_scan anti-cheat defense.\n", encoding="utf-8" + ) + paths = il.resolve_loop_paths(loop) + assert il._gate_script_referenced(paths) is False + assert il._false_completion_credit(paths) == "none" + + +def test_flagship_example_keeps_strong_gate_backed_verdict(): + # Regression guard: the genuinely gate-backed flagship must keep its verdict + # and its invoked false-completion credit — the discriminator must not dock + # the real, double-quoted `holdout_gate.py` invocation in its verify-full. + root = pathlib.Path(__file__).resolve().parent.parent + report = il.inspect_loop(str(root / "examples" / "coverage-repair")) + assert report["verdict"] == "strong" + assert report["score"] >= 80 + assert any("invoked" in signal for signal in report["present"]) From b088ac73c400ba2f80181d2889b1293445d2203b Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Sun, 5 Jul 2026 18:42:50 -0400 Subject: [PATCH 2/5] fix(inspect): deny defines_success credit to unfilled scaffold placeholders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fresh, unedited scaffold scored 86/100 "strong" — its success criteria and task titles are still the literal "REPLACE:" placeholders, so the inspector was crediting a shell as a defined success. A stranger's first `scaffold` then `inspect` wrongly read as a strong loop. The inspector now gates defines_success on the SPEC Success-criteria list being filled: a structurally-present-but-all-placeholder criteria list earns no credit and emits an actionable gap naming the "REPLACE:" convention. Placeholder TASKS titles are flagged the same way. Doctor is unchanged — a fresh scaffold is still a valid in-flight contract (doctor stays clean, a test-asserted invariant); only the inspector's *quality* verdict tightens. A fresh scaffold now lands at 74/ok with placeholder gaps; the flagship (examples/coverage-repair), which has real criteria, keeps 90/strong. Combined with the false-completion execution-evidence fix, the review's keyword-stuffed fake (stuffed prose + echo verify line + stuffed RUNLOG, no real gate) drops from 100/strong to 74/ok with a false-completion gap. Co-Authored-By: Claude Fable 5 --- scripts/inspect_loop.py | 74 +++++++++++++++++++- scripts/test_inspect_loop.py | 126 +++++++++++++++++++++++++++++++++++ scripts/test_scaffold.py | 32 +++++++++ 3 files changed, 231 insertions(+), 1 deletion(-) diff --git a/scripts/inspect_loop.py b/scripts/inspect_loop.py index ddc848f..e4cc154 100644 --- a/scripts/inspect_loop.py +++ b/scripts/inspect_loop.py @@ -165,6 +165,62 @@ def _task_verify_declared(tasks: dict) -> bool: return False +# Scaffold placeholder convention: an unfilled slot is the literal "REPLACE" +# marker (loop/scaffold.py `_substitutions`: "REPLACE: " for filled tokens, +# a bare "REPLACE" for the extra CRITERION_2/3 slots). +_PLACEHOLDER_MSG = ( + "unfilled scaffold placeholders ('REPLACE: ...') — replace them with " + "concrete, verifiable text before this loop can claim to define {what}" +) +_SECTION_HEADING_RE = re.compile(r"\s*#{1,6}\s+(?P.*\S)\s*$") +_LIST_ITEM_RE = re.compile(r"\s*(?:\d+[.)]|[-*+])\s+(?P<item>.*\S)\s*$") + + +def _is_placeholder(text: str) -> bool: + """True iff ``text`` is an unfilled scaffold slot ('REPLACE' / 'REPLACE:…').""" + upper = text.strip().upper() + return upper == "REPLACE" or upper.startswith("REPLACE:") + + +def _section_body(text: str, heading: str) -> str | None: + """Return the body of the first ``## <heading>`` section (case-insensitive).""" + lines = text.splitlines() + target = heading.strip().lower() + start = None + for i, line in enumerate(lines): + match = _SECTION_HEADING_RE.match(line) + if match and match.group("title").strip().lower().rstrip(":") == target: + start = i + 1 + break + if start is None: + return None + body: list[str] = [] + for line in lines[start:]: + if _SECTION_HEADING_RE.match(line): + break + body.append(line) + return "\n".join(body) + + +def _success_criteria_all_placeholder(spec_text: str) -> bool: + """True iff the SPEC's Success-criteria list has items and all are unfilled.""" + body = _section_body(spec_text, "success criteria") + if body is None: + return False + items = [m.group("item") for line in body.splitlines() if (m := _LIST_ITEM_RE.match(line))] + return bool(items) and all(_is_placeholder(item) for item in items) + + +def _task_titles_all_placeholder(tasks: dict) -> bool: + """True iff every declared task carries an unfilled 'REPLACE' title.""" + rows = tasks.get("tasks") + if not isinstance(rows, list) or not rows: + return False + titles = [row.get("title") for row in rows if isinstance(row, dict)] + titles = [t for t in titles if isinstance(t, str) and t.strip()] + return bool(titles) and all(_is_placeholder(title) for title in titles) + + def _terminal_states_covered_from_contract(loop: Path) -> int: """Count terminal taxonomy coverage from contract-owned files only.""" @@ -301,7 +357,15 @@ def _evaluate_contract_checks(loop: Path) -> dict[str, object]: policies = manifest.get("policies") if isinstance(manifest, dict) else None manifest_declares_plan = isinstance(policies, dict) and "plan_then_execute" in policies - has_spec_criteria = "success criteria" in spec or "success_criteria" in spec + # A fresh scaffold's Success-criteria list and task titles are the literal + # "REPLACE:" placeholders. A structurally-present-but-unfilled criteria list + # earns no defines_success credit — a shell is not a defined success. + spec_raw = _read_text(paths.spec) + "\n" + _read_text(paths.contract) + criteria_all_placeholder = _success_criteria_all_placeholder(spec_raw) + task_titles_placeholder = _task_titles_all_placeholder(tasks) + + has_criteria_heading = "success criteria" in spec or "success_criteria" in spec + has_spec_criteria = has_criteria_heading and not criteria_all_placeholder has_verify = ( _task_verify_declared(tasks) or _script_exists( @@ -332,6 +396,8 @@ def _evaluate_contract_checks(loop: Path) -> dict[str, object]: "approval_gates": has_approval, "false_completion_defense": _false_completion_credit(paths), "plan_then_execute": has_plan_then_execute, + "_success_criteria_placeholder": has_criteria_heading and criteria_all_placeholder, + "_task_titles_placeholder": task_titles_placeholder, } @@ -385,12 +451,18 @@ def inspect_loop(loop_dir: str) -> dict: if key == "false_completion_defense": score += _grade_false_completion(value, weight, label, gap_msg, present, gaps) continue + if key == "defines_success" and not value and results.get("_success_criteria_placeholder"): + gaps.append("success criteria are " + _PLACEHOLDER_MSG.format(what="success")) + continue if value: score += weight present.append(label) else: gaps.append(gap_msg) + if results.get("_task_titles_placeholder"): + gaps.append("TASKS.json titles are " + _PLACEHOLDER_MSG.format(what="its tasks")) + terminal_points = round(_TERMINAL_WEIGHT * covered / len(TERMINAL_STATES)) score += terminal_points if covered == len(TERMINAL_STATES): diff --git a/scripts/test_inspect_loop.py b/scripts/test_inspect_loop.py index bd4c7ae..166ee04 100644 --- a/scripts/test_inspect_loop.py +++ b/scripts/test_inspect_loop.py @@ -561,3 +561,129 @@ def test_flagship_example_keeps_strong_gate_backed_verdict(): assert report["verdict"] == "strong" assert report["score"] >= 80 assert any("invoked" in signal for signal in report["present"]) + + +# --- M2: unfilled scaffold placeholders earn no defines_success credit -------- + + +def test_placeholder_success_criteria_earn_no_defines_success_credit(tmp_path): + loop = tmp_path / "ph" + loop.mkdir() + (loop / "SPEC.md").write_text( + "# SPEC\n## Success criteria\n" + "1. REPLACE: first success criterion\n2. REPLACE\n3. REPLACE\n", + encoding="utf-8", + ) + report = il.inspect_loop(str(loop)) + present = " ".join(report["present"]).lower() + gaps = " ".join(report["gaps"]).lower() + assert "defines verifiable success criteria" not in present + # The gap names the scaffold placeholder convention actionably. + assert "replace" in gaps and "placeholder" in gaps + + +def test_real_success_criteria_still_earn_defines_success_credit(tmp_path): + loop = tmp_path / "real" + loop.mkdir() + (loop / "SPEC.md").write_text( + "# SPEC\n## Success criteria\n1. coverage >= 80% (scripts/verify-full)\n", + encoding="utf-8", + ) + report = il.inspect_loop(str(loop)) + present = " ".join(report["present"]).lower() + assert "defines verifiable success criteria" in present + + +def test_one_filled_criterion_among_placeholders_still_earns_credit(tmp_path): + # Partial fill is real intent — a single concrete criterion keeps the credit. + loop = tmp_path / "partial" + loop.mkdir() + (loop / "SPEC.md").write_text( + "# SPEC\n## Success criteria\n" + "1. coverage >= 80% (scripts/verify-full)\n2. REPLACE\n", + encoding="utf-8", + ) + report = il.inspect_loop(str(loop)) + present = " ".join(report["present"]).lower() + assert "defines verifiable success criteria" in present + + +def test_placeholder_task_titles_flagged_as_gap(tmp_path): + loop = tmp_path / "pht" + loop.mkdir() + (loop / "TASKS.json").write_text( + json.dumps({"tasks": [{"id": "T1", "title": "REPLACE: first task", + "verify": "scripts/verify-fast"}]}), + encoding="utf-8", + ) + report = il.inspect_loop(str(loop)) + gaps = " ".join(report["gaps"]).lower() + assert "task" in gaps and "placeholder" in gaps + + +def test_fresh_scaffold_is_not_strong_with_placeholder_gaps(tmp_path): + # M2: an unedited scaffold scored 86/strong; it must now land at ok-or-below + # and name the placeholder convention so a stranger knows what to fill. + from loop.scaffold import scaffold + + target = tmp_path / "fresh-scaffold" + scaffold(target) + report = il.inspect_loop(str(target)) + gaps = " ".join(report["gaps"]).lower() + + assert report["verdict"] != "strong" + assert report["score"] < 80 + assert "replace" in gaps and "placeholder" in gaps + + +def test_keyword_stuffed_fake_scores_below_strong_with_false_completion_gap(tmp_path): + # Acceptance (M2 + M3): the review's keyword-stuffed fake — SPEC/WORKFLOW + # prose stuffed with checklist vocabulary, all 7 terminal states, an echo + # "verify" line, a stuffed RUNLOG, and NO real gate files. It scored + # 100/strong before the fixes; it must now land materially below strong with + # a false-completion gap (the echo trick and stuffed prose buy no defense + # credit — M3 — and the unfilled success criteria earn no defines_success + # credit — M2). + fake = tmp_path / "fake" + (fake / ".loop").mkdir(parents=True) + (fake / "scripts").mkdir() + (fake / "SPEC.md").write_text( + "# SPEC — totally-done\n" + "This loop has verifiable success, independent verification, approval " + "gates, false-completion defense, held-out anti-cheat, plan-then-execute.\n" + "## Success criteria\n" + "1. REPLACE: first success criterion\n" + "2. REPLACE\n", + encoding="utf-8", + ) + (fake / "WORKFLOW.md").write_text( + "# WORKFLOW\n## Approval Gates\nApproval gate on side-effects.\n" + "## Plan-then-execute\nPlan-then-execute for untrusted reads.\n" + "## Anti-cheat\nfalse-completion defense via held-out holdout gate anti-cheat.\n" + "## Terminal States\n" + "Succeeded, FailedUnverifiable, FailedBlocked, FailedBudget, " + "FailedSafety, FailedSpecGap, AbortedByHuman.\n", + encoding="utf-8", + ) + (fake / "TASKS.json").write_text( + json.dumps({"tasks": [{"id": "T1", "title": "REPLACE: first task", + "verify": "scripts/verify-fast"}]}), + encoding="utf-8", + ) + (fake / "scripts" / "verify-fast").write_text( + '#!/bin/sh\necho "holdout_gate anticheat_scan all clean"\n', encoding="utf-8" + ) + (fake / "RUNLOG.md").write_text( + "Iteration 1: holdout gate anti-cheat false-completion defense engaged. " + "Everything looks done and fine.\n", + encoding="utf-8", + ) + + report = il.inspect_loop(str(fake)) + present = " ".join(report["present"]).lower() + gaps = " ".join(report["gaps"]).lower() + + assert "false-completion defense" not in present + assert "false-completion" in gaps or "false completion" in gaps + assert report["verdict"] != "strong" + assert report["score"] < 80 diff --git a/scripts/test_scaffold.py b/scripts/test_scaffold.py index 8bbd4ac..6278b44 100644 --- a/scripts/test_scaffold.py +++ b/scripts/test_scaffold.py @@ -7,6 +7,7 @@ from __future__ import annotations +import importlib.util import json import pathlib import subprocess @@ -16,6 +17,12 @@ ROOT = pathlib.Path(__file__).resolve().parent.parent +_il_spec = importlib.util.spec_from_file_location( + "il", pathlib.Path(__file__).parent / "inspect_loop.py" +) +il = importlib.util.module_from_spec(_il_spec) +_il_spec.loader.exec_module(il) + def test_scaffolded_contract_passes_doctor_unedited(tmp_path): from loop.contract import validate_contract @@ -127,6 +134,31 @@ def test_missing_terminal_is_an_issue_when_state_declares_a_terminal(tmp_path): ) +def test_fresh_scaffold_is_doctor_clean_but_inspects_below_strong(tmp_path): + """The scaffold is a valid in-flight contract (doctor-clean) yet a fresh, + unedited one is not a *strong* loop: its success criteria and task titles are + still the literal `REPLACE:` placeholders. Doctor stays green; the inspector + must dock defines_success and flag the placeholder convention so a stranger + knows the loop is a shell to fill, not a finished proof.""" + from loop.contract import validate_contract + from loop.scaffold import scaffold + + target = tmp_path / "fresh" + scaffold(target) + + # Doctor is unchanged — a fresh scaffold is a valid in-flight contract. + assert validate_contract(target)["ok"] is True + + report = il.inspect_loop(str(target)) + gaps = " ".join(report["gaps"]).lower() + present = " ".join(report["present"]).lower() + + assert report["verdict"] != "strong" + assert report["score"] < 80 + assert "defines verifiable success criteria" not in present + assert "replace" in gaps and "placeholder" in gaps + + def test_scaffold_cli_subcommand(tmp_path): target = tmp_path / "cli-loop" result = subprocess.run( From 7443f741cce06ab4b0d356a3c2bc937e453a0846 Mon Sep 17 00:00:00 2001 From: Sollan Systems <khall0239@gmail.com> Date: Sun, 5 Jul 2026 19:07:33 -0400 Subject: [PATCH 3/5] fix(inspect): allowlist gate invocation + cap defense-less loops below strong MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Close four inspector-honesty holes the adversarial verifier flagged: 1. Genuine-invocation ALLOWLIST for false-completion "invoked" credit (_gate_invoked_in_verify): credit only when an interpreter (python/python3/python3.N/uv/bash/sh/exec) runs a gate script named in its args, or the gate script is invoked directly by path. grep/cat/ls/test/head/ wc/find/echo/printf that merely reference the .py now earn ZERO invoked credit — the old inert-emitter denylist let all of them evade. 2. Score cap: when false-completion defense grades "none", the total is capped at 79 with an explicit gap, so "strong" (>=80) and a fail-under-80 CI gate are unreachable by pure keyword stuffing. "wired"/"invoked" grades are uncapped. 3. independent_verification substance: a verify-* script earns credit only if it has >=1 non-inert executable line — a file merely named verify-fast whose body is echo/printf/no-ops is not verification. The shipped scaffold's for-loop existence check keeps its credit. 4. Adversarial regressions: grep/cat/ls/test/head/wc/find evasions, the grep-swapped M3 fake, the stuffed fake crossing 80, echo-only verify bodies, and the allowlist true positives. Flagship (examples/coverage-repair) holds 90/strong; fresh scaffold holds 74/ok and stays doctor-clean. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- scripts/inspect_loop.py | 116 +++++++++++++++++---- scripts/test_inspect_loop.py | 196 +++++++++++++++++++++++++++++++++++ 2 files changed, 291 insertions(+), 21 deletions(-) diff --git a/scripts/inspect_loop.py b/scripts/inspect_loop.py index e4cc154..b3f2442 100644 --- a/scripts/inspect_loop.py +++ b/scripts/inspect_loop.py @@ -17,11 +17,15 @@ assertions the loop makes about itself. The three grades are: * **invoked** (full credit) — a ``scripts/verify-*`` gate *executes* a holdout - / anti-cheat gate script (``python3 scripts/holdout_gate.py``, not an - ``echo "holdout_gate"`` that only prints the token), OR ``RUNLOG.md`` / - ``.loop/receipts/*.jsonl`` records an actual run (a gate token AND a run-word - on one line — a ``holdout_gate`` verdict / anti-cheat scan result — not a - bare token in stuffed prose). + / anti-cheat gate script: an interpreter (``python``/``python3``/``python3.N``/ + ``uv``/``bash``/``sh``/``exec``) runs a gate script named in its arguments + (``python3 scripts/holdout_gate.py``), or the gate script is invoked directly + by path (``./scripts/holdout_gate.py``). A command that only *references* the + file — ``echo``/``printf`` printing it, or ``grep``/``cat``/``ls``/``test``/ + ``head``/``wc``/``find`` reading, listing, or searching it — earns *nothing*. + OR ``RUNLOG.md`` / ``.loop/receipts/*.jsonl`` records an actual run (a gate + token AND a run-word on one line — a ``holdout_gate`` verdict / anti-cheat + scan result — not a bare token in stuffed prose). * **wired** (partial credit, half the weight) — a gate script file exists (``scripts/holdout_gate.py`` / ``anticheat_scan.py`` / ``anti_cheat.py``) and is referenced from the contract's verify surface (SPEC / WORKFLOW / @@ -44,6 +48,7 @@ import json import re +import shlex import sys from pathlib import Path @@ -125,12 +130,19 @@ class _Paths: # gate uses (`python3 scripts/holdout_gate.py`, `./scripts/anticheat_scan.py`). # The bare token ("holdout_gate") without .py is prose, not an invocation. _GATE_SCRIPT_RE = re.compile(r"\b(?:holdout_gate|anticheat_scan|anti_cheat)\.py\b") -# Commands whose arguments are inert output, not executed gates: a token inside -# `echo`/`printf`/`:` (or the `true`/`false` no-ops) is printed, never run. -_INERT_EMITTERS = frozenset({"echo", "printf", ":", "true", "false"}) +# Genuine-invocation ALLOWLIST: "invoked" credit requires an executable line that +# actually *runs* a gate script. Either an interpreter leads the command and a +# gate script is named in its arguments, or the gate script is invoked directly +# by path. Everything else — a reference via echo/printf/grep/cat/ls/test/head/ +# wc/find, or any other non-interpreter command — earns nothing. +_GATE_INTERPRETERS = frozenset({"python", "python3", "uv", "bash", "sh", "exec"}) +_VERSIONED_PYTHON_RE = re.compile(r"^python3\.\d+$") # Split a shell line into command segments so a real invocation chained after an # inert emitter (`echo x && python3 ...holdout_gate.py`) is still discovered. _SEGMENT_SPLIT_RE = re.compile(r"&&|\|\||[;|()&]") +# A verify-* line is "inert" (no verification substance) when its leading command +# only prints or no-ops. A body of nothing but these plus comments is not proof. +_INERT_LINE_COMMANDS = frozenset({"echo", "printf", "exit", "true", "false", ":"}) def _read_text(path: Path) -> str: @@ -252,12 +264,50 @@ def _leading_command(segment: str) -> str: return parts[0] if parts else "" +def _shell_tokens(segment: str) -> list[str]: + """Tokenize a shell segment (subshell openers stripped, quotes removed).""" + seg = segment.lstrip("({ \t") + try: + return shlex.split(seg, posix=True) + except ValueError: + return seg.split() + + +def _basename(token: str) -> str: + """The trailing path component of a shell token (posix `/` separator).""" + return token.rsplit("/", 1)[-1] + + +def _is_gate_interpreter(token: str) -> bool: + return token in _GATE_INTERPRETERS or bool(_VERSIONED_PYTHON_RE.match(token)) + + +def _segment_runs_gate(segment: str) -> bool: + """True iff this shell segment genuinely *executes* a gate script. + + Allowlisted shapes only: an interpreter (python/python3/python3.N/uv/bash/sh/ + exec) whose arguments name a gate script, or the gate script invoked directly + by path (`./scripts/holdout_gate.py`). A leading grep/cat/ls/test/head/wc/find + — or echo/printf — that merely references the file is not an execution. + """ + tokens = _shell_tokens(segment) + if not tokens: + return False + lead, args = tokens[0], tokens[1:] + if _basename(lead) in _GATE_SCRIPTS: + return True + if _is_gate_interpreter(lead): + return any(_basename(arg) in _GATE_SCRIPTS for arg in args) + return False + + def _gate_invoked_in_verify(workspace: Path) -> bool: """A verify-* script genuinely *executes* a holdout/anti-cheat gate. Credit requires the gate script (`holdout_gate.py` / `anticheat_scan.py` / - `anti_cheat.py`) to be *invoked* as a command — not merely printed. A token - inside an `echo`/`printf`/`:` argument is inert output and earns nothing. + `anti_cheat.py`) to be *run* — an interpreter invocation or a direct + by-path call. A command that only prints, reads, lists, or searches the file + (echo/printf/grep/cat/ls/test/head/wc/find) earns nothing. """ for script in _verify_scripts(workspace): for line in _read_text(script).splitlines(): @@ -268,8 +318,26 @@ def _gate_invoked_in_verify(workspace: Path) -> bool: segment = segment.strip() if not segment or not _GATE_SCRIPT_RE.search(segment): continue - if _leading_command(segment) in _INERT_EMITTERS: - continue + if _segment_runs_gate(segment): + return True + return False + + +def _verify_script_has_substance(workspace: Path) -> bool: + """A verify-* script exists whose body has ≥1 non-inert executable line. + + A body of nothing but comments and printing/no-op commands (echo/printf/exit/ + true/false/`:`) is not verification — a file merely *named* ``verify-fast`` + earns no independent-verification credit. The shipped scaffold's verify-fast + keeps credit: its contract-file existence ``for``/``if`` loop is substantive. + """ + for script in _verify_scripts(workspace): + for line in _read_text(script).splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + lead = _leading_command(stripped) + if lead and lead not in _INERT_LINE_COMMANDS: return True return False @@ -366,17 +434,11 @@ def _evaluate_contract_checks(loop: Path) -> dict[str, object]: has_criteria_heading = "success criteria" in spec or "success_criteria" in spec has_spec_criteria = has_criteria_heading and not criteria_all_placeholder + # A verify-* script earns credit only if it actually verifies — a body of + # nothing but echo/printf/no-ops (a file merely *named* verify-fast) does not. has_verify = ( _task_verify_declared(tasks) - or _script_exists( - paths.workspace, - "verify-fast", - "verify-fast.sh", - "verify-full", - "verify-full.sh", - "verify-safety", - "verify-safety.sh", - ) + or _verify_script_has_substance(paths.workspace) or "scripts/verify" in spec ) has_approval = ( @@ -486,6 +548,18 @@ def inspect_loop(loop_dir: str) -> dict: ) score = max(0, min(100, score)) + + # False-completion defense is the anti-gaming keystone: a loop with NO real + # holdout/anti-cheat gate (grade "none") must never reach "strong" or clear a + # fail-under-80 CI gate on keyword stuffing alone. Cap it below the threshold. + # "wired"/"invoked" grades — a gate that at least exists — are uncapped. + if results["false_completion_defense"] == "none" and score >= 80: + score = 79 + gaps.append( + "no false-completion defense — score capped below 'strong'; " + "wire a holdout/anti-cheat gate" + ) + return { "target": str(loop), "score": score, diff --git a/scripts/test_inspect_loop.py b/scripts/test_inspect_loop.py index 166ee04..ccaf8b8 100644 --- a/scripts/test_inspect_loop.py +++ b/scripts/test_inspect_loop.py @@ -687,3 +687,199 @@ def test_keyword_stuffed_fake_scores_below_strong_with_false_completion_gap(tmp_ assert "false-completion" in gaps or "false completion" in gaps assert report["verdict"] != "strong" assert report["score"] < 80 + + +# --- must-fix #1 (allowlist): only a genuine interpreter/script invocation of a +# --- gate .py earns "invoked" credit. grep/cat/ls/test/head/wc/find evade. ----- + +_GATE_EVASION_COMMANDS = ("grep", "cat", "ls", "test -f", "head", "wc -l", "find") + + +def test_non_executing_commands_referencing_gate_py_earn_no_invoked_credit(tmp_path): + # must-fix #1 / D(i): grep/cat/ls/test/head/wc/find that merely *reference* the + # gate .py (read/list/search it) never *execute* it — ZERO invoked credit. + # A real gate file is present so the only thing under test is the invocation + # shape: a non-executing reference must not upgrade to "invoked". + for i, cmd in enumerate(_GATE_EVASION_COMMANDS): + loop = tmp_path / f"evade{i}" + (loop / "scripts").mkdir(parents=True) + (loop / "scripts" / "holdout_gate.py").write_text("# gate\n", encoding="utf-8") + (loop / "scripts" / "verify-fast").write_text( + f"#!/bin/sh\n{cmd} scripts/holdout_gate.py\n", encoding="utf-8" + ) + # Unit: the invocation detector rejects the non-executing reference. + assert il._gate_invoked_in_verify(loop) is False, cmd + # Report: no "(invoked)" credit is shown for the false-completion defense. + report = il.inspect_loop(str(loop)) + present = " ".join(report["present"]).lower() + assert "(invoked)" not in present, cmd + + +def test_grep_evasion_literal_m3_fake_scores_below_strong(tmp_path): + # must-fix #1 acceptance: the literal M3 fake with `echo` swapped for `grep` + # (no real gate file, only a grep reference in verify-fast) scored 88/strong + # before the fix. grep earns no invoked credit and, with no gate file, the + # defense grades "none" → the score cap holds it below strong. + fake = tmp_path / "grepfake" + (fake / ".loop").mkdir(parents=True) + (fake / "scripts").mkdir() + (fake / "SPEC.md").write_text( + "# SPEC\n## Success criteria\n1. coverage >= 80% (scripts/verify-full)\n", + encoding="utf-8", + ) + (fake / "WORKFLOW.md").write_text( + "# WORKFLOW\n## Approval Gates\nApproval gate on side-effects.\n" + "## Plan-then-execute\nPlan-then-execute for untrusted reads.\n" + "## Terminal States\n" + "Succeeded, FailedUnverifiable, FailedBlocked, FailedBudget, " + "FailedSafety, FailedSpecGap, AbortedByHuman.\n", + encoding="utf-8", + ) + (fake / "TASKS.json").write_text( + json.dumps({"tasks": [{"id": "T1", "title": "ship it", + "verify": "scripts/verify-fast"}]}), + encoding="utf-8", + ) + (fake / "scripts" / "verify-fast").write_text( + '#!/bin/sh\ngrep "holdout_gate.py anticheat_scan" ./SPEC.md\n', + encoding="utf-8", + ) + + report = il.inspect_loop(str(fake)) + present = " ".join(report["present"]).lower() + assert "(invoked)" not in present + assert report["verdict"] != "strong" + assert report["score"] < 80 + + +def test_allowlisted_invocations_keep_invoked_credit(tmp_path): + # must-fix #1 / D(iii): the allowlisted true positives keep FULL credit. + invocations = ( + "python3 scripts/holdout_gate.py\n", + "bash scripts/anticheat_scan.py\n", + "./scripts/holdout_gate.py\n", + "uv run python scripts/anticheat_scan.py\n", + ) + for i, body in enumerate(invocations): + loop = tmp_path / f"real{i}" + (loop / "scripts").mkdir(parents=True) + (loop / "scripts" / "verify-fast").write_text( + "#!/bin/sh\n" + body, encoding="utf-8" + ) + assert il._gate_invoked_in_verify(loop) is True, body + + +def test_versioned_python_interpreter_invocation_counts(tmp_path): + # must-fix #1: `python3.12 scripts/holdout_gate.py` is a genuine invocation. + loop = tmp_path / "pyver" + (loop / "scripts").mkdir(parents=True) + (loop / "scripts" / "verify-fast").write_text( + "#!/bin/sh\npython3.12 scripts/holdout_gate.py --strict\n", encoding="utf-8" + ) + assert il._gate_invoked_in_verify(loop) is True + + +# --- must-fix #2 (score cap): "strong" is unreachable without a real defense --- + + +def test_stuffed_fake_with_no_defense_cannot_reach_strong(tmp_path): + # must-fix #2 / D(ii): a stuffed fake — real-looking success criteria, an + # echo-only verify-fast, prose approval/plan headings, all 7 terminal states, + # but NO real gate files — must score < 80 and verdict != "strong". Pure + # keyword stuffing with zero false-completion defense cannot buy a strong + # verdict (or clear the fail-under-80 CI gate). + fake = tmp_path / "stuffed" + (fake / ".loop").mkdir(parents=True) + (fake / "scripts").mkdir() + (fake / "SPEC.md").write_text( + "# SPEC\n## Success criteria\n" + "1. unit + integration coverage >= 80% (scripts/verify-full)\n" + "2. zero lint errors (scripts/verify-fast)\n" + "## Evidence Rules\nEach criterion maps to a scripts/verify-* command.\n", + encoding="utf-8", + ) + (fake / "WORKFLOW.md").write_text( + "# WORKFLOW\n## Approval Gates\nPause on destructive / secret / production.\n" + "## Plan-then-execute\nPrecommit the execution graph for untrusted reads.\n" + "## Terminal States\n" + "Succeeded, FailedUnverifiable, FailedBlocked, FailedBudget, " + "FailedSafety, FailedSpecGap, AbortedByHuman.\n", + encoding="utf-8", + ) + (fake / "scripts" / "verify-fast").write_text( + '#!/bin/sh\necho "holdout_gate anticheat_scan all clean"\n', encoding="utf-8" + ) + + report = il.inspect_loop(str(fake)) + present = " ".join(report["present"]).lower() + gaps = " ".join(report["gaps"]).lower() + + assert report["verdict"] != "strong" + assert report["score"] < 80 + # The false-completion defense grades "none" (no invoked/wired gate). + assert "false-completion defense" not in present + # The cap gap explicitly names the reason strong is out of reach. + assert "capped" in gaps and "false-completion defense" in gaps + + +def test_score_cap_only_bites_when_defense_is_none(tmp_path): + # must-fix #2: the cap targets grade "none" only. A genuinely gate-backed loop + # ("invoked") is never capped — the flagship keeps its full score. + root = pathlib.Path(__file__).resolve().parent.parent + report = il.inspect_loop(str(root / "examples" / "coverage-repair")) + gaps = " ".join(report["gaps"]).lower() + assert report["verdict"] == "strong" + assert report["score"] >= 80 + assert "capped" not in gaps + + +# --- must-fix #3 (verify substance): an echo-only verify-* body is not proof ---- + + +def test_echo_only_verify_script_earns_no_independent_verification(tmp_path): + # must-fix #3 / failure #3: a file merely NAMED scripts/verify-fast whose whole + # body is echo earns ZERO independent_verification credit — printing is not + # verifying. (No resolving task.verify, no SPEC verify reference.) + loop = tmp_path / "echoverify" + (loop / "scripts").mkdir(parents=True) + (loop / "scripts" / "verify-fast").write_text( + '#!/bin/sh\necho "checking..."\necho "all good"\nexit 0\n', encoding="utf-8" + ) + assert il._verify_script_has_substance(loop.resolve()) is False + report = il.inspect_loop(str(loop)) + present = " ".join(report["present"]).lower() + gaps = " ".join(report["gaps"]).lower() + assert "independent verification" not in present + assert "no independent verification" in gaps + + +def test_substantive_verify_script_earns_independent_verification(tmp_path): + # must-fix #3: a verify-* script with a real executable check (an existence + # loop) earns the credit — the substance gate is not a blanket denial. + loop = tmp_path / "realverify" + (loop / "scripts").mkdir(parents=True) + (loop / "scripts" / "verify-fast").write_text( + "#!/bin/sh\n" + 'for f in SPEC.md WORKFLOW.md; do\n' + ' [ -f "$1/$f" ] || exit 1\n' + "done\n" + 'echo "PASS"\n', + encoding="utf-8", + ) + assert il._verify_script_has_substance(loop.resolve()) is True + report = il.inspect_loop(str(loop)) + present = " ".join(report["present"]).lower() + assert "independent verification" in present + + +def test_fresh_scaffold_keeps_independent_verification_credit(tmp_path): + # must-fix #3 invariant: the shipped scaffold's verify-fast has a real for-loop + # existence check, so a fresh scaffold KEEPS independent_verification credit. + from loop.scaffold import scaffold + + target = tmp_path / "fresh-scaffold-verify" + scaffold(target) + assert il._verify_script_has_substance(target.resolve()) is True + report = il.inspect_loop(str(target)) + present = " ".join(report["present"]).lower() + assert "independent verification" in present From 0a590a9e43821bbdd6beb079730a02a7736947b4 Mon Sep 17 00:00:00 2001 From: Sollan Systems <khall0239@gmail.com> Date: Tue, 7 Jul 2026 20:31:59 -0400 Subject: [PATCH 4/5] =?UTF-8?q?fix(inspect):=20close=20re-verify=20round-2?= =?UTF-8?q?=20laundering=20=E2=80=94=20comments,=20redirects,=20uv=20subco?= =?UTF-8?q?mmands,=20ghost=20gates,=20prose=20run-words?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Blockers from the round-2 adversarial verify: - a gate named only in a trailing shell comment, as a redirection sink, or in a uv non-run subcommand (pip install/add) no longer earns invoked credit - invoked credit for a workspace-relative gate path now requires the gate file to exist on disk (full credit no longer has a weaker precondition than the half-credit wired grade); $VAR/absolute paths keep shape-only credit - RUNLOG credit requires the gate .py path plus a verdict word; ordinary English run-words (ran/result/passed-as-prose/clean) removed from the set - honest wrapper prefixes (env/time/nohup/nice/command) stay transparent --- scripts/inspect_loop.py | 122 +++++++++++++++++++++++-------- scripts/test_inspect_loop.py | 135 ++++++++++++++++++++++++++++++++++- 2 files changed, 226 insertions(+), 31 deletions(-) diff --git a/scripts/inspect_loop.py b/scripts/inspect_loop.py index b3f2442..71bd7a4 100644 --- a/scripts/inspect_loop.py +++ b/scripts/inspect_loop.py @@ -18,14 +18,17 @@ * **invoked** (full credit) — a ``scripts/verify-*`` gate *executes* a holdout / anti-cheat gate script: an interpreter (``python``/``python3``/``python3.N``/ - ``uv``/``bash``/``sh``/``exec``) runs a gate script named in its arguments + ``uv run``/``bash``/``sh``/``exec``, optionally behind a transparent wrapper + like ``env``/``time``/``nohup``) runs a gate script named in its arguments (``python3 scripts/holdout_gate.py``), or the gate script is invoked directly by path (``./scripts/holdout_gate.py``). A command that only *references* the - file — ``echo``/``printf`` printing it, or ``grep``/``cat``/``ls``/``test``/ - ``head``/``wc``/``find`` reading, listing, or searching it — earns *nothing*. - OR ``RUNLOG.md`` / ``.loop/receipts/*.jsonl`` records an actual run (a gate - token AND a run-word on one line — a ``holdout_gate`` verdict / anti-cheat - scan result — not a bare token in stuffed prose). + file — ``echo``/``printf`` printing it, ``grep``/``cat``/``ls``/``test``/ + ``head``/``wc``/``find`` reading it, a trailing ``# comment`` naming it, or a + redirection using it as a sink — earns *nothing*. A workspace-relative gate + path must exist on disk; only unresolvable paths (``$VAR``/absolute) earn + shape-only credit. OR ``RUNLOG.md`` / ``.loop/receipts/*.jsonl`` records an + actual run (the gate ``.py`` path AND a verdict word on one line — not a bare + token in stuffed prose; ordinary English like "ran"/"result" never counts). * **wired** (partial credit, half the weight) — a gate script file exists (``scripts/holdout_gate.py`` / ``anticheat_scan.py`` / ``anti_cheat.py``) and is referenced from the contract's verify surface (SPEC / WORKFLOW / @@ -123,7 +126,7 @@ class _Paths: # recorded run from mere prose ("anti-cheat", "false-completion"). _GATE_TOKENS = ("holdout_gate", "anticheat_scan", "anti_cheat") _GATE_SCRIPTS = ("holdout_gate.py", "anticheat_scan.py", "anti_cheat.py") -_GATE_RUN_WORDS = ("verdict", "scan", "result", "passed", "failed", "ran", "clean", "flagged") +_GATE_RUN_WORDS = ("verdict", "pass", "fail", "flagged", "clean", "exit 0") _FALSE_COMPLETION_PARTIAL_DIVISOR = 2 # wired-but-unrun earns half the weight # A gate script referenced as a *.py path — the invocation shape a real verify @@ -134,9 +137,15 @@ class _Paths: # actually *runs* a gate script. Either an interpreter leads the command and a # gate script is named in its arguments, or the gate script is invoked directly # by path. Everything else — a reference via echo/printf/grep/cat/ls/test/head/ -# wc/find, or any other non-interpreter command — earns nothing. -_GATE_INTERPRETERS = frozenset({"python", "python3", "uv", "bash", "sh", "exec"}) +# wc/find, or any other non-interpreter command — earns nothing. `uv` executes +# only through its `run` subcommand (pip install / add merely reference). +_GATE_INTERPRETERS = frozenset({"python", "python3", "bash", "sh", "exec"}) _VERSIONED_PYTHON_RE = re.compile(r"^python3\.\d+$") +# Wrapper commands that transparently run their argument command. +_TRANSPARENT_PREFIXES = frozenset({"env", "time", "nohup", "nice", "command"}) +# A token that is (or opens) a shell redirection: everything after it is a file +# operand, not part of the executed command (`python3 x.py > holdout_gate.py`). +_REDIRECTION_RE = re.compile(r"^\d*(>>?|<)") # Split a shell line into command segments so a real invocation chained after an # inert emitter (`echo x && python3 ...holdout_gate.py`) is still discovered. _SEGMENT_SPLIT_RE = re.compile(r"&&|\|\||[;|()&]") @@ -265,12 +274,21 @@ def _leading_command(segment: str) -> str: def _shell_tokens(segment: str) -> list[str]: - """Tokenize a shell segment (subshell openers stripped, quotes removed).""" + """Tokenize a shell segment (subshell openers stripped, quotes removed). + + Unquoted ``#`` starts a comment — everything after it is prose, not command + content, so a gate named only in a trailing comment never tokenizes. + """ seg = segment.lstrip("({ \t") try: - return shlex.split(seg, posix=True) + return shlex.split(seg, posix=True, comments=True) except ValueError: - return seg.split() + tokens: list[str] = [] + for tok in seg.split(): + if tok.startswith("#"): + break + tokens.append(tok) + return tokens def _basename(token: str) -> str: @@ -282,23 +300,63 @@ def _is_gate_interpreter(token: str) -> bool: return token in _GATE_INTERPRETERS or bool(_VERSIONED_PYTHON_RE.match(token)) -def _segment_runs_gate(segment: str) -> bool: +def _statically_resolvable(token: str) -> bool: + """A gate path we can check on disk: workspace-relative, no expansion.""" + return not (token.startswith(("/", "~")) or "$" in token or "`" in token) + + +def _gate_on_disk(workspace: Path, token: str) -> bool: + rel = token[2:] if token.startswith("./") else token + if (workspace / rel).is_file(): + return True + return "/" not in rel and (workspace / "scripts" / rel).is_file() + + +def _gate_arg_credits(workspace: Path, token: str) -> bool: + """A token earns gate credit: names a gate script that plausibly exists. + + A workspace-relative path must exist on disk — an invocation SHAPE of a + non-existent gate is stolen valor ("invoked" full credit must not have a + weaker precondition than "wired" half credit, which checks existence). + Unresolvable paths ($VAR / absolute / backticks) keep shape-only credit: + the flagship's gate lives outside the example workspace behind ``$REPO``. + """ + if _basename(token) not in _GATE_SCRIPTS: + return False + if not _statically_resolvable(token): + return True + return _gate_on_disk(workspace, token) + + +def _segment_runs_gate(segment: str, workspace: Path) -> bool: """True iff this shell segment genuinely *executes* a gate script. - Allowlisted shapes only: an interpreter (python/python3/python3.N/uv/bash/sh/ - exec) whose arguments name a gate script, or the gate script invoked directly - by path (`./scripts/holdout_gate.py`). A leading grep/cat/ls/test/head/wc/find - — or echo/printf — that merely references the file is not an execution. + Allowlisted shapes only: an interpreter (python/python3/python3.N/uv run/ + bash/sh/exec, optionally behind env/time/nohup/nice/command) whose arguments + name a gate script, or the gate script invoked directly by path + (`./scripts/holdout_gate.py`). A leading grep/cat/ls/test/head/wc/find — or + echo/printf — that merely references the file is not an execution; neither + is a gate path that appears only as a redirection sink. """ tokens = _shell_tokens(segment) + for i, tok in enumerate(tokens): + if _REDIRECTION_RE.match(tok): + tokens = tokens[:i] + break + while tokens and tokens[0] in _TRANSPARENT_PREFIXES: + tokens = tokens[1:] if not tokens: return False lead, args = tokens[0], tokens[1:] if _basename(lead) in _GATE_SCRIPTS: - return True - if _is_gate_interpreter(lead): - return any(_basename(arg) in _GATE_SCRIPTS for arg in args) - return False + return _gate_arg_credits(workspace, lead) + if lead == "uv": + if not args or args[0] != "run": + return False + args = args[1:] + elif not _is_gate_interpreter(lead): + return False + return any(_gate_arg_credits(workspace, arg) for arg in args) def _gate_invoked_in_verify(workspace: Path) -> bool: @@ -318,7 +376,7 @@ def _gate_invoked_in_verify(workspace: Path) -> bool: segment = segment.strip() if not segment or not _GATE_SCRIPT_RE.search(segment): continue - if _segment_runs_gate(segment): + if _segment_runs_gate(segment, workspace): return True return False @@ -342,14 +400,20 @@ def _verify_script_has_substance(workspace: Path) -> bool: return False -def _records_gate_run(low: str) -> bool: - """A gate token AND an independent run-word share one lowered line. +def _records_gate_run(low: str, require_script_path: bool = False) -> bool: + """A gate token AND an independent verdict word share one lowered line. - The run-word is checked against the residue *after* the gate tokens are - removed, so a token that itself contains a run-word (``anticheat_scan`` ⊃ - ``scan``) cannot self-satisfy — a bare token earns nothing. + The verdict word is checked against the residue *after* the gate tokens are + removed, so a token that itself contains one cannot self-satisfy — a bare + token earns nothing. The word list is verdict vocabulary only: ordinary + English ("ran", "result", "the deadline passed") is narration, not a record. + With ``require_script_path`` (the RUNLOG bar) the line must name the actual + gate ``.py`` path, not just the bare token. """ - if not any(token in low for token in _GATE_TOKENS): + if require_script_path: + if not _GATE_SCRIPT_RE.search(low): + return False + elif not any(token in low for token in _GATE_TOKENS): return False residue = low for token in _GATE_TOKENS: @@ -372,7 +436,7 @@ def _receipt_records_gate(line: str) -> bool: def _gate_run_recorded(paths) -> bool: """RUNLOG.md / .loop/receipts/*.jsonl record an actual gate run.""" for line in _read_text(paths.runlog).splitlines(): - if _records_gate_run(line.lower()): + if _records_gate_run(line.lower(), require_script_path=True): return True receipts = paths.loop_dir / "receipts" if receipts.is_dir(): diff --git a/scripts/test_inspect_loop.py b/scripts/test_inspect_loop.py index ccaf8b8..563d7ab 100644 --- a/scripts/test_inspect_loop.py +++ b/scripts/test_inspect_loop.py @@ -445,13 +445,18 @@ def test_documented_cli_by_path_reads_dotloop_manifest(tmp_path): # --- M3: false-completion credit grades on execution evidence, not tokens ---- -def _verify_gate_loop(root: pathlib.Path, name: str, verify_body: str) -> pathlib.Path: +def _verify_gate_loop( + root: pathlib.Path, name: str, verify_body: str, gate_files: bool = False +) -> pathlib.Path: """A loop whose only signal is a single verify-safety script body.""" d = root / name (d / "scripts").mkdir(parents=True) (d / "scripts" / "verify-safety").write_text( "#!/bin/sh\n" + verify_body, encoding="utf-8" ) + if gate_files: + (d / "scripts" / "holdout_gate.py").write_text("# gate\n", encoding="utf-8") + (d / "scripts" / "anticheat_scan.py").write_text("# gate\n", encoding="utf-8") return d @@ -483,7 +488,9 @@ def test_real_gate_invocations_keep_invoked_credit(tmp_path): 'echo "gate:" && python3 scripts/holdout_gate.py\n', ) for i, body in enumerate(real): - loop = _verify_gate_loop(tmp_path, f"real{i}", body) + # rv2: gate files exist on disk — invocation credit requires the real + # precondition; the $REPO shape stays shape-only (unresolvable path). + loop = _verify_gate_loop(tmp_path, f"real{i}", body, gate_files=True) assert il._gate_invoked_in_verify(loop) is True, body @@ -766,6 +773,8 @@ def test_allowlisted_invocations_keep_invoked_credit(tmp_path): (loop / "scripts" / "verify-fast").write_text( "#!/bin/sh\n" + body, encoding="utf-8" ) + (loop / "scripts" / "holdout_gate.py").write_text("# gate\n", encoding="utf-8") + (loop / "scripts" / "anticheat_scan.py").write_text("# gate\n", encoding="utf-8") assert il._gate_invoked_in_verify(loop) is True, body @@ -776,6 +785,7 @@ def test_versioned_python_interpreter_invocation_counts(tmp_path): (loop / "scripts" / "verify-fast").write_text( "#!/bin/sh\npython3.12 scripts/holdout_gate.py --strict\n", encoding="utf-8" ) + (loop / "scripts" / "holdout_gate.py").write_text("# gate\n", encoding="utf-8") assert il._gate_invoked_in_verify(loop) is True @@ -883,3 +893,124 @@ def test_fresh_scaffold_keeps_independent_verification_credit(tmp_path): report = il.inspect_loop(str(target)) present = " ".join(report["present"]).lower() assert "independent verification" in present + + +# --- re-verify round 2: non-executing content must not launder "invoked", ---- +# --- and genuine wrapped/uv-run invocations must keep it ---------------------- + + +def _rv2_loop(tmp_path, name, verify_line, gate_file=False): + loop = tmp_path / name + (loop / ".loop").mkdir(parents=True) + (loop / "scripts").mkdir() + (loop / "scripts" / "verify-fast").write_text( + f"#!/bin/sh\n{verify_line}\n", encoding="utf-8" + ) + if gate_file: + (loop / "scripts" / "holdout_gate.py").write_text("# gate\n", encoding="utf-8") + return loop + + +def test_trailing_comment_naming_gate_earns_no_invoked_credit(tmp_path): + # rv2 blocker 1: a gate named only in a trailing shell comment is never run — + # the actual command is a no-op interpreter call. + for i, line in enumerate([ + "python3 -V # scripts/holdout_gate.py anti-cheat", + "python3 --version # holdout_gate.py", + ]): + loop = _rv2_loop(tmp_path, f"cmt{i}", line, gate_file=True) + assert il._gate_invoked_in_verify(loop) is False, line + + +def test_redirection_sink_naming_gate_earns_no_invoked_credit(tmp_path): + # rv2 blocker 1: a gate path that is only a redirect target is never executed. + for i, line in enumerate([ + "python3 realwork.py > scripts/holdout_gate.py", + "python3 realwork.py 2>scripts/holdout_gate.py", + ]): + loop = _rv2_loop(tmp_path, f"redir{i}", line, gate_file=True) + assert il._gate_invoked_in_verify(loop) is False, line + + +def test_uv_non_run_subcommands_earn_no_invoked_credit(tmp_path): + # rv2 blocker 1: only `uv run` executes; pip install / add merely reference. + for i, line in enumerate([ + "uv pip install scripts/holdout_gate.py", + "uv add scripts/holdout_gate.py", + ]): + loop = _rv2_loop(tmp_path, f"uv{i}", line, gate_file=True) + assert il._gate_invoked_in_verify(loop) is False, line + runs = _rv2_loop(tmp_path, "uvrun", "uv run scripts/holdout_gate.py", gate_file=True) + assert il._gate_invoked_in_verify(runs) is True + + +def test_invoked_requires_gate_file_for_resolvable_paths(tmp_path): + # rv2 blocker 3: a genuine invocation SHAPE naming a workspace-relative gate + # that does not exist earns nothing — "invoked" (full credit) must not have a + # weaker precondition than "wired" (half credit, which checks existence). + ghost = _rv2_loop(tmp_path, "ghost", "python3 scripts/holdout_gate.py") + assert il._gate_invoked_in_verify(ghost) is False + report = il.inspect_loop(str(ghost)) + assert report["score"] < 80 + assert report["verdict"] != "strong" + + real = _rv2_loop(tmp_path, "real", "python3 scripts/holdout_gate.py", gate_file=True) + assert il._gate_invoked_in_verify(real) is True + + +def test_unresolvable_gate_paths_keep_shape_credit(tmp_path): + # rv2 blocker 3 counter-case: a $VAR path cannot be checked statically — the + # flagship's `python3 "$REPO/scripts/holdout_gate.py"` (gate lives at repo + # root, outside the example workspace) must keep credit. + loop = _rv2_loop(tmp_path, "var", 'python3 "$REPO/scripts/holdout_gate.py"') + assert il._gate_invoked_in_verify(loop) is True + + +def test_by_path_invocation_requires_gate_file(tmp_path): + ghost = _rv2_loop(tmp_path, "bypath", "./scripts/holdout_gate.py") + assert il._gate_invoked_in_verify(ghost) is False + real = _rv2_loop(tmp_path, "bypath2", "./scripts/holdout_gate.py", gate_file=True) + assert il._gate_invoked_in_verify(real) is True + + +def test_wrapper_prefixed_real_invocations_keep_credit(tmp_path): + # rv2 note: honest wrapper prefixes are transparent, not disqualifying. + for i, line in enumerate([ + "time python3 scripts/holdout_gate.py", + "env python3 scripts/holdout_gate.py", + "nohup python3 scripts/holdout_gate.py", + ]): + loop = _rv2_loop(tmp_path, f"wrap{i}", line, gate_file=True) + assert il._gate_invoked_in_verify(loop) is True, line + + +def test_runlog_common_english_prose_earns_no_recorded_credit(tmp_path): + # rv2 blocker 2: narrative sentences whose only "run-word" is ordinary English + # (passed/ran/result/clean) are self-narration, not gate records. + for i, line in enumerate([ + "Iteration 3: the review deadline passed before we could wire a real holdout_gate.", + "as a result, holdout_gate is our chosen plan", + "we ran out of time to wire holdout_gate", + "keep the holdout_gate design clean", + ]): + loop = tmp_path / f"prose{i}" + (loop / ".loop").mkdir(parents=True) + (loop / "RUNLOG.md").write_text(line + "\n", encoding="utf-8") + paths = il.resolve_loop_paths(loop) + assert il._gate_run_recorded(paths) is False, line + + +def test_runlog_record_requires_script_path_not_bare_token(tmp_path): + # rv2 blocker 2: RUNLOG credit needs the actual .py path plus a verdict word — + # a bare token with prose around it is not a record. + bare = tmp_path / "baretok" + (bare / ".loop").mkdir(parents=True) + (bare / "RUNLOG.md").write_text("holdout_gate verdict: pass\n", encoding="utf-8") + assert il._gate_run_recorded(il.resolve_loop_paths(bare)) is False + + real = tmp_path / "realrec" + (real / ".loop").mkdir(parents=True) + (real / "RUNLOG.md").write_text( + "gate: scripts/holdout_gate.py -> verdict PASS (0 flagged)\n", encoding="utf-8" + ) + assert il._gate_run_recorded(il.resolve_loop_paths(real)) is True From 6a7d41f6e8ffba39dd6edab7ef7f14dd7495a857 Mon Sep 17 00:00:00 2001 From: Sollan Systems <khall0239@gmail.com> Date: Tue, 7 Jul 2026 20:46:52 -0400 Subject: [PATCH 5/5] =?UTF-8?q?fix(inspect):=20round-3=20residuals=20?= =?UTF-8?q?=E2=80=94=20compound=20redirect=20sinks,=20whole-word=20verdict?= =?UTF-8?q?=20tokens,=20records=20need=20a=20gate=20on=20disk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - redirection expressions are stripped BEFORE segment splitting, so >&/>|/&> sinks can no longer sever into a bare-path segment that reads as a direct gate invocation; a real invocation with a 2>&1 suffix keeps credit - verdict words match on word boundaries: cleanup/passphrase/surpassed no longer satisfy the clean/pass bar in RUNLOG prose or receipt JSON - RUNLOG/receipt record credit requires a gate script somewhere on disk — a record-shaped line about a tool that does not exist earns nothing --- scripts/inspect_loop.py | 32 ++++++++++-- scripts/test_inspect_loop.py | 95 +++++++++++++++++++++++++++++++----- 2 files changed, 109 insertions(+), 18 deletions(-) diff --git a/scripts/inspect_loop.py b/scripts/inspect_loop.py index 71bd7a4..7bbbc8f 100644 --- a/scripts/inspect_loop.py +++ b/scripts/inspect_loop.py @@ -27,8 +27,9 @@ redirection using it as a sink — earns *nothing*. A workspace-relative gate path must exist on disk; only unresolvable paths (``$VAR``/absolute) earn shape-only credit. OR ``RUNLOG.md`` / ``.loop/receipts/*.jsonl`` records an - actual run (the gate ``.py`` path AND a verdict word on one line — not a bare - token in stuffed prose; ordinary English like "ran"/"result" never counts). + actual run (the gate ``.py`` path AND a whole-word verdict token on one line, + with a gate script present on disk — not a bare token in stuffed prose; + ordinary English like "ran"/"result"/"cleanup"/"passphrase" never counts). * **wired** (partial credit, half the weight) — a gate script file exists (``scripts/holdout_gate.py`` / ``anticheat_scan.py`` / ``anti_cheat.py``) and is referenced from the contract's verify surface (SPEC / WORKFLOW / @@ -126,7 +127,11 @@ class _Paths: # recorded run from mere prose ("anti-cheat", "false-completion"). _GATE_TOKENS = ("holdout_gate", "anticheat_scan", "anti_cheat") _GATE_SCRIPTS = ("holdout_gate.py", "anticheat_scan.py", "anti_cheat.py") -_GATE_RUN_WORDS = ("verdict", "pass", "fail", "flagged", "clean", "exit 0") +# Verdict vocabulary, matched on WORD BOUNDARIES: "cleanup"/"passphrase"/ +# "surpassed" must never satisfy the bar the way "clean"/"pass" do. +_GATE_RUN_WORDS_RE = re.compile( + r"\b(?:verdict|pass|passed|fail|failed|flagged|clean)\b|\bexit 0\b" +) _FALSE_COMPLETION_PARTIAL_DIVISOR = 2 # wired-but-unrun earns half the weight # A gate script referenced as a *.py path — the invocation shape a real verify @@ -146,6 +151,13 @@ class _Paths: # A token that is (or opens) a shell redirection: everything after it is a file # operand, not part of the executed command (`python3 x.py > holdout_gate.py`). _REDIRECTION_RE = re.compile(r"^\d*(>>?|<)") +# Strip whole redirection expressions (operator + file operand) from a line +# BEFORE segment splitting: compound operators (`>&`, `>|`, `&>`) contain the +# very characters the segment splitter cuts on, so a redirect sink would +# otherwise be severed into its own segment and read as a bare-path invocation. +_REDIRECTION_STRIP_RE = re.compile( + r"(?:\d*(?:>>|>\||>&|>|<<-|<<|<&|<)|&>>?)\s*\S*" +) # Split a shell line into command segments so a real invocation chained after an # inert emitter (`echo x && python3 ...holdout_gate.py`) is still discovered. _SEGMENT_SPLIT_RE = re.compile(r"&&|\|\||[;|()&]") @@ -372,6 +384,10 @@ def _gate_invoked_in_verify(workspace: Path) -> bool: stripped = line.strip() if not stripped or stripped.startswith("#"): continue + # Redirection expressions go first: `>&`/`>|` contain segment-split + # characters, so a sink severed into its own segment would read as + # a bare-path invocation of a file the command never executes. + stripped = _REDIRECTION_STRIP_RE.sub(" ", stripped) for segment in _SEGMENT_SPLIT_RE.split(stripped): segment = segment.strip() if not segment or not _GATE_SCRIPT_RE.search(segment): @@ -418,7 +434,7 @@ def _records_gate_run(low: str, require_script_path: bool = False) -> bool: residue = low for token in _GATE_TOKENS: residue = residue.replace(token, " ") - return any(word in residue for word in _GATE_RUN_WORDS) + return bool(_GATE_RUN_WORDS_RE.search(residue)) def _receipt_records_gate(line: str) -> bool: @@ -434,7 +450,13 @@ def _receipt_records_gate(line: str) -> bool: def _gate_run_recorded(paths) -> bool: - """RUNLOG.md / .loop/receipts/*.jsonl record an actual gate run.""" + """RUNLOG.md / .loop/receipts/*.jsonl record an actual gate run. + + A record of a run implies a gate that can run: with no gate script anywhere + on disk, record-shaped prose is a claim about a tool that does not exist. + """ + if not _script_exists(paths.workspace, *_GATE_SCRIPTS): + return False for line in _read_text(paths.runlog).splitlines(): if _records_gate_run(line.lower(), require_script_path=True): return True diff --git a/scripts/test_inspect_loop.py b/scripts/test_inspect_loop.py index 563d7ab..833fb56 100644 --- a/scripts/test_inspect_loop.py +++ b/scripts/test_inspect_loop.py @@ -508,9 +508,12 @@ def test_bare_gate_token_in_runlog_earns_no_recorded_credit(tmp_path): def test_recorded_gate_run_with_run_word_earns_credit(tmp_path): - # M3(b): a real recorded run — token AND an independent run-word on one line. + # M3(b): a real recorded run — token AND an independent run-word on one line, + # with the gate script present on disk (rv3: a record implies a real gate). loop = tmp_path / "rl2" (loop / ".loop").mkdir(parents=True) + (loop / "scripts").mkdir() + (loop / "scripts" / "holdout_gate.py").write_text("# gate\n", encoding="utf-8") (loop / "RUNLOG.md").write_text( "gate: scripts/holdout_gate.py target/manifest.json -> verdict Succeeded\n", encoding="utf-8", @@ -523,6 +526,8 @@ def test_receipts_jsonl_records_gate_run(tmp_path): # M3(b): a structured receipt line is parsed as JSON and matched on fields. loop = tmp_path / "rc" (loop / ".loop" / "receipts").mkdir(parents=True) + (loop / "scripts").mkdir() + (loop / "scripts" / "holdout_gate.py").write_text("# gate\n", encoding="utf-8") (loop / ".loop" / "receipts" / "run.jsonl").write_text( json.dumps({"event": "holdout_gate", "verdict": "Succeeded"}) + "\n", encoding="utf-8", @@ -1000,17 +1005,81 @@ def test_runlog_common_english_prose_earns_no_recorded_credit(tmp_path): assert il._gate_run_recorded(paths) is False, line +def _rv2_runlog_loop(tmp_path, name, line, gate_file=True): + loop = tmp_path / name + (loop / ".loop").mkdir(parents=True) + (loop / "RUNLOG.md").write_text(line + "\n", encoding="utf-8") + if gate_file: + (loop / "scripts").mkdir() + (loop / "scripts" / "holdout_gate.py").write_text("# gate\n", encoding="utf-8") + return il.resolve_loop_paths(loop) + + def test_runlog_record_requires_script_path_not_bare_token(tmp_path): # rv2 blocker 2: RUNLOG credit needs the actual .py path plus a verdict word — - # a bare token with prose around it is not a record. - bare = tmp_path / "baretok" - (bare / ".loop").mkdir(parents=True) - (bare / "RUNLOG.md").write_text("holdout_gate verdict: pass\n", encoding="utf-8") - assert il._gate_run_recorded(il.resolve_loop_paths(bare)) is False - - real = tmp_path / "realrec" - (real / ".loop").mkdir(parents=True) - (real / "RUNLOG.md").write_text( - "gate: scripts/holdout_gate.py -> verdict PASS (0 flagged)\n", encoding="utf-8" - ) - assert il._gate_run_recorded(il.resolve_loop_paths(real)) is True + # a bare token with prose around it is not a record (gate file present, so + # the .py-path requirement is the discriminating layer). + bare = _rv2_runlog_loop(tmp_path, "baretok", "holdout_gate verdict: pass") + assert il._gate_run_recorded(bare) is False + + real = _rv2_runlog_loop( + tmp_path, "realrec", "gate: scripts/holdout_gate.py -> verdict PASS (0 flagged)" + ) + assert il._gate_run_recorded(real) is True + + +def test_runlog_record_requires_gate_script_on_disk(tmp_path): + # rv3: a record-shaped line about a gate that exists nowhere on disk is a + # claim about a tool that does not exist — no credit. + ghost = _rv2_runlog_loop( + tmp_path, "ghostrec", + "gate: scripts/holdout_gate.py -> verdict PASS (0 flagged)", gate_file=False, + ) + assert il._gate_run_recorded(ghost) is False + + +def test_substring_run_words_earn_no_recorded_credit(tmp_path): + # rv3: verdict words match whole words only — cleanup/passphrase/surpassed + # must never satisfy the bar the way clean/pass do (gate file present, .py + # path named: the word boundary is the only discriminating layer). + for i, line in enumerate([ + "We designed scripts/holdout_gate.py but ran out of time; cleanup pending.", + "blocked: scripts/holdout_gate.py needs a passphrase we do not have yet.", + "scripts/holdout_gate.py budget surpassed this iteration.", + ]): + paths = _rv2_runlog_loop(tmp_path, f"substr{i}", line) + assert il._gate_run_recorded(paths) is False, line + + +def test_substring_run_words_in_receipts_earn_no_credit(tmp_path): + # rv3: the receipt path shares the word-boundary bar. + loop = tmp_path / "rcsub" + (loop / ".loop" / "receipts").mkdir(parents=True) + (loop / "scripts").mkdir() + (loop / "scripts" / "holdout_gate.py").write_text("# gate\n", encoding="utf-8") + (loop / ".loop" / "receipts" / "run.jsonl").write_text( + json.dumps({"note": "will add holdout_gate later", "status": "surpassed budget"}) + + "\n", + encoding="utf-8", + ) + assert il._gate_run_recorded(il.resolve_loop_paths(loop)) is False + + +def test_compound_redirect_sinks_earn_no_invoked_credit(tmp_path): + # rv3: `>&` / `>|` / `&>` contain segment-split characters — the severed sink + # must not read as a bare-path gate invocation. + for i, line in enumerate([ + "python3 realwork.py >& scripts/holdout_gate.py", + "python3 realwork.py >| scripts/holdout_gate.py", + "python3 realwork.py &> scripts/holdout_gate.py", + ]): + loop = _rv2_loop(tmp_path, f"cmpd{i}", line, gate_file=True) + assert il._gate_invoked_in_verify(loop) is False, line + + +def test_real_invocation_with_fd_redirect_keeps_credit(tmp_path): + # rv3 counter-case: stripping redirections must not strip the command. + loop = _rv2_loop( + tmp_path, "fdredir", "python3 scripts/holdout_gate.py 2>&1", gate_file=True + ) + assert il._gate_invoked_in_verify(loop) is True