diff --git a/scripts/inspect_loop.py b/scripts/inspect_loop.py index 61374d0..7bbbc8f 100644 --- a/scripts/inspect_loop.py +++ b/scripts/inspect_loop.py @@ -16,10 +16,20 @@ key, or the phrase "false-completion" in prose earns *nothing* — those are assertions the loop makes about itself. The three grades are: - * **invoked** (full credit) — a ``scripts/verify-*`` gate invokes a holdout / - anti-cheat gate on an executable (non-comment) line, OR ``RUNLOG.md`` / - ``.loop/receipts/*.jsonl`` records an actual run (a ``holdout_gate`` verdict - / anti-cheat scan result). + * **invoked** (full credit) — a ``scripts/verify-*`` gate *executes* a holdout + / anti-cheat gate script: an interpreter (``python``/``python3``/``python3.N``/ + ``uv run``/``bash``/``sh``/``exec``, optionally behind a transparent wrapper + like ``env``/``time``/``nohup``) runs a gate script named in its arguments + (``python3 scripts/holdout_gate.py``), or the gate script is invoked directly + by path (``./scripts/holdout_gate.py``). A command that only *references* the + file — ``echo``/``printf`` printing it, ``grep``/``cat``/``ls``/``test``/ + ``head``/``wc``/``find`` reading it, a trailing ``# comment`` naming it, or a + redirection using it as a sink — earns *nothing*. A workspace-relative gate + path must exist on disk; only unresolvable paths (``$VAR``/absolute) earn + shape-only credit. OR ``RUNLOG.md`` / ``.loop/receipts/*.jsonl`` records an + actual run (the gate ``.py`` path AND a whole-word verdict token on one line, + with a gate script present on disk — not a bare token in stuffed prose; + ordinary English like "ran"/"result"/"cleanup"/"passphrase" never counts). * **wired** (partial credit, half the weight) — a gate script file exists (``scripts/holdout_gate.py`` / ``anticheat_scan.py`` / ``anti_cheat.py``) and is referenced from the contract's verify surface (SPEC / WORKFLOW / @@ -41,6 +51,8 @@ from __future__ import annotations import json +import re +import shlex import sys from pathlib import Path @@ -115,9 +127,44 @@ class _Paths: # recorded run from mere prose ("anti-cheat", "false-completion"). _GATE_TOKENS = ("holdout_gate", "anticheat_scan", "anti_cheat") _GATE_SCRIPTS = ("holdout_gate.py", "anticheat_scan.py", "anti_cheat.py") -_GATE_RUN_WORDS = ("verdict", "scan", "result", "passed", "failed", "ran", "clean", "flagged") +# Verdict vocabulary, matched on WORD BOUNDARIES: "cleanup"/"passphrase"/ +# "surpassed" must never satisfy the bar the way "clean"/"pass" do. +_GATE_RUN_WORDS_RE = re.compile( + r"\b(?:verdict|pass|passed|fail|failed|flagged|clean)\b|\bexit 0\b" +) _FALSE_COMPLETION_PARTIAL_DIVISOR = 2 # wired-but-unrun earns half the weight +# A gate script referenced as a *.py path — the invocation shape a real verify +# gate uses (`python3 scripts/holdout_gate.py`, `./scripts/anticheat_scan.py`). +# The bare token ("holdout_gate") without .py is prose, not an invocation. +_GATE_SCRIPT_RE = re.compile(r"\b(?:holdout_gate|anticheat_scan|anti_cheat)\.py\b") +# Genuine-invocation ALLOWLIST: "invoked" credit requires an executable line that +# actually *runs* a gate script. Either an interpreter leads the command and a +# gate script is named in its arguments, or the gate script is invoked directly +# by path. Everything else — a reference via echo/printf/grep/cat/ls/test/head/ +# wc/find, or any other non-interpreter command — earns nothing. `uv` executes +# only through its `run` subcommand (pip install / add merely reference). +_GATE_INTERPRETERS = frozenset({"python", "python3", "bash", "sh", "exec"}) +_VERSIONED_PYTHON_RE = re.compile(r"^python3\.\d+$") +# Wrapper commands that transparently run their argument command. +_TRANSPARENT_PREFIXES = frozenset({"env", "time", "nohup", "nice", "command"}) +# A token that is (or opens) a shell redirection: everything after it is a file +# operand, not part of the executed command (`python3 x.py > holdout_gate.py`). +_REDIRECTION_RE = re.compile(r"^\d*(>>?|<)") +# Strip whole redirection expressions (operator + file operand) from a line +# BEFORE segment splitting: compound operators (`>&`, `>|`, `&>`) contain the +# very characters the segment splitter cuts on, so a redirect sink would +# otherwise be severed into its own segment and read as a bare-path invocation. +_REDIRECTION_STRIP_RE = re.compile( + r"(?:\d*(?:>>|>\||>&|>|<<-|<<|<&|<)|&>>?)\s*\S*" +) +# Split a shell line into command segments so a real invocation chained after an +# inert emitter (`echo x && python3 ...holdout_gate.py`) is still discovered. +_SEGMENT_SPLIT_RE = re.compile(r"&&|\|\||[;|()&]") +# A verify-* line is "inert" (no verification substance) when its leading command +# only prints or no-ops. A body of nothing but these plus comments is not proof. +_INERT_LINE_COMMANDS = frozenset({"echo", "printf", "exit", "true", "false", ":"}) + def _read_text(path: Path) -> str: try: @@ -151,6 +198,62 @@ def _task_verify_declared(tasks: dict) -> bool: return False +# Scaffold placeholder convention: an unfilled slot is the literal "REPLACE" +# marker (loop/scaffold.py `_substitutions`: "REPLACE: " for filled tokens, +# a bare "REPLACE" for the extra CRITERION_2/3 slots). +_PLACEHOLDER_MSG = ( + "unfilled scaffold placeholders ('REPLACE: ...') — replace them with " + "concrete, verifiable text before this loop can claim to define {what}" +) +_SECTION_HEADING_RE = re.compile(r"\s*#{1,6}\s+(?P.*\S)\s*$") +_LIST_ITEM_RE = re.compile(r"\s*(?:\d+[.)]|[-*+])\s+(?P<item>.*\S)\s*$") + + +def _is_placeholder(text: str) -> bool: + """True iff ``text`` is an unfilled scaffold slot ('REPLACE' / 'REPLACE:…').""" + upper = text.strip().upper() + return upper == "REPLACE" or upper.startswith("REPLACE:") + + +def _section_body(text: str, heading: str) -> str | None: + """Return the body of the first ``## <heading>`` section (case-insensitive).""" + lines = text.splitlines() + target = heading.strip().lower() + start = None + for i, line in enumerate(lines): + match = _SECTION_HEADING_RE.match(line) + if match and match.group("title").strip().lower().rstrip(":") == target: + start = i + 1 + break + if start is None: + return None + body: list[str] = [] + for line in lines[start:]: + if _SECTION_HEADING_RE.match(line): + break + body.append(line) + return "\n".join(body) + + +def _success_criteria_all_placeholder(spec_text: str) -> bool: + """True iff the SPEC's Success-criteria list has items and all are unfilled.""" + body = _section_body(spec_text, "success criteria") + if body is None: + return False + items = [m.group("item") for line in body.splitlines() if (m := _LIST_ITEM_RE.match(line))] + return bool(items) and all(_is_placeholder(item) for item in items) + + +def _task_titles_all_placeholder(tasks: dict) -> bool: + """True iff every declared task carries an unfilled 'REPLACE' title.""" + rows = tasks.get("tasks") + if not isinstance(rows, list) or not rows: + return False + titles = [row.get("title") for row in rows if isinstance(row, dict)] + titles = [t for t in titles if isinstance(t, str) and t.strip()] + return bool(titles) and all(_is_placeholder(title) for title in titles) + + def _terminal_states_covered_from_contract(loop: Path) -> int: """Count terminal taxonomy coverage from contract-owned files only.""" @@ -175,33 +278,194 @@ def _verify_scripts(workspace: Path) -> list[Path]: return sorted(p for p in scripts.glob("verify-*") if p.is_file()) +def _leading_command(segment: str) -> str: + """The command word of a shell segment (past subshell/group openers).""" + stripped = segment.lstrip("({ \t") + parts = stripped.split(None, 1) + return parts[0] if parts else "" + + +def _shell_tokens(segment: str) -> list[str]: + """Tokenize a shell segment (subshell openers stripped, quotes removed). + + Unquoted ``#`` starts a comment — everything after it is prose, not command + content, so a gate named only in a trailing comment never tokenizes. + """ + seg = segment.lstrip("({ \t") + try: + return shlex.split(seg, posix=True, comments=True) + except ValueError: + tokens: list[str] = [] + for tok in seg.split(): + if tok.startswith("#"): + break + tokens.append(tok) + return tokens + + +def _basename(token: str) -> str: + """The trailing path component of a shell token (posix `/` separator).""" + return token.rsplit("/", 1)[-1] + + +def _is_gate_interpreter(token: str) -> bool: + return token in _GATE_INTERPRETERS or bool(_VERSIONED_PYTHON_RE.match(token)) + + +def _statically_resolvable(token: str) -> bool: + """A gate path we can check on disk: workspace-relative, no expansion.""" + return not (token.startswith(("/", "~")) or "$" in token or "`" in token) + + +def _gate_on_disk(workspace: Path, token: str) -> bool: + rel = token[2:] if token.startswith("./") else token + if (workspace / rel).is_file(): + return True + return "/" not in rel and (workspace / "scripts" / rel).is_file() + + +def _gate_arg_credits(workspace: Path, token: str) -> bool: + """A token earns gate credit: names a gate script that plausibly exists. + + A workspace-relative path must exist on disk — an invocation SHAPE of a + non-existent gate is stolen valor ("invoked" full credit must not have a + weaker precondition than "wired" half credit, which checks existence). + Unresolvable paths ($VAR / absolute / backticks) keep shape-only credit: + the flagship's gate lives outside the example workspace behind ``$REPO``. + """ + if _basename(token) not in _GATE_SCRIPTS: + return False + if not _statically_resolvable(token): + return True + return _gate_on_disk(workspace, token) + + +def _segment_runs_gate(segment: str, workspace: Path) -> bool: + """True iff this shell segment genuinely *executes* a gate script. + + Allowlisted shapes only: an interpreter (python/python3/python3.N/uv run/ + bash/sh/exec, optionally behind env/time/nohup/nice/command) whose arguments + name a gate script, or the gate script invoked directly by path + (`./scripts/holdout_gate.py`). A leading grep/cat/ls/test/head/wc/find — or + echo/printf — that merely references the file is not an execution; neither + is a gate path that appears only as a redirection sink. + """ + tokens = _shell_tokens(segment) + for i, tok in enumerate(tokens): + if _REDIRECTION_RE.match(tok): + tokens = tokens[:i] + break + while tokens and tokens[0] in _TRANSPARENT_PREFIXES: + tokens = tokens[1:] + if not tokens: + return False + lead, args = tokens[0], tokens[1:] + if _basename(lead) in _GATE_SCRIPTS: + return _gate_arg_credits(workspace, lead) + if lead == "uv": + if not args or args[0] != "run": + return False + args = args[1:] + elif not _is_gate_interpreter(lead): + return False + return any(_gate_arg_credits(workspace, arg) for arg in args) + + def _gate_invoked_in_verify(workspace: Path) -> bool: - """A verify-* script runs a holdout/anti-cheat gate on an executable line.""" + """A verify-* script genuinely *executes* a holdout/anti-cheat gate. + + Credit requires the gate script (`holdout_gate.py` / `anticheat_scan.py` / + `anti_cheat.py`) to be *run* — an interpreter invocation or a direct + by-path call. A command that only prints, reads, lists, or searches the file + (echo/printf/grep/cat/ls/test/head/wc/find) earns nothing. + """ for script in _verify_scripts(workspace): for line in _read_text(script).splitlines(): stripped = line.strip() if not stripped or stripped.startswith("#"): continue - if any(token in stripped for token in _GATE_TOKENS): + # Redirection expressions go first: `>&`/`>|` contain segment-split + # characters, so a sink severed into its own segment would read as + # a bare-path invocation of a file the command never executes. + stripped = _REDIRECTION_STRIP_RE.sub(" ", stripped) + for segment in _SEGMENT_SPLIT_RE.split(stripped): + segment = segment.strip() + if not segment or not _GATE_SCRIPT_RE.search(segment): + continue + if _segment_runs_gate(segment, workspace): + return True + return False + + +def _verify_script_has_substance(workspace: Path) -> bool: + """A verify-* script exists whose body has ≥1 non-inert executable line. + + A body of nothing but comments and printing/no-op commands (echo/printf/exit/ + true/false/`:`) is not verification — a file merely *named* ``verify-fast`` + earns no independent-verification credit. The shipped scaffold's verify-fast + keeps credit: its contract-file existence ``for``/``if`` loop is substantive. + """ + for script in _verify_scripts(workspace): + for line in _read_text(script).splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + lead = _leading_command(stripped) + if lead and lead not in _INERT_LINE_COMMANDS: return True return False +def _records_gate_run(low: str, require_script_path: bool = False) -> bool: + """A gate token AND an independent verdict word share one lowered line. + + The verdict word is checked against the residue *after* the gate tokens are + removed, so a token that itself contains one cannot self-satisfy — a bare + token earns nothing. The word list is verdict vocabulary only: ordinary + English ("ran", "result", "the deadline passed") is narration, not a record. + With ``require_script_path`` (the RUNLOG bar) the line must name the actual + gate ``.py`` path, not just the bare token. + """ + if require_script_path: + if not _GATE_SCRIPT_RE.search(low): + return False + elif not any(token in low for token in _GATE_TOKENS): + return False + residue = low + for token in _GATE_TOKENS: + residue = residue.replace(token, " ") + return bool(_GATE_RUN_WORDS_RE.search(residue)) + + +def _receipt_records_gate(line: str) -> bool: + """A receipt line is a real gate record: parseable JSON with gate+run fields.""" + line = line.strip() + if not line: + return False + try: + obj = json.loads(line) + except json.JSONDecodeError: + return False + return _records_gate_run(json.dumps(obj).lower()) + + def _gate_run_recorded(paths) -> bool: - """RUNLOG.md / .loop/receipts/*.jsonl record an actual gate run.""" - texts = [_read_text(paths.runlog)] + """RUNLOG.md / .loop/receipts/*.jsonl record an actual gate run. + + A record of a run implies a gate that can run: with no gate script anywhere + on disk, record-shaped prose is a claim about a tool that does not exist. + """ + if not _script_exists(paths.workspace, *_GATE_SCRIPTS): + return False + for line in _read_text(paths.runlog).splitlines(): + if _records_gate_run(line.lower(), require_script_path=True): + return True receipts = paths.loop_dir / "receipts" if receipts.is_dir(): - texts.extend(_read_text(p) for p in sorted(receipts.glob("*.jsonl"))) - for text in texts: - for line in text.splitlines(): - low = line.lower() - if any(token in low for token in _GATE_TOKENS): - return True - if ("holdout" in low or "anticheat" in low or "anti-cheat" in low) and any( - word in low for word in _GATE_RUN_WORDS - ): - return True + for receipt in sorted(receipts.glob("*.jsonl")): + for line in _read_text(receipt).splitlines(): + if _receipt_records_gate(line): + return True return False @@ -247,18 +511,20 @@ def _evaluate_contract_checks(loop: Path) -> dict[str, object]: policies = manifest.get("policies") if isinstance(manifest, dict) else None manifest_declares_plan = isinstance(policies, dict) and "plan_then_execute" in policies - has_spec_criteria = "success criteria" in spec or "success_criteria" in spec + # A fresh scaffold's Success-criteria list and task titles are the literal + # "REPLACE:" placeholders. A structurally-present-but-unfilled criteria list + # earns no defines_success credit — a shell is not a defined success. + spec_raw = _read_text(paths.spec) + "\n" + _read_text(paths.contract) + criteria_all_placeholder = _success_criteria_all_placeholder(spec_raw) + task_titles_placeholder = _task_titles_all_placeholder(tasks) + + has_criteria_heading = "success criteria" in spec or "success_criteria" in spec + has_spec_criteria = has_criteria_heading and not criteria_all_placeholder + # A verify-* script earns credit only if it actually verifies — a body of + # nothing but echo/printf/no-ops (a file merely *named* verify-fast) does not. has_verify = ( _task_verify_declared(tasks) - or _script_exists( - paths.workspace, - "verify-fast", - "verify-fast.sh", - "verify-full", - "verify-full.sh", - "verify-safety", - "verify-safety.sh", - ) + or _verify_script_has_substance(paths.workspace) or "scripts/verify" in spec ) has_approval = ( @@ -278,6 +544,8 @@ def _evaluate_contract_checks(loop: Path) -> dict[str, object]: "approval_gates": has_approval, "false_completion_defense": _false_completion_credit(paths), "plan_then_execute": has_plan_then_execute, + "_success_criteria_placeholder": has_criteria_heading and criteria_all_placeholder, + "_task_titles_placeholder": task_titles_placeholder, } @@ -331,12 +599,18 @@ def inspect_loop(loop_dir: str) -> dict: if key == "false_completion_defense": score += _grade_false_completion(value, weight, label, gap_msg, present, gaps) continue + if key == "defines_success" and not value and results.get("_success_criteria_placeholder"): + gaps.append("success criteria are " + _PLACEHOLDER_MSG.format(what="success")) + continue if value: score += weight present.append(label) else: gaps.append(gap_msg) + if results.get("_task_titles_placeholder"): + gaps.append("TASKS.json titles are " + _PLACEHOLDER_MSG.format(what="its tasks")) + terminal_points = round(_TERMINAL_WEIGHT * covered / len(TERMINAL_STATES)) score += terminal_points if covered == len(TERMINAL_STATES): @@ -360,6 +634,18 @@ def inspect_loop(loop_dir: str) -> dict: ) score = max(0, min(100, score)) + + # False-completion defense is the anti-gaming keystone: a loop with NO real + # holdout/anti-cheat gate (grade "none") must never reach "strong" or clear a + # fail-under-80 CI gate on keyword stuffing alone. Cap it below the threshold. + # "wired"/"invoked" grades — a gate that at least exists — are uncapped. + if results["false_completion_defense"] == "none" and score >= 80: + score = 79 + gaps.append( + "no false-completion defense — score capped below 'strong'; " + "wire a holdout/anti-cheat gate" + ) + return { "target": str(loop), "score": score, diff --git a/scripts/test_inspect_loop.py b/scripts/test_inspect_loop.py index fffa433..833fb56 100644 --- a/scripts/test_inspect_loop.py +++ b/scripts/test_inspect_loop.py @@ -440,3 +440,646 @@ def test_documented_cli_by_path_reads_dotloop_manifest(tmp_path): gap_text = " ".join(report["gaps"]).lower() assert "plan-then-execute" not in present_text, report assert "plan-then-execute" in gap_text, report + + +# --- M3: false-completion credit grades on execution evidence, not tokens ---- + + +def _verify_gate_loop( + root: pathlib.Path, name: str, verify_body: str, gate_files: bool = False +) -> pathlib.Path: + """A loop whose only signal is a single verify-safety script body.""" + d = root / name + (d / "scripts").mkdir(parents=True) + (d / "scripts" / "verify-safety").write_text( + "#!/bin/sh\n" + verify_body, encoding="utf-8" + ) + if gate_files: + (d / "scripts" / "holdout_gate.py").write_text("# gate\n", encoding="utf-8") + (d / "scripts" / "anticheat_scan.py").write_text("# gate\n", encoding="utf-8") + return d + + +def test_string_literal_gate_tokens_earn_no_invoked_credit(tmp_path): + # M3(a): a token inside echo/printf/':' args (or any quoted-string emit) is + # inert output, not an executed gate — it must earn ZERO invoked credit. + exploits = ( + 'echo "holdout_gate"\n', + "printf 'holdout_gate\\n'\n", + ": holdout_gate\n", + 'echo "run the holdout_gate please"\n', + 'echo "holdout_gate.py"\n', # even the script name, still just printed + ) + for i, body in enumerate(exploits): + loop = _verify_gate_loop(tmp_path, f"exploit{i}", body) + assert il._gate_invoked_in_verify(loop) is False, body + + +def test_real_gate_invocations_keep_invoked_credit(tmp_path): + # M3(a): the token as part of a genuinely invoked command keeps full credit. + real = ( + "python3 scripts/holdout_gate.py --strict\n", + "bash scripts/anticheat_scan.py\n", + "./scripts/holdout_gate.py\n", + "uv run python scripts/anticheat_scan.py\n", + # the flagship shape: the gate path is double-quoted, invoked by python3. + 'python3 "$REPO/scripts/holdout_gate.py" "$EX/target/manifest.json" --cwd "$EX/target"\n', + # chained after an echo — the real invocation must still count. + 'echo "gate:" && python3 scripts/holdout_gate.py\n', + ) + for i, body in enumerate(real): + # rv2: gate files exist on disk — invocation credit requires the real + # precondition; the $REPO shape stays shape-only (unresolvable path). + loop = _verify_gate_loop(tmp_path, f"real{i}", body, gate_files=True) + assert il._gate_invoked_in_verify(loop) is True, body + + +def test_bare_gate_token_in_runlog_earns_no_recorded_credit(tmp_path): + # M3(b): a bare gate token in RUNLOG prose is a self-narration, not a run — + # even when the token itself (anticheat_scan) contains a run-word ("scan"). + loop = tmp_path / "rl" + (loop / ".loop").mkdir(parents=True) + (loop / "RUNLOG.md").write_text( + "This loop uses holdout_gate and anticheat_scan for false-completion defense.\n", + encoding="utf-8", + ) + paths = il.resolve_loop_paths(loop) + assert il._gate_run_recorded(paths) is False + + +def test_recorded_gate_run_with_run_word_earns_credit(tmp_path): + # M3(b): a real recorded run — token AND an independent run-word on one line, + # with the gate script present on disk (rv3: a record implies a real gate). + loop = tmp_path / "rl2" + (loop / ".loop").mkdir(parents=True) + (loop / "scripts").mkdir() + (loop / "scripts" / "holdout_gate.py").write_text("# gate\n", encoding="utf-8") + (loop / "RUNLOG.md").write_text( + "gate: scripts/holdout_gate.py target/manifest.json -> verdict Succeeded\n", + encoding="utf-8", + ) + paths = il.resolve_loop_paths(loop) + assert il._gate_run_recorded(paths) is True + + +def test_receipts_jsonl_records_gate_run(tmp_path): + # M3(b): a structured receipt line is parsed as JSON and matched on fields. + loop = tmp_path / "rc" + (loop / ".loop" / "receipts").mkdir(parents=True) + (loop / "scripts").mkdir() + (loop / "scripts" / "holdout_gate.py").write_text("# gate\n", encoding="utf-8") + (loop / ".loop" / "receipts" / "run.jsonl").write_text( + json.dumps({"event": "holdout_gate", "verdict": "Succeeded"}) + "\n", + encoding="utf-8", + ) + paths = il.resolve_loop_paths(loop) + assert il._gate_run_recorded(paths) is True + + +def test_prose_renamed_to_jsonl_earns_no_recorded_credit(tmp_path): + # M3(b): a non-JSON prose line stuffed into a .jsonl file is not a receipt. + loop = tmp_path / "rc2" + (loop / ".loop" / "receipts").mkdir(parents=True) + (loop / ".loop" / "receipts" / "run.jsonl").write_text( + "holdout_gate anticheat_scan verdict clean passed\n", encoding="utf-8" + ) + paths = il.resolve_loop_paths(loop) + assert il._gate_run_recorded(paths) is False + + +def test_stuffed_fake_without_gate_file_cannot_reach_wired(tmp_path): + # M3(c): file-exists + surface-reference is the wired bar; with NO gate script + # file, a stuffed contract that merely names the gate cannot reach "wired". + loop = tmp_path / "nofile" + (loop / "scripts").mkdir(parents=True) + (loop / "WORKFLOW.md").write_text( + "# WORKFLOW\nHigh-value tasks are gated by scripts/holdout_gate.py anti-cheat.\n", + encoding="utf-8", + ) + (loop / "SPEC.md").write_text( + "# SPEC\nholdout_gate anticheat_scan anti-cheat defense.\n", encoding="utf-8" + ) + paths = il.resolve_loop_paths(loop) + assert il._gate_script_referenced(paths) is False + assert il._false_completion_credit(paths) == "none" + + +def test_flagship_example_keeps_strong_gate_backed_verdict(): + # Regression guard: the genuinely gate-backed flagship must keep its verdict + # and its invoked false-completion credit — the discriminator must not dock + # the real, double-quoted `holdout_gate.py` invocation in its verify-full. + root = pathlib.Path(__file__).resolve().parent.parent + report = il.inspect_loop(str(root / "examples" / "coverage-repair")) + assert report["verdict"] == "strong" + assert report["score"] >= 80 + assert any("invoked" in signal for signal in report["present"]) + + +# --- M2: unfilled scaffold placeholders earn no defines_success credit -------- + + +def test_placeholder_success_criteria_earn_no_defines_success_credit(tmp_path): + loop = tmp_path / "ph" + loop.mkdir() + (loop / "SPEC.md").write_text( + "# SPEC\n## Success criteria\n" + "1. REPLACE: first success criterion\n2. REPLACE\n3. REPLACE\n", + encoding="utf-8", + ) + report = il.inspect_loop(str(loop)) + present = " ".join(report["present"]).lower() + gaps = " ".join(report["gaps"]).lower() + assert "defines verifiable success criteria" not in present + # The gap names the scaffold placeholder convention actionably. + assert "replace" in gaps and "placeholder" in gaps + + +def test_real_success_criteria_still_earn_defines_success_credit(tmp_path): + loop = tmp_path / "real" + loop.mkdir() + (loop / "SPEC.md").write_text( + "# SPEC\n## Success criteria\n1. coverage >= 80% (scripts/verify-full)\n", + encoding="utf-8", + ) + report = il.inspect_loop(str(loop)) + present = " ".join(report["present"]).lower() + assert "defines verifiable success criteria" in present + + +def test_one_filled_criterion_among_placeholders_still_earns_credit(tmp_path): + # Partial fill is real intent — a single concrete criterion keeps the credit. + loop = tmp_path / "partial" + loop.mkdir() + (loop / "SPEC.md").write_text( + "# SPEC\n## Success criteria\n" + "1. coverage >= 80% (scripts/verify-full)\n2. REPLACE\n", + encoding="utf-8", + ) + report = il.inspect_loop(str(loop)) + present = " ".join(report["present"]).lower() + assert "defines verifiable success criteria" in present + + +def test_placeholder_task_titles_flagged_as_gap(tmp_path): + loop = tmp_path / "pht" + loop.mkdir() + (loop / "TASKS.json").write_text( + json.dumps({"tasks": [{"id": "T1", "title": "REPLACE: first task", + "verify": "scripts/verify-fast"}]}), + encoding="utf-8", + ) + report = il.inspect_loop(str(loop)) + gaps = " ".join(report["gaps"]).lower() + assert "task" in gaps and "placeholder" in gaps + + +def test_fresh_scaffold_is_not_strong_with_placeholder_gaps(tmp_path): + # M2: an unedited scaffold scored 86/strong; it must now land at ok-or-below + # and name the placeholder convention so a stranger knows what to fill. + from loop.scaffold import scaffold + + target = tmp_path / "fresh-scaffold" + scaffold(target) + report = il.inspect_loop(str(target)) + gaps = " ".join(report["gaps"]).lower() + + assert report["verdict"] != "strong" + assert report["score"] < 80 + assert "replace" in gaps and "placeholder" in gaps + + +def test_keyword_stuffed_fake_scores_below_strong_with_false_completion_gap(tmp_path): + # Acceptance (M2 + M3): the review's keyword-stuffed fake — SPEC/WORKFLOW + # prose stuffed with checklist vocabulary, all 7 terminal states, an echo + # "verify" line, a stuffed RUNLOG, and NO real gate files. It scored + # 100/strong before the fixes; it must now land materially below strong with + # a false-completion gap (the echo trick and stuffed prose buy no defense + # credit — M3 — and the unfilled success criteria earn no defines_success + # credit — M2). + fake = tmp_path / "fake" + (fake / ".loop").mkdir(parents=True) + (fake / "scripts").mkdir() + (fake / "SPEC.md").write_text( + "# SPEC — totally-done\n" + "This loop has verifiable success, independent verification, approval " + "gates, false-completion defense, held-out anti-cheat, plan-then-execute.\n" + "## Success criteria\n" + "1. REPLACE: first success criterion\n" + "2. REPLACE\n", + encoding="utf-8", + ) + (fake / "WORKFLOW.md").write_text( + "# WORKFLOW\n## Approval Gates\nApproval gate on side-effects.\n" + "## Plan-then-execute\nPlan-then-execute for untrusted reads.\n" + "## Anti-cheat\nfalse-completion defense via held-out holdout gate anti-cheat.\n" + "## Terminal States\n" + "Succeeded, FailedUnverifiable, FailedBlocked, FailedBudget, " + "FailedSafety, FailedSpecGap, AbortedByHuman.\n", + encoding="utf-8", + ) + (fake / "TASKS.json").write_text( + json.dumps({"tasks": [{"id": "T1", "title": "REPLACE: first task", + "verify": "scripts/verify-fast"}]}), + encoding="utf-8", + ) + (fake / "scripts" / "verify-fast").write_text( + '#!/bin/sh\necho "holdout_gate anticheat_scan all clean"\n', encoding="utf-8" + ) + (fake / "RUNLOG.md").write_text( + "Iteration 1: holdout gate anti-cheat false-completion defense engaged. " + "Everything looks done and fine.\n", + encoding="utf-8", + ) + + report = il.inspect_loop(str(fake)) + present = " ".join(report["present"]).lower() + gaps = " ".join(report["gaps"]).lower() + + assert "false-completion defense" not in present + assert "false-completion" in gaps or "false completion" in gaps + assert report["verdict"] != "strong" + assert report["score"] < 80 + + +# --- must-fix #1 (allowlist): only a genuine interpreter/script invocation of a +# --- gate .py earns "invoked" credit. grep/cat/ls/test/head/wc/find evade. ----- + +_GATE_EVASION_COMMANDS = ("grep", "cat", "ls", "test -f", "head", "wc -l", "find") + + +def test_non_executing_commands_referencing_gate_py_earn_no_invoked_credit(tmp_path): + # must-fix #1 / D(i): grep/cat/ls/test/head/wc/find that merely *reference* the + # gate .py (read/list/search it) never *execute* it — ZERO invoked credit. + # A real gate file is present so the only thing under test is the invocation + # shape: a non-executing reference must not upgrade to "invoked". + for i, cmd in enumerate(_GATE_EVASION_COMMANDS): + loop = tmp_path / f"evade{i}" + (loop / "scripts").mkdir(parents=True) + (loop / "scripts" / "holdout_gate.py").write_text("# gate\n", encoding="utf-8") + (loop / "scripts" / "verify-fast").write_text( + f"#!/bin/sh\n{cmd} scripts/holdout_gate.py\n", encoding="utf-8" + ) + # Unit: the invocation detector rejects the non-executing reference. + assert il._gate_invoked_in_verify(loop) is False, cmd + # Report: no "(invoked)" credit is shown for the false-completion defense. + report = il.inspect_loop(str(loop)) + present = " ".join(report["present"]).lower() + assert "(invoked)" not in present, cmd + + +def test_grep_evasion_literal_m3_fake_scores_below_strong(tmp_path): + # must-fix #1 acceptance: the literal M3 fake with `echo` swapped for `grep` + # (no real gate file, only a grep reference in verify-fast) scored 88/strong + # before the fix. grep earns no invoked credit and, with no gate file, the + # defense grades "none" → the score cap holds it below strong. + fake = tmp_path / "grepfake" + (fake / ".loop").mkdir(parents=True) + (fake / "scripts").mkdir() + (fake / "SPEC.md").write_text( + "# SPEC\n## Success criteria\n1. coverage >= 80% (scripts/verify-full)\n", + encoding="utf-8", + ) + (fake / "WORKFLOW.md").write_text( + "# WORKFLOW\n## Approval Gates\nApproval gate on side-effects.\n" + "## Plan-then-execute\nPlan-then-execute for untrusted reads.\n" + "## Terminal States\n" + "Succeeded, FailedUnverifiable, FailedBlocked, FailedBudget, " + "FailedSafety, FailedSpecGap, AbortedByHuman.\n", + encoding="utf-8", + ) + (fake / "TASKS.json").write_text( + json.dumps({"tasks": [{"id": "T1", "title": "ship it", + "verify": "scripts/verify-fast"}]}), + encoding="utf-8", + ) + (fake / "scripts" / "verify-fast").write_text( + '#!/bin/sh\ngrep "holdout_gate.py anticheat_scan" ./SPEC.md\n', + encoding="utf-8", + ) + + report = il.inspect_loop(str(fake)) + present = " ".join(report["present"]).lower() + assert "(invoked)" not in present + assert report["verdict"] != "strong" + assert report["score"] < 80 + + +def test_allowlisted_invocations_keep_invoked_credit(tmp_path): + # must-fix #1 / D(iii): the allowlisted true positives keep FULL credit. + invocations = ( + "python3 scripts/holdout_gate.py\n", + "bash scripts/anticheat_scan.py\n", + "./scripts/holdout_gate.py\n", + "uv run python scripts/anticheat_scan.py\n", + ) + for i, body in enumerate(invocations): + loop = tmp_path / f"real{i}" + (loop / "scripts").mkdir(parents=True) + (loop / "scripts" / "verify-fast").write_text( + "#!/bin/sh\n" + body, encoding="utf-8" + ) + (loop / "scripts" / "holdout_gate.py").write_text("# gate\n", encoding="utf-8") + (loop / "scripts" / "anticheat_scan.py").write_text("# gate\n", encoding="utf-8") + assert il._gate_invoked_in_verify(loop) is True, body + + +def test_versioned_python_interpreter_invocation_counts(tmp_path): + # must-fix #1: `python3.12 scripts/holdout_gate.py` is a genuine invocation. + loop = tmp_path / "pyver" + (loop / "scripts").mkdir(parents=True) + (loop / "scripts" / "verify-fast").write_text( + "#!/bin/sh\npython3.12 scripts/holdout_gate.py --strict\n", encoding="utf-8" + ) + (loop / "scripts" / "holdout_gate.py").write_text("# gate\n", encoding="utf-8") + assert il._gate_invoked_in_verify(loop) is True + + +# --- must-fix #2 (score cap): "strong" is unreachable without a real defense --- + + +def test_stuffed_fake_with_no_defense_cannot_reach_strong(tmp_path): + # must-fix #2 / D(ii): a stuffed fake — real-looking success criteria, an + # echo-only verify-fast, prose approval/plan headings, all 7 terminal states, + # but NO real gate files — must score < 80 and verdict != "strong". Pure + # keyword stuffing with zero false-completion defense cannot buy a strong + # verdict (or clear the fail-under-80 CI gate). + fake = tmp_path / "stuffed" + (fake / ".loop").mkdir(parents=True) + (fake / "scripts").mkdir() + (fake / "SPEC.md").write_text( + "# SPEC\n## Success criteria\n" + "1. unit + integration coverage >= 80% (scripts/verify-full)\n" + "2. zero lint errors (scripts/verify-fast)\n" + "## Evidence Rules\nEach criterion maps to a scripts/verify-* command.\n", + encoding="utf-8", + ) + (fake / "WORKFLOW.md").write_text( + "# WORKFLOW\n## Approval Gates\nPause on destructive / secret / production.\n" + "## Plan-then-execute\nPrecommit the execution graph for untrusted reads.\n" + "## Terminal States\n" + "Succeeded, FailedUnverifiable, FailedBlocked, FailedBudget, " + "FailedSafety, FailedSpecGap, AbortedByHuman.\n", + encoding="utf-8", + ) + (fake / "scripts" / "verify-fast").write_text( + '#!/bin/sh\necho "holdout_gate anticheat_scan all clean"\n', encoding="utf-8" + ) + + report = il.inspect_loop(str(fake)) + present = " ".join(report["present"]).lower() + gaps = " ".join(report["gaps"]).lower() + + assert report["verdict"] != "strong" + assert report["score"] < 80 + # The false-completion defense grades "none" (no invoked/wired gate). + assert "false-completion defense" not in present + # The cap gap explicitly names the reason strong is out of reach. + assert "capped" in gaps and "false-completion defense" in gaps + + +def test_score_cap_only_bites_when_defense_is_none(tmp_path): + # must-fix #2: the cap targets grade "none" only. A genuinely gate-backed loop + # ("invoked") is never capped — the flagship keeps its full score. + root = pathlib.Path(__file__).resolve().parent.parent + report = il.inspect_loop(str(root / "examples" / "coverage-repair")) + gaps = " ".join(report["gaps"]).lower() + assert report["verdict"] == "strong" + assert report["score"] >= 80 + assert "capped" not in gaps + + +# --- must-fix #3 (verify substance): an echo-only verify-* body is not proof ---- + + +def test_echo_only_verify_script_earns_no_independent_verification(tmp_path): + # must-fix #3 / failure #3: a file merely NAMED scripts/verify-fast whose whole + # body is echo earns ZERO independent_verification credit — printing is not + # verifying. (No resolving task.verify, no SPEC verify reference.) + loop = tmp_path / "echoverify" + (loop / "scripts").mkdir(parents=True) + (loop / "scripts" / "verify-fast").write_text( + '#!/bin/sh\necho "checking..."\necho "all good"\nexit 0\n', encoding="utf-8" + ) + assert il._verify_script_has_substance(loop.resolve()) is False + report = il.inspect_loop(str(loop)) + present = " ".join(report["present"]).lower() + gaps = " ".join(report["gaps"]).lower() + assert "independent verification" not in present + assert "no independent verification" in gaps + + +def test_substantive_verify_script_earns_independent_verification(tmp_path): + # must-fix #3: a verify-* script with a real executable check (an existence + # loop) earns the credit — the substance gate is not a blanket denial. + loop = tmp_path / "realverify" + (loop / "scripts").mkdir(parents=True) + (loop / "scripts" / "verify-fast").write_text( + "#!/bin/sh\n" + 'for f in SPEC.md WORKFLOW.md; do\n' + ' [ -f "$1/$f" ] || exit 1\n' + "done\n" + 'echo "PASS"\n', + encoding="utf-8", + ) + assert il._verify_script_has_substance(loop.resolve()) is True + report = il.inspect_loop(str(loop)) + present = " ".join(report["present"]).lower() + assert "independent verification" in present + + +def test_fresh_scaffold_keeps_independent_verification_credit(tmp_path): + # must-fix #3 invariant: the shipped scaffold's verify-fast has a real for-loop + # existence check, so a fresh scaffold KEEPS independent_verification credit. + from loop.scaffold import scaffold + + target = tmp_path / "fresh-scaffold-verify" + scaffold(target) + assert il._verify_script_has_substance(target.resolve()) is True + report = il.inspect_loop(str(target)) + present = " ".join(report["present"]).lower() + assert "independent verification" in present + + +# --- re-verify round 2: non-executing content must not launder "invoked", ---- +# --- and genuine wrapped/uv-run invocations must keep it ---------------------- + + +def _rv2_loop(tmp_path, name, verify_line, gate_file=False): + loop = tmp_path / name + (loop / ".loop").mkdir(parents=True) + (loop / "scripts").mkdir() + (loop / "scripts" / "verify-fast").write_text( + f"#!/bin/sh\n{verify_line}\n", encoding="utf-8" + ) + if gate_file: + (loop / "scripts" / "holdout_gate.py").write_text("# gate\n", encoding="utf-8") + return loop + + +def test_trailing_comment_naming_gate_earns_no_invoked_credit(tmp_path): + # rv2 blocker 1: a gate named only in a trailing shell comment is never run — + # the actual command is a no-op interpreter call. + for i, line in enumerate([ + "python3 -V # scripts/holdout_gate.py anti-cheat", + "python3 --version # holdout_gate.py", + ]): + loop = _rv2_loop(tmp_path, f"cmt{i}", line, gate_file=True) + assert il._gate_invoked_in_verify(loop) is False, line + + +def test_redirection_sink_naming_gate_earns_no_invoked_credit(tmp_path): + # rv2 blocker 1: a gate path that is only a redirect target is never executed. + for i, line in enumerate([ + "python3 realwork.py > scripts/holdout_gate.py", + "python3 realwork.py 2>scripts/holdout_gate.py", + ]): + loop = _rv2_loop(tmp_path, f"redir{i}", line, gate_file=True) + assert il._gate_invoked_in_verify(loop) is False, line + + +def test_uv_non_run_subcommands_earn_no_invoked_credit(tmp_path): + # rv2 blocker 1: only `uv run` executes; pip install / add merely reference. + for i, line in enumerate([ + "uv pip install scripts/holdout_gate.py", + "uv add scripts/holdout_gate.py", + ]): + loop = _rv2_loop(tmp_path, f"uv{i}", line, gate_file=True) + assert il._gate_invoked_in_verify(loop) is False, line + runs = _rv2_loop(tmp_path, "uvrun", "uv run scripts/holdout_gate.py", gate_file=True) + assert il._gate_invoked_in_verify(runs) is True + + +def test_invoked_requires_gate_file_for_resolvable_paths(tmp_path): + # rv2 blocker 3: a genuine invocation SHAPE naming a workspace-relative gate + # that does not exist earns nothing — "invoked" (full credit) must not have a + # weaker precondition than "wired" (half credit, which checks existence). + ghost = _rv2_loop(tmp_path, "ghost", "python3 scripts/holdout_gate.py") + assert il._gate_invoked_in_verify(ghost) is False + report = il.inspect_loop(str(ghost)) + assert report["score"] < 80 + assert report["verdict"] != "strong" + + real = _rv2_loop(tmp_path, "real", "python3 scripts/holdout_gate.py", gate_file=True) + assert il._gate_invoked_in_verify(real) is True + + +def test_unresolvable_gate_paths_keep_shape_credit(tmp_path): + # rv2 blocker 3 counter-case: a $VAR path cannot be checked statically — the + # flagship's `python3 "$REPO/scripts/holdout_gate.py"` (gate lives at repo + # root, outside the example workspace) must keep credit. + loop = _rv2_loop(tmp_path, "var", 'python3 "$REPO/scripts/holdout_gate.py"') + assert il._gate_invoked_in_verify(loop) is True + + +def test_by_path_invocation_requires_gate_file(tmp_path): + ghost = _rv2_loop(tmp_path, "bypath", "./scripts/holdout_gate.py") + assert il._gate_invoked_in_verify(ghost) is False + real = _rv2_loop(tmp_path, "bypath2", "./scripts/holdout_gate.py", gate_file=True) + assert il._gate_invoked_in_verify(real) is True + + +def test_wrapper_prefixed_real_invocations_keep_credit(tmp_path): + # rv2 note: honest wrapper prefixes are transparent, not disqualifying. + for i, line in enumerate([ + "time python3 scripts/holdout_gate.py", + "env python3 scripts/holdout_gate.py", + "nohup python3 scripts/holdout_gate.py", + ]): + loop = _rv2_loop(tmp_path, f"wrap{i}", line, gate_file=True) + assert il._gate_invoked_in_verify(loop) is True, line + + +def test_runlog_common_english_prose_earns_no_recorded_credit(tmp_path): + # rv2 blocker 2: narrative sentences whose only "run-word" is ordinary English + # (passed/ran/result/clean) are self-narration, not gate records. + for i, line in enumerate([ + "Iteration 3: the review deadline passed before we could wire a real holdout_gate.", + "as a result, holdout_gate is our chosen plan", + "we ran out of time to wire holdout_gate", + "keep the holdout_gate design clean", + ]): + loop = tmp_path / f"prose{i}" + (loop / ".loop").mkdir(parents=True) + (loop / "RUNLOG.md").write_text(line + "\n", encoding="utf-8") + paths = il.resolve_loop_paths(loop) + assert il._gate_run_recorded(paths) is False, line + + +def _rv2_runlog_loop(tmp_path, name, line, gate_file=True): + loop = tmp_path / name + (loop / ".loop").mkdir(parents=True) + (loop / "RUNLOG.md").write_text(line + "\n", encoding="utf-8") + if gate_file: + (loop / "scripts").mkdir() + (loop / "scripts" / "holdout_gate.py").write_text("# gate\n", encoding="utf-8") + return il.resolve_loop_paths(loop) + + +def test_runlog_record_requires_script_path_not_bare_token(tmp_path): + # rv2 blocker 2: RUNLOG credit needs the actual .py path plus a verdict word — + # a bare token with prose around it is not a record (gate file present, so + # the .py-path requirement is the discriminating layer). + bare = _rv2_runlog_loop(tmp_path, "baretok", "holdout_gate verdict: pass") + assert il._gate_run_recorded(bare) is False + + real = _rv2_runlog_loop( + tmp_path, "realrec", "gate: scripts/holdout_gate.py -> verdict PASS (0 flagged)" + ) + assert il._gate_run_recorded(real) is True + + +def test_runlog_record_requires_gate_script_on_disk(tmp_path): + # rv3: a record-shaped line about a gate that exists nowhere on disk is a + # claim about a tool that does not exist — no credit. + ghost = _rv2_runlog_loop( + tmp_path, "ghostrec", + "gate: scripts/holdout_gate.py -> verdict PASS (0 flagged)", gate_file=False, + ) + assert il._gate_run_recorded(ghost) is False + + +def test_substring_run_words_earn_no_recorded_credit(tmp_path): + # rv3: verdict words match whole words only — cleanup/passphrase/surpassed + # must never satisfy the bar the way clean/pass do (gate file present, .py + # path named: the word boundary is the only discriminating layer). + for i, line in enumerate([ + "We designed scripts/holdout_gate.py but ran out of time; cleanup pending.", + "blocked: scripts/holdout_gate.py needs a passphrase we do not have yet.", + "scripts/holdout_gate.py budget surpassed this iteration.", + ]): + paths = _rv2_runlog_loop(tmp_path, f"substr{i}", line) + assert il._gate_run_recorded(paths) is False, line + + +def test_substring_run_words_in_receipts_earn_no_credit(tmp_path): + # rv3: the receipt path shares the word-boundary bar. + loop = tmp_path / "rcsub" + (loop / ".loop" / "receipts").mkdir(parents=True) + (loop / "scripts").mkdir() + (loop / "scripts" / "holdout_gate.py").write_text("# gate\n", encoding="utf-8") + (loop / ".loop" / "receipts" / "run.jsonl").write_text( + json.dumps({"note": "will add holdout_gate later", "status": "surpassed budget"}) + + "\n", + encoding="utf-8", + ) + assert il._gate_run_recorded(il.resolve_loop_paths(loop)) is False + + +def test_compound_redirect_sinks_earn_no_invoked_credit(tmp_path): + # rv3: `>&` / `>|` / `&>` contain segment-split characters — the severed sink + # must not read as a bare-path gate invocation. + for i, line in enumerate([ + "python3 realwork.py >& scripts/holdout_gate.py", + "python3 realwork.py >| scripts/holdout_gate.py", + "python3 realwork.py &> scripts/holdout_gate.py", + ]): + loop = _rv2_loop(tmp_path, f"cmpd{i}", line, gate_file=True) + assert il._gate_invoked_in_verify(loop) is False, line + + +def test_real_invocation_with_fd_redirect_keeps_credit(tmp_path): + # rv3 counter-case: stripping redirections must not strip the command. + loop = _rv2_loop( + tmp_path, "fdredir", "python3 scripts/holdout_gate.py 2>&1", gate_file=True + ) + assert il._gate_invoked_in_verify(loop) is True diff --git a/scripts/test_scaffold.py b/scripts/test_scaffold.py index 8bbd4ac..6278b44 100644 --- a/scripts/test_scaffold.py +++ b/scripts/test_scaffold.py @@ -7,6 +7,7 @@ from __future__ import annotations +import importlib.util import json import pathlib import subprocess @@ -16,6 +17,12 @@ ROOT = pathlib.Path(__file__).resolve().parent.parent +_il_spec = importlib.util.spec_from_file_location( + "il", pathlib.Path(__file__).parent / "inspect_loop.py" +) +il = importlib.util.module_from_spec(_il_spec) +_il_spec.loader.exec_module(il) + def test_scaffolded_contract_passes_doctor_unedited(tmp_path): from loop.contract import validate_contract @@ -127,6 +134,31 @@ def test_missing_terminal_is_an_issue_when_state_declares_a_terminal(tmp_path): ) +def test_fresh_scaffold_is_doctor_clean_but_inspects_below_strong(tmp_path): + """The scaffold is a valid in-flight contract (doctor-clean) yet a fresh, + unedited one is not a *strong* loop: its success criteria and task titles are + still the literal `REPLACE:` placeholders. Doctor stays green; the inspector + must dock defines_success and flag the placeholder convention so a stranger + knows the loop is a shell to fill, not a finished proof.""" + from loop.contract import validate_contract + from loop.scaffold import scaffold + + target = tmp_path / "fresh" + scaffold(target) + + # Doctor is unchanged — a fresh scaffold is a valid in-flight contract. + assert validate_contract(target)["ok"] is True + + report = il.inspect_loop(str(target)) + gaps = " ".join(report["gaps"]).lower() + present = " ".join(report["present"]).lower() + + assert report["verdict"] != "strong" + assert report["score"] < 80 + assert "defines verifiable success criteria" not in present + assert "replace" in gaps and "placeholder" in gaps + + def test_scaffold_cli_subcommand(tmp_path): target = tmp_path / "cli-loop" result = subprocess.run(