diff --git a/docs/developer-guide.md b/docs/developer-guide.md index cc82ee36..61716533 100644 --- a/docs/developer-guide.md +++ b/docs/developer-guide.md @@ -346,7 +346,7 @@ crates/ sysknife-brain/ LLM planner, provider adapters, safety fence sysknife-types/ Shared domain types (CallerRole, RiskLevel, JobState, …) sysknife-core/ Config loading, shared constants - sysknife-daemon/ Privileged executor, 189 actions with an `ActionSpec`, + sysknife-daemon/ Privileged executor, 191 actions with an `ActionSpec`, IPC, rollback, SQLite sysknife-proto/ Protobuf definitions (future use) apps/ diff --git a/scripts/check_evidence_claims.py b/scripts/check_evidence_claims.py index 8bf99e9d..d8d380dc 100755 --- a/scripts/check_evidence_claims.py +++ b/scripts/check_evidence_claims.py @@ -647,7 +647,21 @@ def check_bare_story_counts( return problems -def check_action_figures(texts: dict[str, str], catalogue: int) -> list[str]: +def count_action_specs(root: Path) -> int: + """Derive the executor ActionSpec count from the generated reference table.""" + path = root / "docs/action-reference.md" + if not path.exists(): + raise Failure("docs/action-reference.md is missing; cannot derive ActionSpec count") + row = re.compile(r"^\| `[A-Za-z0-9_]+` \|") + count = sum(1 for line in path.read_text(encoding="utf-8").splitlines() if row.match(line)) + if count == 0: + raise Failure("docs/action-reference.md contains no generated ActionSpec rows") + return count + + +def check_action_figures( + texts: dict[str, str], catalogue: int, action_specs: int +) -> list[str]: """Catch a bare "N actions" that is not the catalogue size. `check_figure` only sees the exact noun it is given ("typed actions"), so @@ -664,11 +678,22 @@ def check_action_figures(texts: dict[str, str], catalogue: int) -> list[str]: drifting. """ bare = re.compile(r"\b([0-9]{2,})\s+(?:typed\s+)?actions\b", re.IGNORECASE) + subset = re.compile( + r"\b([0-9]+)\s+actions\s+(?:with|have)\s+an\s+`?ActionSpec`?\b", + re.IGNORECASE, + ) problems = [] for rel, text in texts.items(): for line in text.splitlines(): - if "ActionSpec" in line: + subset_match = subset.search(line) + if subset_match: + count = int(subset_match.group(1)) + if count != action_specs: + problems.append( + f"{rel}: claims {count} actions with an ActionSpec, " + f"derived {action_specs} from docs/action-reference.md" + ) continue for match in bare.finditer(line): count = int(match.group(1)) @@ -806,7 +831,7 @@ def main() -> int: problems += check_debian_only_prose_claims(texts, root) problems += check_bare_story_counts(texts, story_runs, root) problems += check_validated_tiers(texts, root) - problems += check_action_figures(texts, count_actions(root)) + problems += check_action_figures(texts, count_actions(root), count_action_specs(root)) expected_tests = f"{baseline['tests']:,} Rust tests" for rel in REQUIRE_TEST_COUNT: diff --git a/tests/release/public-claims.test.sh b/tests/release/public-claims.test.sh index 75b875a3..f038bcc2 100755 --- a/tests/release/public-claims.test.sh +++ b/tests/release/public-claims.test.sh @@ -700,6 +700,58 @@ sed -i "s/${actions} typed actions/999 typed actions/" "$fixture/docs/introducti assert_rejected 'action count that disagrees with the catalogue source' cp "$repo_root/docs/introduction.md" "$fixture/docs/introduction.md" +# An ActionSpec qualifier must not exempt a stale subset count from evidence. +python3 - "$fixture/docs/developer-guide.md" <<'PYEOF' +import re +import sys +from pathlib import Path +path = Path(sys.argv[1]) +text = path.read_text(encoding="utf-8") +updated, count = re.subn( + r"[0-9]+ actions with an `ActionSpec`", + "4 actions with an `ActionSpec`", + text, + count=1, +) +if count != 1: + raise SystemExit("could not find the ActionSpec count in the fixture") +path.write_text(updated, encoding="utf-8") +PYEOF +assert_rejected_with_diagnostic \ + 'ActionSpec count that disagrees with the generated table' \ + 'developer-guide.md' 'claims 4 actions with an ActionSpec' 'derived' +cp "$repo_root/docs/developer-guide.md" "$fixture/docs/developer-guide.md" + +# The generated-table reader must fail closed on an empty table and count rows exactly. +python3 - "$repo_root/scripts/check_evidence_claims.py" <<'PYEOF' +import importlib.util +import tempfile +from pathlib import Path +import sys +spec = importlib.util.spec_from_file_location("checker", sys.argv[1]) +mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mod) +with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + docs = root / "docs" + docs.mkdir() + ref = docs / "action-reference.md" + ref.write_text( + "| Action | Command |\n|---|---|\n" + "| `One` | `one` |\n| `Two2` | `two` |\n| `Three_3` | `three` |\n", + encoding="utf-8", + ) + if mod.count_action_specs(root) != 3: + raise SystemExit("ActionSpec row fixture did not derive exactly 3") + ref.write_text("| Action | Command |\n|---|---|\n", encoding="utf-8") + try: + mod.count_action_specs(root) + except mod.Failure: + pass + else: + raise SystemExit("empty ActionSpec table did not fail closed") +PYEOF + # No evidence at all must fail loudly rather than pass for lack of anything to # compare against. mv "$fixture/tests/evidence/workspace-tests.json" "$fixture/tests/evidence/held.json"