Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/developer-guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,7 @@ crates/
sysknife-brain/ LLM planner, provider adapters, safety fence
sysknife-types/ Shared domain types (CallerRole, RiskLevel, JobState, …)
sysknife-core/ Config loading, shared constants
sysknife-daemon/ Privileged executor, 189 actions with an `ActionSpec`,
sysknife-daemon/ Privileged executor, 191 actions with an `ActionSpec`,
IPC, rollback, SQLite
sysknife-proto/ Protobuf definitions (future use)
apps/
Expand Down
31 changes: 28 additions & 3 deletions scripts/check_evidence_claims.py
Original file line number Diff line number Diff line change
Expand Up @@ -647,7 +647,21 @@ def check_bare_story_counts(
return problems


def check_action_figures(texts: dict[str, str], catalogue: int) -> list[str]:
def count_action_specs(root: Path) -> int:
"""Derive the executor ActionSpec count from the generated reference table."""
path = root / "docs/action-reference.md"
if not path.exists():
raise Failure("docs/action-reference.md is missing; cannot derive ActionSpec count")
row = re.compile(r"^\| `[A-Za-z0-9_]+` \|")
count = sum(1 for line in path.read_text(encoding="utf-8").splitlines() if row.match(line))
if count == 0:
raise Failure("docs/action-reference.md contains no generated ActionSpec rows")
return count


def check_action_figures(
texts: dict[str, str], catalogue: int, action_specs: int
) -> list[str]:
"""Catch a bare "N actions" that is not the catalogue size.

`check_figure` only sees the exact noun it is given ("typed actions"), so
Expand All @@ -664,11 +678,22 @@ def check_action_figures(texts: dict[str, str], catalogue: int) -> list[str]:
drifting.
"""
bare = re.compile(r"\b([0-9]{2,})\s+(?:typed\s+)?actions\b", re.IGNORECASE)
subset = re.compile(
r"\b([0-9]+)\s+actions\s+(?:with|have)\s+an\s+`?ActionSpec`?\b",
re.IGNORECASE,
)

problems = []
for rel, text in texts.items():
for line in text.splitlines():
if "ActionSpec" in line:
subset_match = subset.search(line)
if subset_match:
count = int(subset_match.group(1))
if count != action_specs:
problems.append(
f"{rel}: claims {count} actions with an ActionSpec, "
f"derived {action_specs} from docs/action-reference.md"
)
continue
for match in bare.finditer(line):
count = int(match.group(1))
Expand Down Expand Up @@ -806,7 +831,7 @@ def main() -> int:
problems += check_debian_only_prose_claims(texts, root)
problems += check_bare_story_counts(texts, story_runs, root)
problems += check_validated_tiers(texts, root)
problems += check_action_figures(texts, count_actions(root))
problems += check_action_figures(texts, count_actions(root), count_action_specs(root))

expected_tests = f"{baseline['tests']:,} Rust tests"
for rel in REQUIRE_TEST_COUNT:
Expand Down
52 changes: 52 additions & 0 deletions tests/release/public-claims.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -700,6 +700,58 @@ sed -i "s/${actions} typed actions/999 typed actions/" "$fixture/docs/introducti
assert_rejected 'action count that disagrees with the catalogue source'
cp "$repo_root/docs/introduction.md" "$fixture/docs/introduction.md"

# An ActionSpec qualifier must not exempt a stale subset count from evidence.
python3 - "$fixture/docs/developer-guide.md" <<'PYEOF'
import re
import sys
from pathlib import Path
path = Path(sys.argv[1])
text = path.read_text(encoding="utf-8")
updated, count = re.subn(
r"[0-9]+ actions with an `ActionSpec`",
"4 actions with an `ActionSpec`",
text,
count=1,
)
if count != 1:
raise SystemExit("could not find the ActionSpec count in the fixture")
path.write_text(updated, encoding="utf-8")
PYEOF
assert_rejected_with_diagnostic \
'ActionSpec count that disagrees with the generated table' \
'developer-guide.md' 'claims 4 actions with an ActionSpec' 'derived'
cp "$repo_root/docs/developer-guide.md" "$fixture/docs/developer-guide.md"

# The generated-table reader must fail closed on an empty table and count rows exactly.
python3 - "$repo_root/scripts/check_evidence_claims.py" <<'PYEOF'
import importlib.util
import tempfile
from pathlib import Path
import sys
spec = importlib.util.spec_from_file_location("checker", sys.argv[1])
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
docs = root / "docs"
docs.mkdir()
ref = docs / "action-reference.md"
ref.write_text(
"| Action | Command |\n|---|---|\n"
"| `One` | `one` |\n| `Two2` | `two` |\n| `Three_3` | `three` |\n",
encoding="utf-8",
)
if mod.count_action_specs(root) != 3:
raise SystemExit("ActionSpec row fixture did not derive exactly 3")
ref.write_text("| Action | Command |\n|---|---|\n", encoding="utf-8")
try:
mod.count_action_specs(root)
except mod.Failure:
pass
else:
raise SystemExit("empty ActionSpec table did not fail closed")
PYEOF

# No evidence at all must fail loudly rather than pass for lack of anything to
# compare against.
mv "$fixture/tests/evidence/workspace-tests.json" "$fixture/tests/evidence/held.json"
Expand Down