fix(ci): run the tests the badge claims to run

.github/ci/guard.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+"""CI guard: prove the suite actually ran, and that it still runs what it claims.
+
+Two failure modes, both of which shipped in this portfolio and neither of which
+an ordinary green build catches:
+
+1. CI reports success without executing a single assertion. `uv run <file>`
+   did exactly this for weeks — every suite imported, defined its test
+   functions, and exited 0. A `--collect-only` count cannot detect it, because
+   collection is a separate process from the run: revert the run step to the
+   broken form and the count is still right. So this reads the JUnit report the
+   run step itself produced. No report means pytest never ran.
+
+2. The advertised test count is true only on the author's machine. Suites
+   gated on gitignored artifacts skip wholesale in a clean checkout while
+   collection still counts them, so "1002 tests" quietly becomes 835 for
+   everyone else. That is why there are two floors: `collected` guards against
+   tests disappearing, `executed` guards against them silently going dark.
+
+Usage: guard.py <junit.xml> <collected-floor> <executed-floor>
+"""
+import sys
+import xml.etree.ElementTree as ET
+
+report = sys.argv[1]
+collected_floor = int(sys.argv[2])
+executed_floor = int(sys.argv[3])
+
+try:
+    root = ET.parse(report).getroot()
+except (OSError, ET.ParseError) as exc:
+    sys.exit(f"::error::no JUnit report at {report} ({exc}) — pytest did not run")
+
+suites = root.findall("testsuite") or [root]
+collected = sum(int(s.get("tests", 0)) for s in suites)
+skipped = sum(int(s.get("skipped", 0)) for s in suites)
+bad = sum(int(s.get("failures", 0)) + int(s.get("errors", 0)) for s in suites)
+executed = collected - skipped
+
+print(f"run reported: {collected} collected, {executed} executed, {skipped} skipped, {bad} failed/errored")
+print(f"floors:       {collected_floor} collected, {executed_floor} executed")
+
+if bad:
+    sys.exit(f"::error::{bad} test(s) failed or errored")
+if executed == 0:
+    sys.exit("::error::0 tests executed — the vacuous-CI mode this guard exists to catch")
+if collected < collected_floor:
+    sys.exit(f"::error::{collected} collected, expected >= {collected_floor} — tests have disappeared")
+if executed < executed_floor:
+    sys.exit(
+        f"::error::{executed} executed, expected >= {executed_floor} — "
+        f"{skipped} skipped; a suite has gone dark rather than failing"
+    )
+print("ok")

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-# Runs every offline test suite (test_*.py) on push to main and on PRs.
+# Runs every offline test suite on push to main and on PRs.
 # The suites need no API key and make no paid calls — that's by design;
 # they gate everything that does spend (see README "How to re-run").
 name: CI
@@ -16,15 +16,18 @@ jobs:
       - uses: astral-sh/setup-uv@v8.3.0  # no floating v8 major tag exists
         with:
           enable-cache: true
-      - name: Run all offline test suites
-        run: |
-          rc=0
-          for f in test_*.py; do
-            echo "::group::$f"
-            if ! uv run "$f"; then
-              echo "FAILED: $f"
-              rc=1
-            fi
-            echo "::endgroup::"
-          done
-          exit $rc
+
+      # `uv run <file>` executes a module as a plain script. These suites are bare
+      # `def test_*` collections with no `__main__` entrypoint, so that form imported
+      # each file and exited 0 without running a single assertion — CI passed
+      # vacuously under a green badge until the portfolio's 2026-08-07 audit caught
+      # it. pytest has to be the thing invoked.
+      - name: Run the offline suites
+        run: uv run pytest -q --junit-xml=junit.xml
+
+      # Reads the report the run step itself produced, so it can tell "pytest ran
+      # and executed N tests" from "pytest was never invoked" — a distinction a
+      # separate --collect-only count cannot make.
+      - name: Guard — the suites actually ran, and still have their tests
+        if: always()
+        run: python3 .github/ci/guard.py junit.xml 256 256

.gitignore

@@ -12,3 +12,4 @@ __pycache__/
 
 # fetched raw data (refetchable; the pre-committed filtered bank IS committed)
 data/raw/
+junit.xml

sandbox.py

@@ -13,6 +13,7 @@
 from __future__ import annotations
 
 import json
+import os
 import subprocess
 import tempfile
 import uuid
@@ -80,6 +81,19 @@ def run_tests(
         tdir.mkdir()
         for i, s in enumerate(inputs):
             (tdir / f"in_{i}.txt").write_text(s)
+        # The container runs as `nobody` (65534) against a read-only bind mount,
+        # but TemporaryDirectory() is 0700 and host-user-owned. On Linux — which
+        # includes GitHub's runners — that makes /work unreadable to the container
+        # user, and every run dies with `[Errno 13] Permission denied` before
+        # emitting a verdict. macOS's Docker mount hides this by not mapping host
+        # ownership through. Widen the throwaway tree so the unprivileged user can
+        # read it: it lives for one container, is mounted read-only, and holds only
+        # the generated program and its test inputs — no secrets, no expected
+        # outputs (the container never sees those, by design).
+        os.chmod(work, 0o755)
+        os.chmod(tdir, 0o755)
+        for f in (work / "prog.py", work / "_runner.py", *tdir.iterdir()):
+            os.chmod(f, 0o644)
         cmd = [
             "docker", "run", "--rm", "--name", name,
             "--network=none", "--cpus=1", "--memory=512m", "--pids-limit=128",