diff --git a/.github/ci/guard.py b/.github/ci/guard.py new file mode 100644 index 0000000..76e0da0 --- /dev/null +++ b/.github/ci/guard.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""CI guard: prove the suite actually ran, and that it still runs what it claims. + +Two failure modes, both of which shipped in this portfolio and neither of which +an ordinary green build catches: + +1. CI reports success without executing a single assertion. `uv run ` + did exactly this for weeks — every suite imported, defined its test + functions, and exited 0. A `--collect-only` count cannot detect it, because + collection is a separate process from the run: revert the run step to the + broken form and the count is still right. So this reads the JUnit report the + run step itself produced. No report means pytest never ran. + +2. The advertised test count is true only on the author's machine. Suites + gated on gitignored artifacts skip wholesale in a clean checkout while + collection still counts them, so "1002 tests" quietly becomes 835 for + everyone else. That is why there are two floors: `collected` guards against + tests disappearing, `executed` guards against them silently going dark. + +Usage: guard.py +""" +import sys +import xml.etree.ElementTree as ET + +report = sys.argv[1] +collected_floor = int(sys.argv[2]) +executed_floor = int(sys.argv[3]) + +try: + root = ET.parse(report).getroot() +except (OSError, ET.ParseError) as exc: + sys.exit(f"::error::no JUnit report at {report} ({exc}) — pytest did not run") + +suites = root.findall("testsuite") or [root] +collected = sum(int(s.get("tests", 0)) for s in suites) +skipped = sum(int(s.get("skipped", 0)) for s in suites) +bad = sum(int(s.get("failures", 0)) + int(s.get("errors", 0)) for s in suites) +executed = collected - skipped + +print(f"run reported: {collected} collected, {executed} executed, {skipped} skipped, {bad} failed/errored") +print(f"floors: {collected_floor} collected, {executed_floor} executed") + +if bad: + sys.exit(f"::error::{bad} test(s) failed or errored") +if executed == 0: + sys.exit("::error::0 tests executed — the vacuous-CI mode this guard exists to catch") +if collected < collected_floor: + sys.exit(f"::error::{collected} collected, expected >= {collected_floor} — tests have disappeared") +if executed < executed_floor: + sys.exit( + f"::error::{executed} executed, expected >= {executed_floor} — " + f"{skipped} skipped; a suite has gone dark rather than failing" + ) +print("ok") diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f08b26d..7167ef6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -# Runs every offline test suite (test_*.py) on push to main and on PRs. +# Runs every offline test suite on push to main and on PRs. # The suites need no API key and make no paid calls — that's by design; # they gate everything that does spend (see README "How to re-run"). name: CI @@ -16,15 +16,18 @@ jobs: - uses: astral-sh/setup-uv@v8.3.0 # no floating v8 major tag exists with: enable-cache: true - - name: Run all offline test suites - run: | - rc=0 - for f in test_*.py; do - echo "::group::$f" - if ! uv run "$f"; then - echo "FAILED: $f" - rc=1 - fi - echo "::endgroup::" - done - exit $rc + + # `uv run ` executes a module as a plain script. These suites are bare + # `def test_*` collections with no `__main__` entrypoint, so that form imported + # each file and exited 0 without running a single assertion — CI passed + # vacuously under a green badge until the portfolio's 2026-08-07 audit caught + # it. pytest has to be the thing invoked. + - name: Run the offline suites + run: uv run pytest -q --junit-xml=junit.xml + + # Reads the report the run step itself produced, so it can tell "pytest ran + # and executed N tests" from "pytest was never invoked" — a distinction a + # separate --collect-only count cannot make. + - name: Guard — the suites actually ran, and still have their tests + if: always() + run: python3 .github/ci/guard.py junit.xml 256 256 diff --git a/.gitignore b/.gitignore index 8ababa2..4b0cd95 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ __pycache__/ # fetched raw data (refetchable; the pre-committed filtered bank IS committed) data/raw/ +junit.xml diff --git a/sandbox.py b/sandbox.py index 2f02afd..0425ecf 100644 --- a/sandbox.py +++ b/sandbox.py @@ -13,6 +13,7 @@ from __future__ import annotations import json +import os import subprocess import tempfile import uuid @@ -80,6 +81,19 @@ def run_tests( tdir.mkdir() for i, s in enumerate(inputs): (tdir / f"in_{i}.txt").write_text(s) + # The container runs as `nobody` (65534) against a read-only bind mount, + # but TemporaryDirectory() is 0700 and host-user-owned. On Linux — which + # includes GitHub's runners — that makes /work unreadable to the container + # user, and every run dies with `[Errno 13] Permission denied` before + # emitting a verdict. macOS's Docker mount hides this by not mapping host + # ownership through. Widen the throwaway tree so the unprivileged user can + # read it: it lives for one container, is mounted read-only, and holds only + # the generated program and its test inputs — no secrets, no expected + # outputs (the container never sees those, by design). + os.chmod(work, 0o755) + os.chmod(tdir, 0o755) + for f in (work / "prog.py", work / "_runner.py", *tdir.iterdir()): + os.chmod(f, 0o644) cmd = [ "docker", "run", "--rm", "--name", name, "--network=none", "--cpus=1", "--memory=512m", "--pids-limit=128",