test: cover the exit-code wiring of the corpus checks

README.md

@@ -19,7 +19,7 @@ by Michel Giehl; see [`NOTICE.md`](NOTICE.md). Not affiliated with, endorsed
 by, or approved by Riot Games.
 
 **Current state:** `cargo +1.86.0 test --workspace --locked` **563 passing**,
-`tools/tests` **481 passing** -- see [Status](#status) for the rest.
+`tools/tests` **503 passing** -- see [Status](#status) for the rest.
 
 - Run it: [`docs/USAGE.md`](docs/USAGE.md)
 - What's extractable: [`docs/DATA.md`](docs/DATA.md)
@@ -281,7 +281,7 @@ it as one gives the year 3626.
 
 Work in progress. Currently verified: `cargo +1.86.0 test --workspace --locked`
 **563 passing**, strict workspace `clippy -D warnings` **0**, `cargo fmt` clean,
-and `check_ascii` on 117 files. The Python suite in `tools/tests` has 481 tests.
+and `check_ascii` on 118 files. The Python suite in `tools/tests` has 503 tests.
 
 Re-measure per-crate counts with `cargo test -p <crate>`. Counts are omitted
 from the table below on purpose -- they go stale, and re-measuring is one line.

docs/USAGE.md

@@ -493,7 +493,7 @@ records the fields that failed the bar and why.
 | `compare_rpc_params.py` | RPC parameter comparison |
 | `compare_with_csharp.py` | Diff against the C# parser |
 | `check_effect_decoder.py` | Effect decoder (12 cases) |
-| `check_ascii.py` | Rust source ASCII sweep (117 files) |
+| `check_ascii.py` | Rust source ASCII sweep (118 files) |
 | `check_docs.py` | This document itself (below) |
 | `atomic_io.py` | Internal containment, recursive-removal and atomic-replacement helpers shared by mutating tools |
 
@@ -602,9 +602,9 @@ deliberately sequential for accuracy.
 cargo +1.86.0 test --workspace --locked                              # 563 passing
 cargo +1.86.0 clippy --workspace --all-targets --all-features --locked -- -D warnings
 cargo +1.86.0 fmt --check
-python -W error tools/check_ascii.py --check                         # 117 files
+python -W error tools/check_ascii.py --check                         # 118 files
 python -W error tools/check_effect_decoder.py --check                # 12 cases
-python -W error -m unittest discover -s tools/tests -p "test_*.py"   # 481 passing
+python -W error -m unittest discover -s tools/tests -p "test_*.py"   # 503 passing
 python -W error tools/check_docs.py --fast
 python -W error tools/apply_type_corrections.py --check              # 133 corrections
 python -W error tools/extract_checksum_types.py --export tools/fixtures/checksum_export --check

tools/tests/test_check_decode_errors_corpus.py

@@ -19,7 +19,11 @@
 the numbers add up without going to read the Rust source. See `LIVE_EXPORT`
 and `ReconcileTests`.
 """
+import contextlib
+import io
+import os
 import sys
+import tempfile
 import unittest
 from pathlib import Path
 
@@ -317,5 +321,182 @@ def test_a_corpus_where_no_checkpoint_blob_decoded_is_not_a_pass(self):
         self.assertTrue(dead)
 
 
+#: Stand-in for `vrfkit.exe`, playing the part `_export_one` expects --
+#: `[str(exe), "export", str(replay), "--out", str(out)]`. Run under
+#: `sys.executable`, the "export" token becomes the script Python executes (the
+#: same trick `test_check_export_baseline.py` uses for its fake `export`), so a
+#: file literally named `export` in the process's cwd stands in for the real
+#: binary. Every helper above (`read_counters`, `dead_counters`, `reconcile`)
+#: is proven correct on synthetic text; none of that proves `main()` actually
+#: calls them and acts on what they return -- which is exactly the shape of
+#: this file's own recorded defect ("OK: every replay reported Decode errors:
+#: 0" printed over an exporter that never ran). These tests are that call.
+FAKE_EXPORT_SCRIPT = '''\
+import sys
+from pathlib import Path
+
+argv = sys.argv
+replay = Path(argv[1])
+out = Path(argv[argv.index("--out") + 1])
+out.mkdir(parents=True, exist_ok=True)
+name = replay.name
+
+if "badexit" in name:
+    print("exporter crashed", file=sys.stderr)
+    raise SystemExit(9)
+
+if "nothingran" in name:
+    # The 13.02 shape one level down: every counter a legitimate zero.
+    print("""
+Rows offered:      0
+Decoded OK:        0
+Decode errors:     0
+Raw/Skip:          0
+Not in table:      0
+No field name:     0
+Struct blobs:      0 decoded / 0 failed
+""")
+    raise SystemExit(0)
+
+if "decodeerr" in name:
+    print("""
+Rows offered:      100
+Decoded OK:        90
+Decode errors:     10
+Raw/Skip:          0
+Not in table:      0
+No field name:     0
+Struct blobs:      5 decoded / 0 failed
+""")
+    raise SystemExit(0)
+
+if "blobfail" in name:
+    print("""
+Rows offered:      100
+Decoded OK:        95
+Decode errors:     0
+Raw/Skip:          3
+Not in table:      2
+No field name:     0
+Struct blobs:      5 decoded / 1 failed
+""")
+    raise SystemExit(0)
+
+if "missingcounter" in name:
+    # "No field name" omitted entirely -- must not read as 0.
+    print("""
+Rows offered:      100
+Decoded OK:        100
+Decode errors:     0
+Raw/Skip:          0
+Not in table:      0
+Struct blobs:      5 decoded / 0 failed
+""")
+    raise SystemExit(0)
+
+if "mismatch" in name:
+    # Every REQUIRED counter present, but the five categories that make up
+    # "Rows offered" undercount it by one -- summary.rs grew a sixth category
+    # this tool does not know to parse yet.
+    print("""
+Rows offered:      100
+Decoded OK:        90
+Decode errors:     0
+Raw/Skip:          5
+Not in table:      3
+No field name:     1
+Struct blobs:      5 decoded / 0 failed
+""")
+    raise SystemExit(0)
+
+print("""
+Rows offered:      100
+Decoded OK:        90
+Decode errors:     0
+Raw/Skip:          5
+Not in table:      3
+No field name:     2
+Struct blobs:      5 decoded / 0 failed
+""")
+'''
+
+
+class MainWiringTests(unittest.TestCase):
+    def setUp(self):
+        self._tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self._tmp.cleanup)
+        self.root = Path(self._tmp.name)
+        (self.root / "export").write_text(FAKE_EXPORT_SCRIPT, encoding="utf-8")
+        self.corpus = self.root / "corpus"
+        self.corpus.mkdir()
+        self._previous_cwd = Path.cwd()
+        os.chdir(self.root)
+        self.addCleanup(os.chdir, self._previous_cwd)
+        self._argv = sys.argv
+
+    def make_replay(self, name: str) -> None:
+        (self.corpus / name).write_bytes(b"not a real replay")
+
+    def run_main(self, extra_args=()):
+        argv = [sys.executable, str(self.corpus), "--jobs", "1", *extra_args]
+        sys.argv = ["check_decode_errors_corpus.py", *argv]
+        out = io.StringIO()
+        try:
+            with contextlib.redirect_stdout(out), contextlib.redirect_stderr(out):
+                code = guard.main()
+        finally:
+            sys.argv = self._argv
+        return code, out.getvalue()
+
+    def test_a_clean_corpus_exits_zero(self):
+        self.make_replay("a.vrf")
+        code, output = self.run_main()
+        self.assertEqual(code, 0, output)
+        self.assertIn("OK:", output)
+
+    def test_decode_errors_fail_the_run(self):
+        self.make_replay("decodeerr.vrf")
+        code, output = self.run_main()
+        self.assertEqual(code, 1, output)
+        self.assertIn("decode errors", output)
+
+    def test_struct_blob_failures_fail_the_run(self):
+        self.make_replay("blobfail.vrf")
+        code, output = self.run_main()
+        self.assertEqual(code, 1, output)
+        self.assertIn("struct-blob", output)
+
+    def test_an_exporter_that_decoded_nothing_fails_the_run(self):
+        """The 13.02 shape, one script down from the Rust regression: every
+        counter is a legitimate zero, `Decode errors: 0` is vacuously true,
+        and only `dead_counters` -- consulted by `main()` -- can catch it."""
+        self.make_replay("nothingran.vrf")
+        code, output = self.run_main()
+        self.assertEqual(code, 1, output)
+        self.assertIn("never moved", output)
+
+    def test_a_missing_required_counter_fails_the_run(self):
+        self.make_replay("missingcounter.vrf")
+        code, output = self.run_main()
+        self.assertEqual(code, 1, output)
+        self.assertIn("did not report the counter", output)
+
+    def test_a_nonzero_exporter_exit_fails_the_run(self):
+        self.make_replay("badexit.vrf")
+        code, output = self.run_main()
+        self.assertEqual(code, 1, output)
+        self.assertIn("did not report the counter", output)
+
+    def test_a_reconciliation_mismatch_fails_the_run(self):
+        self.make_replay("mismatch.vrf")
+        code, output = self.run_main()
+        self.assertEqual(code, 1, output)
+        self.assertIn("do not reconcile", output)
+
+    def test_no_vrf_files_is_a_controlled_failure(self):
+        code, output = self.run_main()
+        self.assertEqual(code, 2, output)
+
+
 if __name__ == "__main__":
     unittest.main()