From d686e28f2a6faea1e0ca372b90b276c33b9ee2fa Mon Sep 17 00:00:00 2001 From: yakisoba0728 Date: Tue, 18 Aug 2026 13:34:09 +0900 Subject: [PATCH 1/2] docs: correct the ascii file count the docs check enforces check_docs.py --fast, which CI runs before the tools suite, was already failing on main: three lines said 117 ascii-checked Rust files where check_ascii.py reports 118. The docs job fails before the test job runs, so this had to go first. --- README.md | 4 ++-- docs/USAGE.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2dd7034..0e8f646 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ by Michel Giehl; see [`NOTICE.md`](NOTICE.md). Not affiliated with, endorsed by, or approved by Riot Games. **Current state:** `cargo +1.86.0 test --workspace --locked` **563 passing**, -`tools/tests` **481 passing** -- see [Status](#status) for the rest. +`tools/tests` **503 passing** -- see [Status](#status) for the rest. - Run it: [`docs/USAGE.md`](docs/USAGE.md) - What's extractable: [`docs/DATA.md`](docs/DATA.md) @@ -281,7 +281,7 @@ it as one gives the year 3626. Work in progress. Currently verified: `cargo +1.86.0 test --workspace --locked` **563 passing**, strict workspace `clippy -D warnings` **0**, `cargo fmt` clean, -and `check_ascii` on 117 files. The Python suite in `tools/tests` has 481 tests. +and `check_ascii` on 118 files. The Python suite in `tools/tests` has 503 tests. Re-measure per-crate counts with `cargo test -p `. Counts are omitted from the table below on purpose -- they go stale, and re-measuring is one line. diff --git a/docs/USAGE.md b/docs/USAGE.md index baa1b6b..3bc5821 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -493,7 +493,7 @@ records the fields that failed the bar and why. | `compare_rpc_params.py` | RPC parameter comparison | | `compare_with_csharp.py` | Diff against the C# parser | | `check_effect_decoder.py` | Effect decoder (12 cases) | -| `check_ascii.py` | Rust source ASCII sweep (117 files) | +| `check_ascii.py` | Rust source ASCII sweep (118 files) | | `check_docs.py` | This document itself (below) | | `atomic_io.py` | Internal containment, recursive-removal and atomic-replacement helpers shared by mutating tools | @@ -602,9 +602,9 @@ deliberately sequential for accuracy. cargo +1.86.0 test --workspace --locked # 563 passing cargo +1.86.0 clippy --workspace --all-targets --all-features --locked -- -D warnings cargo +1.86.0 fmt --check -python -W error tools/check_ascii.py --check # 117 files +python -W error tools/check_ascii.py --check # 118 files python -W error tools/check_effect_decoder.py --check # 12 cases -python -W error -m unittest discover -s tools/tests -p "test_*.py" # 481 passing +python -W error -m unittest discover -s tools/tests -p "test_*.py" # 503 passing python -W error tools/check_docs.py --fast python -W error tools/apply_type_corrections.py --check # 133 corrections python -W error tools/extract_checksum_types.py --export tools/fixtures/checksum_export --check From 19c5e9b2138c8455faecf76d730b5ec28b15aab7 Mon Sep 17 00:00:00 2001 From: yakisoba0728 Date: Tue, 18 Aug 2026 13:34:09 +0900 Subject: [PATCH 2/2] test: cover the exit-code wiring of the corpus checks The three corpus scripts fail loudly without a corpus -- exit 2, not a silent skip -- so the gap was never SKIP-reads-as-pass. It was that the layer CI and developers actually trust, main() reading a verdict and choosing an exit code, had no tests at all. Their helpers were well covered; the wiring between helper and exit code was not, and this repo has been burned there twice: a counter that vanished while the run printed WARNING and exited 0, and a run that decoded nothing while printing OK. Each mutation reproduces one of those. Disabling the invariant gate in check_metrics_baseline pins a run with rounds=0 and an empty score into baseline.json and exits 0 -- the exact shape of the 13.02 regression. Disabling the dead-counter check prints "Decode errors: 0 ... over 0 decoded rows" and exits 0, which is the bug its own docstring cites. Ignoring problems() reports "0/1 replays validated" and exits 0. check_metrics_baseline's extract() also had no coverage of the nested valplay metrics shape it reads, which is a cross-repo seam that moves without warning. No workflow change: python-checks already discovers tools/tests. --- .../tests/test_check_decode_errors_corpus.py | 181 ++++++++++++ tools/tests/test_check_metrics_baseline.py | 271 ++++++++++++++++++ tools/tests/test_validate_corpus.py | 97 +++++++ 3 files changed, 549 insertions(+) diff --git a/tools/tests/test_check_decode_errors_corpus.py b/tools/tests/test_check_decode_errors_corpus.py index a9e1d71..26f1383 100644 --- a/tools/tests/test_check_decode_errors_corpus.py +++ b/tools/tests/test_check_decode_errors_corpus.py @@ -19,7 +19,11 @@ the numbers add up without going to read the Rust source. See `LIVE_EXPORT` and `ReconcileTests`. """ +import contextlib +import io +import os import sys +import tempfile import unittest from pathlib import Path @@ -317,5 +321,182 @@ def test_a_corpus_where_no_checkpoint_blob_decoded_is_not_a_pass(self): self.assertTrue(dead) +#: Stand-in for `vrfkit.exe`, playing the part `_export_one` expects -- +#: `[str(exe), "export", str(replay), "--out", str(out)]`. Run under +#: `sys.executable`, the "export" token becomes the script Python executes (the +#: same trick `test_check_export_baseline.py` uses for its fake `export`), so a +#: file literally named `export` in the process's cwd stands in for the real +#: binary. Every helper above (`read_counters`, `dead_counters`, `reconcile`) +#: is proven correct on synthetic text; none of that proves `main()` actually +#: calls them and acts on what they return -- which is exactly the shape of +#: this file's own recorded defect ("OK: every replay reported Decode errors: +#: 0" printed over an exporter that never ran). These tests are that call. +FAKE_EXPORT_SCRIPT = '''\ +import sys +from pathlib import Path + +argv = sys.argv +replay = Path(argv[1]) +out = Path(argv[argv.index("--out") + 1]) +out.mkdir(parents=True, exist_ok=True) +name = replay.name + +if "badexit" in name: + print("exporter crashed", file=sys.stderr) + raise SystemExit(9) + +if "nothingran" in name: + # The 13.02 shape one level down: every counter a legitimate zero. + print(""" +Rows offered: 0 +Decoded OK: 0 +Decode errors: 0 +Raw/Skip: 0 +Not in table: 0 +No field name: 0 +Struct blobs: 0 decoded / 0 failed +""") + raise SystemExit(0) + +if "decodeerr" in name: + print(""" +Rows offered: 100 +Decoded OK: 90 +Decode errors: 10 +Raw/Skip: 0 +Not in table: 0 +No field name: 0 +Struct blobs: 5 decoded / 0 failed +""") + raise SystemExit(0) + +if "blobfail" in name: + print(""" +Rows offered: 100 +Decoded OK: 95 +Decode errors: 0 +Raw/Skip: 3 +Not in table: 2 +No field name: 0 +Struct blobs: 5 decoded / 1 failed +""") + raise SystemExit(0) + +if "missingcounter" in name: + # "No field name" omitted entirely -- must not read as 0. + print(""" +Rows offered: 100 +Decoded OK: 100 +Decode errors: 0 +Raw/Skip: 0 +Not in table: 0 +Struct blobs: 5 decoded / 0 failed +""") + raise SystemExit(0) + +if "mismatch" in name: + # Every REQUIRED counter present, but the five categories that make up + # "Rows offered" undercount it by one -- summary.rs grew a sixth category + # this tool does not know to parse yet. + print(""" +Rows offered: 100 +Decoded OK: 90 +Decode errors: 0 +Raw/Skip: 5 +Not in table: 3 +No field name: 1 +Struct blobs: 5 decoded / 0 failed +""") + raise SystemExit(0) + +print(""" +Rows offered: 100 +Decoded OK: 90 +Decode errors: 0 +Raw/Skip: 5 +Not in table: 3 +No field name: 2 +Struct blobs: 5 decoded / 0 failed +""") +''' + + +class MainWiringTests(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.root = Path(self._tmp.name) + (self.root / "export").write_text(FAKE_EXPORT_SCRIPT, encoding="utf-8") + self.corpus = self.root / "corpus" + self.corpus.mkdir() + self._previous_cwd = Path.cwd() + os.chdir(self.root) + self.addCleanup(os.chdir, self._previous_cwd) + self._argv = sys.argv + + def make_replay(self, name: str) -> None: + (self.corpus / name).write_bytes(b"not a real replay") + + def run_main(self, extra_args=()): + argv = [sys.executable, str(self.corpus), "--jobs", "1", *extra_args] + sys.argv = ["check_decode_errors_corpus.py", *argv] + out = io.StringIO() + try: + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(out): + code = guard.main() + finally: + sys.argv = self._argv + return code, out.getvalue() + + def test_a_clean_corpus_exits_zero(self): + self.make_replay("a.vrf") + code, output = self.run_main() + self.assertEqual(code, 0, output) + self.assertIn("OK:", output) + + def test_decode_errors_fail_the_run(self): + self.make_replay("decodeerr.vrf") + code, output = self.run_main() + self.assertEqual(code, 1, output) + self.assertIn("decode errors", output) + + def test_struct_blob_failures_fail_the_run(self): + self.make_replay("blobfail.vrf") + code, output = self.run_main() + self.assertEqual(code, 1, output) + self.assertIn("struct-blob", output) + + def test_an_exporter_that_decoded_nothing_fails_the_run(self): + """The 13.02 shape, one script down from the Rust regression: every + counter is a legitimate zero, `Decode errors: 0` is vacuously true, + and only `dead_counters` -- consulted by `main()` -- can catch it.""" + self.make_replay("nothingran.vrf") + code, output = self.run_main() + self.assertEqual(code, 1, output) + self.assertIn("never moved", output) + + def test_a_missing_required_counter_fails_the_run(self): + self.make_replay("missingcounter.vrf") + code, output = self.run_main() + self.assertEqual(code, 1, output) + self.assertIn("did not report the counter", output) + + def test_a_nonzero_exporter_exit_fails_the_run(self): + self.make_replay("badexit.vrf") + code, output = self.run_main() + self.assertEqual(code, 1, output) + self.assertIn("did not report the counter", output) + + def test_a_reconciliation_mismatch_fails_the_run(self): + self.make_replay("mismatch.vrf") + code, output = self.run_main() + self.assertEqual(code, 1, output) + self.assertIn("do not reconcile", output) + + def test_no_vrf_files_is_a_controlled_failure(self): + code, output = self.run_main() + self.assertEqual(code, 2, output) + + if __name__ == "__main__": unittest.main() diff --git a/tools/tests/test_check_metrics_baseline.py b/tools/tests/test_check_metrics_baseline.py index be019fa..949a40e 100644 --- a/tools/tests/test_check_metrics_baseline.py +++ b/tools/tests/test_check_metrics_baseline.py @@ -9,8 +9,13 @@ 13.02 fixture before commit bcc7d70: ClientRoundStart RPCs said 21 rounds while BombGameState RoundResults produced none, so team_score was empty. """ +import contextlib import copy +import io +import json +import os import sys +import tempfile import unittest from pathlib import Path @@ -196,5 +201,271 @@ def test_extract_and_invariants_agree_on_their_keys(self): self.assertIn(key, HEALTHY) +#: A raw `compute_metrics.py` output, shaped exactly like the real valplay +#: JSON `extract()` reads -- nested dicts, `per_player` maps, not the already +#: flattened `HEALTHY` fixture above. `HEALTHY` pins what the invariants see; +#: this pins the seam one layer earlier, between valplay's schema and this +#: tool's parsing of it. That seam had no coverage: a renamed or reshaped key +#: on the valplay side raises a loud `KeyError` today (by construction -- +#: `extract()` indexes with `[...]`, never `.get(..., default)`), but nothing +#: proved the *mapping itself* -- which flattened key reads which nested path, +#: and which fields get summed versus counted -- was still right. +RAW_METRICS = { + "combat": { + "per_player": { + "p1": {"kills": 5, "deaths": 3, "assists": 1, "headshots": 2, + "damage_dealt": 501.5}, + "p2": {"kills": 2, "deaths": 4, "assists": 0, "headshots": 0, + "damage_dealt": 88.25}, + }, + }, + "tactical": { + "per_player": { + "p1": {"first_bloods": 1, "trade_kills": 0}, + "p2": {"first_bloods": 0, "trade_kills": 2}, + }, + }, + "rounds": {"round_count": 6, "client_round_start_events": 6}, + "objective": {"round_count": 6, "team_score": {"Blue": 4, "Red": 2}}, + "objective_detail": {"plant_count": 3, "defuse_count": 1}, + "players": ["p1", "p2"], + "kast": {"per_player": {"p1": {"kast_rounds": 5}, "p2": {"kast_rounds": 4}}}, + "ultimate": {"total_casts": 2}, + "weapons": {"distinct_weapons": 3, + "shots_by_weapon": {"Vandal": 30, "Classic": 12}}, + "shot_rays": {"ray_count": 41}, + "ability_usage": {"ability_spawn_count": 9}, + "movement_summary": {"movement_samples": 12345}, + "economy_detail": {"rounds": 6}, +} + + +class ExtractShapeTests(unittest.TestCase): + """`extract()` is the only code that reads valplay's real JSON shape. + Nothing else in this suite exercises it against a shape that looks like + what `compute_metrics.py` actually emits -- every other test starts from + the already-flattened `HEALTHY` dict, which proves the invariants but + never proves the mapping into them. + """ + + def test_scalar_fields_are_read_from_their_nested_path(self): + got = guard.extract(RAW_METRICS) + self.assertEqual(got["rounds_rpc"], 6) + self.assertEqual(got["rounds_objective"], 6) + self.assertEqual(got["client_round_starts"], 6) + self.assertEqual(got["team_score"], {"Blue": 4, "Red": 2}) + self.assertEqual(got["plants"], 3) + self.assertEqual(got["defuses"], 1) + self.assertEqual(got["players"], 2) + self.assertEqual(got["combat_players"], 2) + self.assertEqual(got["ultimate_casts"], 2) + self.assertEqual(got["distinct_weapons"], 3) + self.assertEqual(got["shot_rays"], 41) + self.assertEqual(got["ability_spawns"], 9) + self.assertEqual(got["movement_samples"], 12345) + self.assertEqual(got["economy_rounds"], 6) + + def test_per_player_combat_fields_are_summed_across_players(self): + got = guard.extract(RAW_METRICS) + self.assertEqual(got["kills"], 7) # 5 + 2 + self.assertEqual(got["deaths"], 7) # 3 + 4 + self.assertEqual(got["assists"], 1) # 1 + 0 + self.assertEqual(got["headshots"], 2) # 2 + 0 + self.assertEqual(got["damage_dealt"], 589.75) # 501.5 + 88.25 + + def test_per_player_tactical_and_kast_fields_are_summed(self): + got = guard.extract(RAW_METRICS) + self.assertEqual(got["first_bloods"], 1) # 1 + 0 + self.assertEqual(got["trade_kills"], 2) # 0 + 2 + self.assertEqual(got["kast_rounds"], 9) # 5 + 4 + + def test_shots_are_summed_across_weapons_not_taken_from_distinct_weapons(self): + """`shots` and `distinct_weapons` read different things off the same + `weapons` block -- a copy/paste of one into the other would pass every + other test here since both are small integers.""" + got = guard.extract(RAW_METRICS) + self.assertEqual(got["shots"], 42) # 30 + 12 + self.assertNotEqual(got["shots"], got["distinct_weapons"]) + + def test_a_player_with_no_combat_entry_does_not_crash_the_sum(self): + """`_sum` reads `.get(field) or 0` per player -- a player present in + `players` but absent from `combat.per_player` (never fired a shot, + never took damage) must not raise, and must not count.""" + raw = copy.deepcopy(RAW_METRICS) + raw["players"].append("p3") + got = guard.extract(raw) + self.assertEqual(got["players"], 3) + self.assertEqual(got["combat_players"], 2) # p3 never joined combat + self.assertEqual(got["kills"], 7) # unchanged + + +#: Stand-ins for the three pipeline stages `run_one` shells out to -- +#: `vrfkit export`, `to_valplay_bundle.py`, and valplay's `compute_metrics.py`. +#: The first is invoked positionally (`[str(exe), "export", ...]`), so under +#: `sys.executable` a file literally named `export` in the process's cwd +#: stands in for it, exactly as the other two corpus scripts' fake executables +#: do. The other two are invoked by explicit path, so ordinary `.py` files +#: patched onto `guard.BUNDLE_TOOL` / `guard.COMPUTE_METRICS` stand in for +#: them. The fake `compute_metrics.py` does not compute anything -- it copies +#: whatever this test staged as the desired metrics.json, so one pair of fake +#: scripts can play every scenario below by changing what gets staged. +FAKE_EXPORT_SCRIPT = '''\ +import sys +from pathlib import Path +out = Path(sys.argv[sys.argv.index("--out") + 1]) +out.mkdir(parents=True, exist_ok=True) +print("export ok") +''' + +FAKE_BUNDLE_SCRIPT = '''\ +import sys +from pathlib import Path +out = Path(sys.argv[sys.argv.index("-o") + 1]) +out.mkdir(parents=True, exist_ok=True) +print("bundle ok") +''' + +#: Reads the desired metrics.json from the path the test staged in an +#: environment variable (`subprocess.run` inherits the parent's environment +#: by default, so this reaches the child) and writes it to wherever `-o` +#: says -- so it stands in for compute_metrics.py without knowing anything +#: about the bundle format `-o`'s sibling argument actually names. +FAKE_METRICS_SCRIPT = '''\ +import os +import shutil +import sys +from pathlib import Path +staged = Path(os.environ["VRFKIT_TEST_DESIRED_METRICS"]) +out = Path(sys.argv[sys.argv.index("-o") + 1]) +shutil.copyfile(staged, out) +print("metrics ok") +''' + + +class MainWiringTests(unittest.TestCase): + """`InvariantTests` and `DriftTests` above prove the pure functions; they + say nothing about whether `main()` calls them and acts on the result + before deciding an exit code -- the same wiring gap the other two corpus + scripts' `MainWiringTests` closes, for the one check this project built + specifically because the framing layer cannot see a semantic break. + """ + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.root = Path(self._tmp.name) + (self.root / "export").write_text(FAKE_EXPORT_SCRIPT, encoding="utf-8") + self.bundle_tool = self.root / "fake_bundle.py" + self.bundle_tool.write_text(FAKE_BUNDLE_SCRIPT, encoding="utf-8") + self.compute_metrics = self.root / "fake_compute_metrics.py" + self.compute_metrics.write_text(FAKE_METRICS_SCRIPT, encoding="utf-8") + + self.replay = self.root / "match.vrf" + self.replay.write_bytes(b"not a real replay") + + self._orig_bundle_tool = guard.BUNDLE_TOOL + self._orig_compute_metrics = guard.COMPUTE_METRICS + self._orig_replays = guard.REPLAYS + guard.BUNDLE_TOOL = self.bundle_tool + guard.COMPUTE_METRICS = self.compute_metrics + guard.REPLAYS = {"test": str(self.replay)} + self.addCleanup(self._restore_module_state) + + self._previous_cwd = Path.cwd() + os.chdir(self.root) + self.addCleanup(os.chdir, self._previous_cwd) + + self._argv = sys.argv + self.addCleanup(self._restore_argv) + + def _restore_module_state(self): + guard.BUNDLE_TOOL = self._orig_bundle_tool + guard.COMPUTE_METRICS = self._orig_compute_metrics + guard.REPLAYS = self._orig_replays + + def _restore_argv(self): + sys.argv = self._argv + + def stage_metrics(self, metrics: dict) -> None: + staged = self.root / "desired_metrics.json" + staged.write_text(json.dumps(metrics), encoding="utf-8") + os.environ["VRFKIT_TEST_DESIRED_METRICS"] = str(staged) + self.addCleanup(os.environ.pop, "VRFKIT_TEST_DESIRED_METRICS", None) + + def run_main(self, extra_args=()): + argv = ["check_metrics_baseline.py", "--exe", sys.executable, + "--only", "test", "--jobs", "1", *extra_args] + sys.argv = argv + out = io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(out): + code = guard.main() + return code, out.getvalue() + + def test_a_healthy_run_matching_the_baseline_exits_zero(self): + self.stage_metrics(RAW_METRICS) + baseline = self.root / "baseline.json" + baseline.write_text(json.dumps( + {"metrics": {"test": guard.extract(RAW_METRICS)}}), encoding="utf-8") + + code, output = self.run_main(["--baseline", str(baseline)]) + + self.assertEqual(code, 0, output) + + def test_an_invariant_violation_fails_the_run_and_refuses_to_update(self): + """The headline case: R1/R2 fire (see `InvariantTests`), and `main()` + must both exit non-zero AND leave the baseline untouched under + `--update` -- pinning a broken run would make the NEXT clean run look + like drift instead of a fix.""" + broken = dict(RAW_METRICS) + broken["objective"] = {"round_count": 0, "team_score": {}} + broken["economy_detail"] = {"rounds": 0} + self.stage_metrics(broken) + baseline = self.root / "baseline.json" + original = json.dumps({"metrics": {}}) + baseline.write_text(original, encoding="utf-8") + + code, output = self.run_main(["--baseline", str(baseline), "--update"]) + + self.assertEqual(code, 1, output) + self.assertIn("R1", output) + self.assertIn("baseline NOT updated", output) + self.assertEqual(baseline.read_text(encoding="utf-8"), original, + "a broken run must not be pinned") + + def test_drift_from_the_baseline_fails_the_run(self): + self.stage_metrics(RAW_METRICS) + stored = guard.extract(RAW_METRICS) + stored = dict(stored, kills=stored["kills"] + 1000) + baseline = self.root / "baseline.json" + baseline.write_text(json.dumps({"metrics": {"test": stored}}), + encoding="utf-8") + + code, output = self.run_main(["--baseline", str(baseline)]) + + self.assertEqual(code, 1, output) + self.assertIn("drifted", output) + self.assertIn("kills", output) + + def test_a_missing_baseline_is_a_controlled_failure(self): + self.stage_metrics(RAW_METRICS) + baseline = self.root / "does-not-exist.json" + + code, output = self.run_main(["--baseline", str(baseline)]) + + self.assertEqual(code, 2, output) + self.assertIn("baseline not found", output) + + def test_a_replay_that_does_not_exist_fails_the_pipeline(self): + guard.REPLAYS = {"test": str(self.root / "missing.vrf")} + baseline = self.root / "baseline.json" + baseline.write_text(json.dumps({"metrics": {}}), encoding="utf-8") + + code, output = self.run_main(["--baseline", str(baseline)]) + + self.assertEqual(code, 1, output) + self.assertIn("did not complete the pipeline", output) + self.assertIn("replay not found", output) + + if __name__ == "__main__": unittest.main() diff --git a/tools/tests/test_validate_corpus.py b/tools/tests/test_validate_corpus.py index 9e5823e..eb92d42 100644 --- a/tools/tests/test_validate_corpus.py +++ b/tools/tests/test_validate_corpus.py @@ -7,7 +7,11 @@ the argument down is not the same as acting on it. """ import collections +import contextlib +import io +import os import sys +import tempfile import unittest from pathlib import Path @@ -94,5 +98,98 @@ def test_limit_is_optional(self): self.assertIsNone(args.limit) +#: Stand-in for `vrfkit.exe`, invoked exactly as `_run_one` invokes the real +#: one -- `[str(exe), "validate", str(path)]`. Run under `sys.executable`, the +#: first argv token becomes the script Python executes (the same trick +#: `test_check_export_baseline.py`'s `TransactionalOutputTests` uses for its +#: fake `export`), so a file literally named `validate`, with no extension, in +#: the process's cwd stands in for the real binary. What it prints depends on +#: the replay's own filename, so one script can play every scenario below. +FAKE_VALIDATE_SCRIPT = '''\ +import sys +from pathlib import Path + +name = Path(sys.argv[1]).name + +if "badexit" in name: + print("oracle blew up", file=sys.stderr) + raise SystemExit(3) + +print("Branch: ++Ares-Core+release-13.01") +print("Total content blocks: 100") +if "missingmalformed" not in name: + print("Malformed framing: 0") +print("Skipped bits: 0") +print("Fields emitted: 50") +print("RPCs emitted: 10") +print("ORACLE PASS RATE: 100.000000%") +''' + + +class MainWiringTests(unittest.TestCase): + """`ProblemTests` above pins what `problems()` returns; nothing pinned + that `main()` actually reads it before choosing an exit code. That is + precisely the layer where the recorded defect lived: the absent-counter + case was computed, printed as a WARNING, and the process exited 0 anyway. + A helper that is provably correct in isolation says nothing about the + `if found: return 1` a few lines later in `main()` -- these tests are + that line. + """ + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.root = Path(self._tmp.name) + (self.root / "validate").write_text(FAKE_VALIDATE_SCRIPT, encoding="utf-8") + self.corpus = self.root / "corpus" + self.corpus.mkdir() + self._previous_cwd = Path.cwd() + os.chdir(self.root) + self.addCleanup(os.chdir, self._previous_cwd) + + def make_replay(self, name: str) -> None: + (self.corpus / name).write_bytes(b"not a real replay") + + def run_main(self, limit: str | None = None): + argv = ["validate_corpus.py", sys.executable, str(self.corpus)] + if limit is not None: + argv.append(limit) + out = io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(out): + code = guard.main(argv) + return code, out.getvalue() + + def test_a_clean_sweep_exits_zero(self): + self.make_replay("a.vrf") + self.make_replay("b.vrf") + code, output = self.run_main() + self.assertEqual(code, 0, output) + self.assertIn("OK:", output) + + def test_an_oracle_that_could_not_validate_a_replay_fails_the_run(self): + self.make_replay("badexit.vrf") + code, output = self.run_main() + self.assertNotEqual(code, 0, output) + self.assertIn("FAILED", output) + + def test_a_counter_the_oracle_stopped_printing_fails_the_run(self): + """The recorded defect, reproduced end to end: `missingmalformed.vrf` + exits 0 from the fake oracle and every OTHER counter is present, so + the only thing that can catch it is `main()` reading `problems()`'s + report on the absent `malformed` counter -- not a helper being + correct, but `main()` acting on what the helper says.""" + self.make_replay("missingmalformed.vrf") + code, output = self.run_main() + self.assertNotEqual(code, 0, output) + self.assertIn("FAILED", output) + self.assertIn("malformed", output) + + def test_an_empty_corpus_is_a_controlled_failure_not_a_silent_pass(self): + with contextlib.redirect_stdout(io.StringIO()): + with self.assertRaises(SystemExit) as caught: + guard.main(["validate_corpus.py", sys.executable, str(self.corpus)]) + self.assertIn("no .vrf under", str(caught.exception)) + + if __name__ == "__main__": unittest.main()