diff --git a/tools/lint_benchmark_configs.py b/tools/lint_benchmark_configs.py new file mode 100644 index 0000000..4cf89c5 --- /dev/null +++ b/tools/lint_benchmark_configs.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Validate mcoplib mxbenchmark config/runner pairs.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +RUNNER_PREFIX = "mcoplib_mxbenchmark_" +RUNNER_SUFFIX = "_runners.py" + + +def collect_errors(root: Path) -> list[str]: + benchmark_dir = root / "benchmark" + config_dir = benchmark_dir / "config" + runners_dir = benchmark_dir / "runners" + errors: list[str] = [] + + if not config_dir.is_dir(): + return [f"missing config directory: {config_dir}"] + if not runners_dir.is_dir(): + return [f"missing runners directory: {runners_dir}"] + + configs = {path.stem: path for path in config_dir.glob("*.json")} + runners = { + path.name[len(RUNNER_PREFIX) : -len(RUNNER_SUFFIX)]: path + for path in runners_dir.glob(f"{RUNNER_PREFIX}*{RUNNER_SUFFIX}") + } + + for name in sorted(set(configs) - set(runners)): + errors.append(f"missing runner for config {configs[name].relative_to(root).as_posix()}") + for name in sorted(set(runners) - set(configs)): + errors.append(f"missing config for runner {runners[name].relative_to(root).as_posix()}") + + for name, path in sorted(configs.items()): + try: + json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + errors.append(f"invalid JSON or unreadable file in {path.relative_to(root).as_posix()}: {exc}") + + return errors + + +def main() -> int: + parser = argparse.ArgumentParser(description="Lint mxbenchmark config and runner pairs.") + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + args = parser.parse_args() + + root = args.root.resolve() + errors = collect_errors(root) + if errors: + print("mxbenchmark config lint failed:", file=sys.stderr) + for error in errors: + print(f" {error}", file=sys.stderr) + return 1 + + print("mxbenchmark config lint passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/unit_test/test_lint_benchmark_configs.py b/unit_test/test_lint_benchmark_configs.py new file mode 100644 index 0000000..c96f083 --- /dev/null +++ b/unit_test/test_lint_benchmark_configs.py @@ -0,0 +1,58 @@ +import json +import tempfile +import unittest +from pathlib import Path + +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from tools.lint_benchmark_configs import collect_errors + + +class LintBenchmarkConfigsTest(unittest.TestCase): + def test_detects_missing_runner(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "benchmark" / "config").mkdir(parents=True) + (root / "benchmark" / "runners").mkdir(parents=True) + (root / "benchmark" / "config" / "foo.json").write_text("{}", encoding="utf-8") + + errors = collect_errors(root) + + self.assertEqual(errors, ["missing runner for config benchmark/config/foo.json"]) + + def test_valid_pair_passes(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "benchmark" / "config").mkdir(parents=True) + (root / "benchmark" / "runners").mkdir(parents=True) + (root / "benchmark" / "config" / "foo.json").write_text( + json.dumps({"samples": 1}), encoding="utf-8" + ) + (root / "benchmark" / "runners" / "mcoplib_mxbenchmark_foo_runners.py").write_text( + "", encoding="utf-8" + ) + + errors = collect_errors(root) + + self.assertEqual(errors, []) + + def test_reports_unreadable_config_file(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "benchmark" / "config").mkdir(parents=True) + (root / "benchmark" / "runners").mkdir(parents=True) + (root / "benchmark" / "config" / "foo.json").write_bytes(b"\xff\xfe") + (root / "benchmark" / "runners" / "mcoplib_mxbenchmark_foo_runners.py").write_text( + "", encoding="utf-8" + ) + + errors = collect_errors(root) + + self.assertEqual(len(errors), 1) + self.assertIn("invalid JSON or unreadable file", errors[0]) + + +if __name__ == "__main__": + unittest.main()