From cfd79865fd2ff49dabf896ae36e2c1bfa560da68 Mon Sep 17 00:00:00 2001 From: papertager <2567587994@qq.com> Date: Tue, 9 Jun 2026 21:18:32 +0800 Subject: [PATCH 1/2] Add mxbenchmark config lint tool --- tools/lint_benchmark_configs.py | 64 ++++++++++++++++++++++++ unit_test/test_lint_benchmark_configs.py | 43 ++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 tools/lint_benchmark_configs.py create mode 100644 unit_test/test_lint_benchmark_configs.py diff --git a/tools/lint_benchmark_configs.py b/tools/lint_benchmark_configs.py new file mode 100644 index 0000000..ea8cde3 --- /dev/null +++ b/tools/lint_benchmark_configs.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Validate mcoplib mxbenchmark config/runner pairs.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +RUNNER_PREFIX = "mcoplib_mxbenchmark_" +RUNNER_SUFFIX = "_runners.py" + + +def collect_errors(root: Path) -> list[str]: + benchmark_dir = root / "benchmark" + config_dir = benchmark_dir / "config" + runners_dir = benchmark_dir / "runners" + errors: list[str] = [] + + if not config_dir.is_dir(): + return [f"missing config directory: {config_dir}"] + if not runners_dir.is_dir(): + return [f"missing runners directory: {runners_dir}"] + + configs = {path.stem: path for path in config_dir.glob("*.json")} + runners = { + path.name[len(RUNNER_PREFIX) : -len(RUNNER_SUFFIX)]: path + for path in runners_dir.glob(f"{RUNNER_PREFIX}*{RUNNER_SUFFIX}") + } + + for name in sorted(set(configs) - set(runners)): + errors.append(f"missing runner for config {configs[name].relative_to(root).as_posix()}") + for name in sorted(set(runners) - set(configs)): + errors.append(f"missing config for runner {runners[name].relative_to(root).as_posix()}") + + for name, path in sorted(configs.items()): + try: + json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + errors.append(f"invalid JSON in {path.relative_to(root).as_posix()}: {exc}") + + return errors + + +def main() -> int: + parser = argparse.ArgumentParser(description="Lint mxbenchmark config and runner pairs.") + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + args = parser.parse_args() + + root = args.root.resolve() + errors = collect_errors(root) + if errors: + print("mxbenchmark config lint failed:", file=sys.stderr) + for error in errors: + print(f" {error}", file=sys.stderr) + return 1 + + print("mxbenchmark config lint passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/unit_test/test_lint_benchmark_configs.py b/unit_test/test_lint_benchmark_configs.py new file mode 100644 index 0000000..87ecd57 --- /dev/null +++ b/unit_test/test_lint_benchmark_configs.py @@ -0,0 +1,43 @@ +import json +import tempfile +import unittest +from pathlib import Path + +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from tools.lint_benchmark_configs import collect_errors + + +class LintBenchmarkConfigsTest(unittest.TestCase): + def test_detects_missing_runner(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "benchmark" / "config").mkdir(parents=True) + (root / "benchmark" / "runners").mkdir(parents=True) + (root / "benchmark" / "config" / "foo.json").write_text("{}", encoding="utf-8") + + errors = collect_errors(root) + + self.assertEqual(errors, ["missing runner for config benchmark/config/foo.json"]) + + def test_valid_pair_passes(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "benchmark" / "config").mkdir(parents=True) + (root / "benchmark" / "runners").mkdir(parents=True) + (root / "benchmark" / "config" / "foo.json").write_text( + json.dumps({"samples": 1}), encoding="utf-8" + ) + (root / "benchmark" / "runners" / "mcoplib_mxbenchmark_foo_runners.py").write_text( + "", encoding="utf-8" + ) + + errors = collect_errors(root) + + self.assertEqual(errors, []) + + +if __name__ == "__main__": + unittest.main() From d6ab61972fda8e1926bffa9937e8e81b81756ebb Mon Sep 17 00:00:00 2001 From: papertager <2567587994@qq.com> Date: Thu, 11 Jun 2026 00:42:23 +0800 Subject: [PATCH 2/2] Handle unreadable benchmark configs cleanly --- tools/lint_benchmark_configs.py | 4 ++-- unit_test/test_lint_benchmark_configs.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/tools/lint_benchmark_configs.py b/tools/lint_benchmark_configs.py index ea8cde3..4cf89c5 100644 --- a/tools/lint_benchmark_configs.py +++ b/tools/lint_benchmark_configs.py @@ -37,8 +37,8 @@ def collect_errors(root: Path) -> list[str]: for name, path in sorted(configs.items()): try: json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError as exc: - errors.append(f"invalid JSON in {path.relative_to(root).as_posix()}: {exc}") + except (OSError, ValueError) as exc: + errors.append(f"invalid JSON or unreadable file in {path.relative_to(root).as_posix()}: {exc}") return errors diff --git a/unit_test/test_lint_benchmark_configs.py b/unit_test/test_lint_benchmark_configs.py index 87ecd57..c96f083 100644 --- a/unit_test/test_lint_benchmark_configs.py +++ b/unit_test/test_lint_benchmark_configs.py @@ -38,6 +38,21 @@ def test_valid_pair_passes(self): self.assertEqual(errors, []) + def test_reports_unreadable_config_file(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "benchmark" / "config").mkdir(parents=True) + (root / "benchmark" / "runners").mkdir(parents=True) + (root / "benchmark" / "config" / "foo.json").write_bytes(b"\xff\xfe") + (root / "benchmark" / "runners" / "mcoplib_mxbenchmark_foo_runners.py").write_text( + "", encoding="utf-8" + ) + + errors = collect_errors(root) + + self.assertEqual(len(errors), 1) + self.assertIn("invalid JSON or unreadable file", errors[0]) + if __name__ == "__main__": unittest.main()