From 268ad3f283aad87c5d4d826353b3d9343c015b05 Mon Sep 17 00:00:00 2001 From: papertager <2567587994@qq.com> Date: Tue, 9 Jun 2026 21:19:22 +0800 Subject: [PATCH 1/2] Add operator source inventory tool --- tools/op_source_inventory.py | 57 +++++++++++++++++++++++++++ unit_test/test_op_source_inventory.py | 27 +++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 tools/op_source_inventory.py create mode 100644 unit_test/test_op_source_inventory.py diff --git a/tools/op_source_inventory.py b/tools/op_source_inventory.py new file mode 100644 index 0000000..31e3214 --- /dev/null +++ b/tools/op_source_inventory.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Build a JSON inventory of mcoplib operator source groups.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +GROUPS = { + "vllm": "op/vllm", + "sglang": "op/sglang", + "lmdeploy": "op/lmdeploy", + "cv": "op/cv", + "native": "op", +} + +SOURCE_SUFFIXES = {".cu", ".cuh", ".cpp", ".cc", ".h", ".hpp", ".py"} + + +def _sources(root: Path, relative_dir: str) -> list[str]: + base = root / relative_dir + if not base.exists(): + return [] + return sorted( + path.relative_to(root).as_posix() + for path in base.rglob("*") + if path.is_file() and path.suffix in SOURCE_SUFFIXES + ) + + +def build_inventory(root: Path) -> dict[str, object]: + groups: dict[str, object] = {} + for name, relative_dir in GROUPS.items(): + files = _sources(root, relative_dir) + groups[name] = {"root": relative_dir, "count": len(files), "files": files} + return {"root": str(root), "groups": groups} + + +def main() -> int: + parser = argparse.ArgumentParser(description="Create mcoplib operator source inventory.") + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + + payload = build_inventory(args.root.resolve()) + text = json.dumps(payload, indent=2, sort_keys=True) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(text + "\n", encoding="utf-8") + else: + print(text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/unit_test/test_op_source_inventory.py b/unit_test/test_op_source_inventory.py new file mode 100644 index 0000000..0d73488 --- /dev/null +++ b/unit_test/test_op_source_inventory.py @@ -0,0 +1,27 @@ +import tempfile +import unittest +from pathlib import Path + +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from tools.op_source_inventory import build_inventory + + +class OpSourceInventoryTest(unittest.TestCase): + def test_inventory_counts_group_sources(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "op" / "vllm").mkdir(parents=True) + (root / "op" / "vllm" / "kernel.cu").write_text("", encoding="utf-8") + (root / "op" / "vllm" / "README.md").write_text("", encoding="utf-8") + + inventory = build_inventory(root) + + self.assertEqual(inventory["groups"]["vllm"]["count"], 1) + self.assertEqual(inventory["groups"]["vllm"]["files"], ["op/vllm/kernel.cu"]) + + +if __name__ == "__main__": + unittest.main() From d4aefe84dcfd12268b5702017be4fbfa8f2f4099 Mon Sep 17 00:00:00 2001 From: papertager <2567587994@qq.com> Date: Thu, 11 Jun 2026 01:01:10 +0800 Subject: [PATCH 2/2] Deduplicate native operator source inventory --- tools/op_source_inventory.py | 13 ++++++++++++- unit_test/test_op_source_inventory.py | 3 +++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/tools/op_source_inventory.py b/tools/op_source_inventory.py index 31e3214..bff793f 100644 --- a/tools/op_source_inventory.py +++ b/tools/op_source_inventory.py @@ -30,9 +30,20 @@ def _sources(root: Path, relative_dir: str) -> list[str]: def build_inventory(root: Path) -> dict[str, object]: + raw_files: dict[str, list[str]] = {} + for name, relative_dir in GROUPS.items(): + raw_files[name] = _sources(root, relative_dir) + + other_files: set[str] = set() + for name in GROUPS: + if name != "native": + other_files.update(raw_files[name]) + if "native" in raw_files: + raw_files["native"] = [path for path in raw_files["native"] if path not in other_files] + groups: dict[str, object] = {} for name, relative_dir in GROUPS.items(): - files = _sources(root, relative_dir) + files = raw_files[name] groups[name] = {"root": relative_dir, "count": len(files), "files": files} return {"root": str(root), "groups": groups} diff --git a/unit_test/test_op_source_inventory.py b/unit_test/test_op_source_inventory.py index 0d73488..58a1bae 100644 --- a/unit_test/test_op_source_inventory.py +++ b/unit_test/test_op_source_inventory.py @@ -16,11 +16,14 @@ def test_inventory_counts_group_sources(self): (root / "op" / "vllm").mkdir(parents=True) (root / "op" / "vllm" / "kernel.cu").write_text("", encoding="utf-8") (root / "op" / "vllm" / "README.md").write_text("", encoding="utf-8") + (root / "op" / "native_kernel.cu").write_text("", encoding="utf-8") inventory = build_inventory(root) self.assertEqual(inventory["groups"]["vllm"]["count"], 1) self.assertEqual(inventory["groups"]["vllm"]["files"], ["op/vllm/kernel.cu"]) + self.assertEqual(inventory["groups"]["native"]["count"], 1) + self.assertEqual(inventory["groups"]["native"]["files"], ["op/native_kernel.cu"]) if __name__ == "__main__":