Docs: unify TraceLens metrics reference (CSV + TraceIndex SQL)

README.md

@@ -31,6 +31,8 @@ TraceLens is a Python library for **automated performance analysis of training a
 
 **TraceLens Agent**: Receive a prioritized human-readable optimization report, derived through an agentic workflow, covering compute kernels, system bottlenecks, and kernel fusion opportunities with root-cause reasoning and concrete resolutions.
 
+**Trace corpus indexing**: Build a searchable SQLite catalog of TraceLens reports so you can find traces by op, category, or kernel name without reopening every raw file. Scanner and importer sit behind a storage interface; SQLite is the first backend.
+
 ---
 
 ## Quick Start
@@ -67,7 +69,7 @@ Generate a performance analysis report from an eager execution PyTorch trace wit
 TraceLens_generate_perf_report_pytorch --profile_json_path path/to/your/trace.json
 ```
 
-This produces an Excel workbook with GPU timeline breakdown, ops summary, roofline metrics and more. For additional details, see [Generate a PyTorch performance report](docs/how-to/generate-perf-report-pytorch.md) and [Performance report column reference](docs/reference/perf-report-columns.md). For other input formats, see [Supported Profile Formats](#supported-profile-formats).
+This produces an Excel workbook with GPU timeline breakdown, ops summary, roofline metrics and more. For additional details, see [Generate a PyTorch performance report](docs/how-to/generate-perf-report-pytorch.md) and [TraceLens metrics reference](docs/reference/tracelens-metrics.md). For other input formats, see [Supported Profile Formats](#supported-profile-formats).
 
 Compare two reports to quantify the impact of a change (see [Compare performance reports](docs/how-to/compare-perf-reports.md)):
 
@@ -79,6 +81,16 @@ TraceLens_compare_perf_reports_pytorch \
     -o comparison.xlsx
 ```
 
+Index traces and existing TraceLens CSV reports (see [Index a corpus of traces](docs/how-to/trace-index.md)):
+
+```bash
+TraceLens_trace_index --db trace_index.sqlite append \
+    --trace-path /path/to/rank0_trace.json.gz \
+    --report-dir path/to/perf_report_csvs
+TraceLens_trace_index --db trace_index.sqlite build --traces-file traces.txt
+TraceLens_trace_index --db trace_index.sqlite search Cijk
+```
+
 For multi-rank runs, generate a collective-communication report across ranks (see [Generate a collective-communication report](docs/how-to/collective-report.md)):
 
 ```bash
@@ -137,8 +149,9 @@ Each format's linked doc covers its full CLI reference. For PyTorch report compa
 | pftrace Reports              | [docs/how-to/generate-perf-report-rocprof.md](docs/how-to/generate-perf-report-rocprof.md)                                       |
 | Compare PyTorch Reports      | [docs/how-to/compare-perf-reports.md](docs/how-to/compare-perf-reports.md)                                                       |
 | Multi-Rank Collective Report | [docs/how-to/collective-report.md](docs/how-to/collective-report.md)                                                             |
-| Performance Report Columns   | [docs/reference/perf-report-columns.md](docs/reference/perf-report-columns.md)                                                   |
+| TraceLens metrics            | [docs/reference/tracelens-metrics.md](docs/reference/tracelens-metrics.md)                                                       |
 | TraceLens Agent              | [docs/how-to/agent.md](docs/how-to/agent.md)                                                                                     |
+| TraceIndex                   | [docs/how-to/trace-index.md](docs/how-to/trace-index.md)                                                                         |
 
 ---
 

TraceLens/TraceIndex/__init__.py

@@ -0,0 +1,31 @@
+###############################################################################
+# Copyright (c) 2024 - 2025 Advanced Micro Devices, Inc. All rights reserved.
+#
+# See LICENSE for license information.
+###############################################################################
+
+"""Trace corpus indexing helpers."""
+
+from .core import (
+    append_trace,
+    build_traces,
+    execute_read_query,
+    generate_report_and_import,
+    search_index,
+)
+from .models import SearchHit, TraceRecord, TraceReport
+from .sqlite_store import SQLiteTraceIndexStore
+from .store import TraceIndexStore
+
+__all__ = [
+    "TraceIndexStore",
+    "SQLiteTraceIndexStore",
+    "SearchHit",
+    "TraceRecord",
+    "TraceReport",
+    "append_trace",
+    "build_traces",
+    "execute_read_query",
+    "generate_report_and_import",
+    "search_index",
+]

TraceLens/TraceIndex/cli.py

@@ -0,0 +1,225 @@
+###############################################################################
+# Copyright (c) 2024 - 2025 Advanced Micro Devices, Inc. All rights reserved.
+#
+# See LICENSE for license information.
+###############################################################################
+
+"""Command line entry point for TraceIndex."""
+
+import argparse
+import json
+from pathlib import Path
+from typing import List, Optional
+
+from TraceLens.TraceIndex.importer import (
+    append_trace,
+    build_traces,
+    report_dir_for_trace,
+)
+from TraceLens.TraceIndex.sqlite_store import SQLiteTraceIndexStore
+from TraceLens.TraceIndex.utils import collect_trace_paths
+
+DEFAULT_DB = Path("trace_index.sqlite")
+
+
+def print_json(payload: object) -> None:
+    print(json.dumps(payload, indent=2, default=str))
+
+
+def create_store(args: argparse.Namespace):
+    if args.backend != "sqlite":
+        raise ValueError("only the sqlite backend is currently implemented")
+    return SQLiteTraceIndexStore(args.db)
+
+
+def append_cmd(args: argparse.Namespace) -> int:
+    store = create_store(args)
+    try:
+        report_dir = args.report_dir
+        generated = report_dir is None
+        if generated:
+            report_dir = report_dir_for_trace(args.trace_path, args.report_root)
+        trace_id = append_trace(
+            store,
+            trace_path=args.trace_path,
+            report_dir=None if generated else report_dir,
+            root=args.root,
+            force=args.force,
+            enable_pseudo_ops=args.enable_pseudo_ops,
+            report_root=args.report_root,
+        )
+        print_json(
+            {
+                "backend": args.backend,
+                "db": args.db,
+                "trace_id": trace_id,
+                "trace_path": args.trace_path,
+                "report_dir": report_dir,
+                "generated_report": generated,
+            }
+        )
+        return 0
+    finally:
+        store.close()
+
+
+def build_cmd(args: argparse.Namespace) -> int:
+    trace_paths = collect_trace_paths(args.traces_file, args.trace_path)
+    if not trace_paths:
+        raise SystemExit(
+            "build requires --traces-file and/or one or more --trace-path values"
+        )
+    store = create_store(args)
+    try:
+        result = build_traces(
+            store,
+            trace_paths,
+            report_root=args.report_root,
+            root=args.root,
+            force=args.force,
+            enable_pseudo_ops=args.enable_pseudo_ops,
+        )
+        print_json(
+            {
+                "backend": args.backend,
+                "db": args.db,
+                "imported": result["imported"],
+                "failed": result["failed"],
+            }
+        )
+        return 1 if result["failed"] else 0
+    finally:
+        store.close()
+
+
+def search_cmd(args: argparse.Namespace) -> int:
+    store = create_store(args)
+    try:
+        store.init_schema()
+        rows = [
+            hit._asdict()
+            for hit in store.search(" ".join(args.terms), limit=args.limit)
+        ]
+        print_json({"backend": args.backend, "rows": rows})
+        return 0
+    finally:
+        store.close()
+
+
+def sqlite_sql_cmd(args: argparse.Namespace) -> int:
+    store = create_store(args)
+    try:
+        store.init_schema()
+        rows = store.execute_read_query(args.sql, limit=args.limit)
+        print_json({"backend": args.backend, "rows": rows})
+        return 0
+    finally:
+        store.close()
+
+
+def serve_cmd(args: argparse.Namespace) -> int:
+    from TraceLens.TraceIndex.server import serve
+
+    serve(
+        db_path=args.db,
+        host=args.host,
+        port=args.port,
+        default_limit=args.default_limit,
+        max_limit=args.max_limit,
+    )
+    return 0
+
+
+def add_generate_args(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "--report-root",
+        type=Path,
+        default=None,
+        help="Directory for generated CSV reports (default: trace_index_reports/)",
+    )
+    parser.add_argument("--root", type=Path, default=None)
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Regenerate CSV reports even if they already exist",
+    )
+    parser.add_argument("--enable-pseudo-ops", action="store_true")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Build and query a TraceIndex catalog of traces."
+    )
+    parser.add_argument(
+        "--backend",
+        choices=["sqlite"],
+        default="sqlite",
+        help="TraceIndex storage backend",
+    )
+    parser.add_argument("--db", type=Path, default=DEFAULT_DB, help="SQLite DB path")
+    sub = parser.add_subparsers(dest="command")
+
+    append = sub.add_parser(
+        "append",
+        help="Append one trace to the catalog, optionally from an existing CSV report",
+    )
+    append.add_argument("--trace-path", type=Path, required=True)
+    append.add_argument(
+        "--report-dir",
+        type=Path,
+        default=None,
+        help="Existing TraceLens CSV report directory. If omitted, generate a training PyTorch report.",
+    )
+    add_generate_args(append)
+    append.set_defaults(func=append_cmd)
+
+    build = sub.add_parser(
+        "build",
+        help="Create or open the catalog and append a batch of traces",
+    )
+    build.add_argument(
+        "--traces-file",
+        type=Path,
+        default=None,
+        help="Text file with one trace path per line (# comments allowed)",
+    )
+    build.add_argument(
+        "--trace-path",
+        type=Path,
+        action="append",
+        default=[],
+        help="Trace path to include. Repeatable, can be combined with --traces-file",
+    )
+    add_generate_args(build)
+    build.set_defaults(func=build_cmd)
+
+    search = sub.add_parser("search", help="Full-text search indexed traces")
+    search.add_argument("terms", nargs="+")
+    search.add_argument("--limit", type=int, default=50)
+    search.set_defaults(func=search_cmd)
+
+    sql = sub.add_parser("sqlite-sql", help="Run a read-only SQLite query")
+    sql.add_argument("sql")
+    sql.add_argument("--limit", type=int, default=500)
+    sql.set_defaults(func=sqlite_sql_cmd)
+
+    serve = sub.add_parser("serve", help="Serve read-only HTTP SQL access")
+    serve.add_argument("--host", default="127.0.0.1")
+    serve.add_argument("--port", type=int, default=8765)
+    serve.add_argument("--default-limit", type=int, default=500)
+    serve.add_argument("--max-limit", type=int, default=5000)
+    serve.set_defaults(func=serve_cmd)
+
+    return parser
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    if args.command is None:
+        parser.error("a command is required")
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())