From ab320baae86037d5853d4659a561476f8ee0d5ee Mon Sep 17 00:00:00 2001 From: Daniel Holanda Date: Wed, 26 Aug 2026 13:19:37 -0700 Subject: [PATCH 01/16] Add evals --- .../analysis-orchestrator/evals/evals.json | 42 +++++ .../analysis-orchestrator/evals/hooks.py | 170 ++++++++++++++++++ .../analysis-orchestrator/evals/machine.yml | 1 + 3 files changed, 213 insertions(+) create mode 100644 TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/evals.json create mode 100644 TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/hooks.py create mode 100644 TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/machine.yml diff --git a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/evals.json b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/evals.json new file mode 100644 index 000000000..f2b082e54 --- /dev/null +++ b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/evals.json @@ -0,0 +1,42 @@ +{ + "evaluations": [ + { + "id": "gemm-01-repeatability", + "skill_should_trigger": true, + "prompt": "Run the full standalone TraceLens analysis workflow locally with analysis mode default.\n- trace_path: {trace_path}\n- platform: {platform}\n- output_dir: {output_dir}\n- venv_path: {venv_path}\n- tracelens_dir: {tracelens_dir}\n", + "files_exist": [ + "analysis_output/analysis.md" + ] + }, + { + "id": "analyze-torch-trace", + "skill_should_trigger": true, + "prompt": "My training step is way slower than I expected. I ran it under the PyTorch profiler and the output is at https://github.com/AMD-AGI/TraceLens/raw/main/tests/traces/mi300/resnet_act_checkpoint.json.gz. Work out where the time is actually going and write it up so I know what to fix first." + }, + { + "id": "compare-two-traces", + "skill_should_trigger": true, + "prompt": "I profiled the same model on two different machines and one is clearly slower, but I can't tell why. The captures are at https://github.com/AMD-AGI/TraceLens/raw/main/tests/traces/mi300/facebook_timesformer-base-finetuned-k400__1016002.json.gz and https://github.com/AMD-AGI/TraceLens/raw/main/tests/traces/h100/facebook_timesformer-base-finetuned-k400__1016002.json.gz. Tell me which GPU kernels are worse on the slow one, worst first, and write it up." + }, + { + "id": "agentic-analysis-workflow", + "skill_should_trigger": true, + "prompt": "Run the agentic analysis workflow on the trace at https://github.com/AMD-AGI/TraceLens/raw/main/tests/traces/mi300/gaunernst_bert-small-uncased__1016001.json.gz and produce analysis.md." + }, + { + "id": "kernel-perf-report", + "skill_should_trigger": true, + "prompt": "I captured a GPU kernel trace from a distributed training run: https://github.com/AMD-AGI/TraceLens/raw/main/tests/traces/mi300/llama_70b_fsdp/rank0_trace_no_pyfn.json.gz. I need a prioritized report of the worst performance offenders that I can show stakeholders." + }, + { + "id": "node-event-loop", + "skill_should_trigger": false, + "prompt": "My Node.js server shows high event loop latency under load. Help me profile it and find the blocking call." + }, + { + "id": "cprofile-python-script", + "skill_should_trigger": false, + "prompt": "Profile my Python script with cProfile and tell me which function is eating the most time." + } + ] +} diff --git a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/hooks.py b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/hooks.py new file mode 100644 index 000000000..e6c2e9acc --- /dev/null +++ b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/hooks.py @@ -0,0 +1,170 @@ +# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# +# See LICENSE for license information. + +"""TraceLens setup and scoring for the `gemm-01-repeatability` behavior case. + +Prompts and expectations live in ``evals.json``; this file holds only what the +dataset format cannot express -- an external checkout, a virtualenv, and a +scoring script that lives in someone else's repo. + +The case mirrors what ``run_repeatability_parallel.sh`` schedules first: the +Phase-1 agent workflow on ``gemm_01_compute_few_tiles`` from +``combined_traces_standalone.csv``, then the first Phase-2 eval, +``workflow_scripted_evals.py``, over whatever the agent produced. + +The runner calls, in order: + + * ``setup_session(cache_dir)`` -- once per run. Clones and installs TraceLens + outside any agent workspace, and returns the paths the prompt interpolates. + * ``setup(workspace, case, ctx)`` -- per case. Creates the output directory + inside the agent's workspace and hands back its absolute path. + * ``check(run, case, ctx)`` -- per case, after grading. Runs TraceLens's + own scorer; anything it flags fails the case. + +Environment overrides: ``TRACELENS_REPO_URL`` and ``TRACELENS_REF``. +""" + +from __future__ import annotations + +import csv +import os +import subprocess +import sys +import tarfile +from pathlib import Path + +TRACELENS_REPO_URL = os.environ.get( + "TRACELENS_REPO_URL", "https://github.com/AMD-AGI/TraceLens.git" +) +TRACELENS_REF = os.environ.get("TRACELENS_REF", "").strip() +UNIT_TESTS_ARCHIVE = "unit_tests_standalone.tar.gz" +ANALYSIS_TESTS = "agent_evals/Analysis/analysis_tests" +COMBINED_TRACES_CSV = f"{ANALYSIS_TESTS}/combined_traces_standalone.csv" + +# The default repeatability order starts here. Asserted rather than assumed, so +# an upstream reordering surfaces as a clear failure instead of silently +# scoring a different case than the one this file documents. +EXPECTED_CASE_ID = "gemm_01_compute_few_tiles" + +# An analysis.md this short is a stub, not a report. +MIN_ANALYSIS_BYTES = 100 + + +def _run(cmd: list[str], *, cwd: Path | None = None) -> None: + proc = subprocess.run( + cmd, + cwd=str(cwd) if cwd else None, + capture_output=True, + text=True, + encoding="utf-8", + check=False, + ) + if proc.returncode != 0: + raise RuntimeError( + f"command failed ({proc.returncode}): {' '.join(cmd)}\n" + f"stdout:\n{proc.stdout}\nstderr:\n{proc.stderr}" + ) + + +def _clone_tracelens(cache_dir: Path) -> Path: + dest = cache_dir / "TraceLens" + if dest.exists(): + return dest + cmd = ["git", "clone", "--depth", "1"] + if TRACELENS_REF: + cmd += ["--branch", TRACELENS_REF] + _run([*cmd, TRACELENS_REPO_URL, str(dest)]) + return dest + + +def _extract_unit_tests(tracelens_dir: Path) -> None: + archive = tracelens_dir / ANALYSIS_TESTS / UNIT_TESTS_ARCHIVE + if not archive.is_file(): + raise FileNotFoundError(f"unit test archive not found: {archive}") + if (tracelens_dir / ANALYSIS_TESTS / "unit_tests_standalone").is_dir(): + return + # Archive members are rooted at agent_evals/Analysis/analysis_tests/... + # under the repo, so extract at tracelens_dir rather than at that subtree. + with tarfile.open(archive, "r:gz") as tar: + tar.extractall(path=tracelens_dir) + + +def _install_tracelens_venv(cache_dir: Path, tracelens_dir: Path) -> Path: + venv_dir = cache_dir / ".venv" + if not venv_dir.exists(): + _run([sys.executable, "-m", "venv", str(venv_dir)], cwd=cache_dir) + pip = venv_dir / "bin" / "pip" + python = venv_dir / "bin" / "python" + _run([str(pip), "install", "--upgrade", "pip"], cwd=cache_dir) + _run([str(pip), "install", "-e", str(tracelens_dir)], cwd=cache_dir) + _run([str(python), "-c", "import TraceLens"], cwd=cache_dir) + return venv_dir + + +def setup_session(cache_dir: Path) -> dict: + """Clone and install TraceLens once, outside any agent workspace.""" + print(" [setup] cloning and installing TraceLens (slow, once per run)", flush=True) + tracelens_dir = _clone_tracelens(cache_dir).resolve() + _extract_unit_tests(tracelens_dir) + + with (tracelens_dir / COMBINED_TRACES_CSV).open(newline="", encoding="utf-8") as handle: + row = next(csv.DictReader(handle)) + if row["id"] != EXPECTED_CASE_ID: + raise RuntimeError( + f"expected the first standalone repeatability case to be " + f"{EXPECTED_CASE_ID}, found {row['id']}; upstream reordered the CSV." + ) + + trace_path = (tracelens_dir / row["trace_path"]).resolve() + if not trace_path.is_file(): + raise FileNotFoundError(f"trace file missing after extract: {trace_path}") + + venv_dir = _install_tracelens_venv(cache_dir, tracelens_dir).resolve() + return { + "tracelens_dir": tracelens_dir, + "venv_path": venv_dir, + "trace_path": trace_path, + "platform": row["platform"], + } + + +def setup(workspace: Path, case, ctx: dict) -> dict: + """Create the output directory the prompt points the agent at.""" + output_dir = workspace / "analysis_output" + output_dir.mkdir(parents=True, exist_ok=True) + return {"output_dir": output_dir} + + +def check(run, case, ctx: dict) -> None: + """Score the agent's report with TraceLens's own Phase-2 eval.""" + output_dir = Path(ctx["output_dir"]) + tracelens_dir = Path(ctx["tracelens_dir"]) + venv_python = Path(ctx["venv_path"]) / "bin" / "python" + + analysis_md = output_dir / "analysis.md" + assert analysis_md.stat().st_size >= MIN_ANALYSIS_BYTES, ( + f"analysis.md is only {analysis_md.stat().st_size} bytes; expected at " + f"least {MIN_ANALYSIS_BYTES}" + ) + + results_csv = output_dir / "workflow_scripted_results.csv" + _run( + [ + str(venv_python), + str(tracelens_dir / "agent_evals/Analysis/eval_utils/workflow_scripted_evals.py"), + "--output-dir", str(output_dir), + "--results", str(results_csv), + "--comparison-scope", "standalone", + ], + cwd=tracelens_dir, + ) + + with results_csv.open(newline="", encoding="utf-8") as handle: + rows = list(csv.DictReader(handle)) + assert rows, f"workflow eval produced no rows: {results_csv}" + + failures = [row for row in rows if row.get("result") != "PASS"] + assert not failures, "workflow_scripted_evals.py reported failures:\n" + "\n".join( + f" - {row.get('issue_summary')}: {row.get('details')}" for row in failures[:10] + ) diff --git a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/machine.yml b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/machine.yml new file mode 100644 index 000000000..78cd19714 --- /dev/null +++ b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/machine.yml @@ -0,0 +1 @@ +os: [Linux] From 359ec3f07c5eec26f50ed3f833f21a02f72711a1 Mon Sep 17 00:00:00 2001 From: Daniel Holanda Date: Wed, 26 Aug 2026 14:34:28 -0700 Subject: [PATCH 02/16] Increase max retries for findings file validation --- .../skills/analysis-orchestrator/agents/norm-analyzer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/agents/norm-analyzer.md b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/agents/norm-analyzer.md index a9ab574a4..99f609c7c 100644 --- a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/agents/norm-analyzer.md +++ b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/agents/norm-analyzer.md @@ -161,4 +161,4 @@ print('PASS: Findings file is valid') " '/category_findings/_findings.md' 'compute' '' ``` -If validation fails, fix the findings file and re-run. Max 2 retries. +If validation fails, fix the findings file and re-run. Max 3 retries. From 68e3b6f7f63a43a073c6c70971cd91128cd35267 Mon Sep 17 00:00:00 2001 From: Daniel Holanda Date: Wed, 26 Aug 2026 14:37:35 -0700 Subject: [PATCH 03/16] Update max retries for findings file validation --- .../skills/analysis-orchestrator/agents/norm-analyzer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/agents/norm-analyzer.md b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/agents/norm-analyzer.md index 99f609c7c..a9ab574a4 100644 --- a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/agents/norm-analyzer.md +++ b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/agents/norm-analyzer.md @@ -161,4 +161,4 @@ print('PASS: Findings file is valid') " '/category_findings/_findings.md' 'compute' '' ``` -If validation fails, fix the findings file and re-run. Max 3 retries. +If validation fails, fix the findings file and re-run. Max 2 retries. From 042816541a2556ac56963d910fdfbe5f128ef88d Mon Sep 17 00:00:00 2001 From: Daniel Holanda Date: Thu, 27 Aug 2026 09:48:36 -0700 Subject: [PATCH 04/16] Run black --- .../skills/analysis-orchestrator/evals/hooks.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/hooks.py b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/hooks.py index e6c2e9acc..b7ac74d86 100644 --- a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/hooks.py +++ b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/hooks.py @@ -41,6 +41,7 @@ UNIT_TESTS_ARCHIVE = "unit_tests_standalone.tar.gz" ANALYSIS_TESTS = "agent_evals/Analysis/analysis_tests" COMBINED_TRACES_CSV = f"{ANALYSIS_TESTS}/combined_traces_standalone.csv" +WORKFLOW_EVAL_SCRIPT = "agent_evals/Analysis/eval_utils/workflow_scripted_evals.py" # The default repeatability order starts here. Asserted rather than assumed, so # an upstream reordering surfaces as a clear failure instead of silently @@ -108,7 +109,9 @@ def setup_session(cache_dir: Path) -> dict: tracelens_dir = _clone_tracelens(cache_dir).resolve() _extract_unit_tests(tracelens_dir) - with (tracelens_dir / COMBINED_TRACES_CSV).open(newline="", encoding="utf-8") as handle: + with (tracelens_dir / COMBINED_TRACES_CSV).open( + newline="", encoding="utf-8" + ) as handle: row = next(csv.DictReader(handle)) if row["id"] != EXPECTED_CASE_ID: raise RuntimeError( @@ -152,10 +155,13 @@ def check(run, case, ctx: dict) -> None: _run( [ str(venv_python), - str(tracelens_dir / "agent_evals/Analysis/eval_utils/workflow_scripted_evals.py"), - "--output-dir", str(output_dir), - "--results", str(results_csv), - "--comparison-scope", "standalone", + str(tracelens_dir / WORKFLOW_EVAL_SCRIPT), + "--output-dir", + str(output_dir), + "--results", + str(results_csv), + "--comparison-scope", + "standalone", ], cwd=tracelens_dir, ) From df75ecc7699d5d63650ee3bc95e6c963ec377764 Mon Sep 17 00:00:00 2001 From: Daniel Holanda Date: Thu, 27 Aug 2026 09:57:50 -0700 Subject: [PATCH 05/16] headers --- .coveragerc | 1 + .../Analysis/skills/analysis-orchestrator/evals/hooks.py | 2 ++ .../Analysis/skills/analysis-orchestrator/evals/machine.yml | 6 ++++++ 3 files changed, 9 insertions(+) diff --git a/.coveragerc b/.coveragerc index c4bb7d51c..496ab14dd 100644 --- a/.coveragerc +++ b/.coveragerc @@ -6,6 +6,7 @@ omit = TraceLens/PerfModel/benchmarking/* TraceLens/PerfModel/origami_helper.py TraceLens/PerfModel/run_perf_model.py + TraceLens/Agent/*/skills/*/evals/* [paths] source = diff --git a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/hooks.py b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/hooks.py index b7ac74d86..a99d9e491 100644 --- a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/hooks.py +++ b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/hooks.py @@ -1,6 +1,8 @@ +############################################################################### # Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. # # See LICENSE for license information. +############################################################################### """TraceLens setup and scoring for the `gemm-01-repeatability` behavior case. diff --git a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/machine.yml b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/machine.yml index 78cd19714..189cf20c3 100644 --- a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/machine.yml +++ b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/machine.yml @@ -1 +1,7 @@ +############################################################################### +# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# +# See LICENSE for license information. +############################################################################### + os: [Linux] From 050978e7a4d14f9cbba7820c5b604378a48801af Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 29 Aug 2026 16:01:50 +0000 Subject: [PATCH 06/16] Sync analysis-orchestrator evals with amd/skills#193 Replace the interpolated gemm-01-repeatability prompt with a self-contained one that fetches and unpacks the standalone unit-test trace itself, widen the expected artifact list, add an executive-summary behavior expectation, and drop the hooks.py clone/venv setup the new prompt no longer needs. Co-authored-by: Daniel Holanda --- .../analysis-orchestrator/evals/evals.json | 12 +- .../analysis-orchestrator/evals/hooks.py | 178 ------------------ 2 files changed, 10 insertions(+), 180 deletions(-) delete mode 100644 TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/hooks.py diff --git a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/evals.json b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/evals.json index f2b082e54..efca33d38 100644 --- a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/evals.json +++ b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/evals.json @@ -3,9 +3,17 @@ { "id": "gemm-01-repeatability", "skill_should_trigger": true, - "prompt": "Run the full standalone TraceLens analysis workflow locally with analysis mode default.\n- trace_path: {trace_path}\n- platform: {platform}\n- output_dir: {output_dir}\n- venv_path: {venv_path}\n- tracelens_dir: {tracelens_dir}\n", + "prompt": "Run the full standalone TraceLens analysis workflow on locally with the default analysis mode, and write everything to analysis_output/ in the current directory. The trace ships inside https://github.com/AMD-AGI/TraceLens/raw/main/agent_evals/Analysis/analysis_tests/unit_tests_standalone.tar.gz. Unpack that and analyze agent_evals/Analysis/analysis_tests/unit_tests_standalone/gemm/gemm_01_compute_few_tiles_analysis_output/gemm_01_compute_few_tiles.json.", "files_exist": [ - "analysis_output/analysis.md" + "analysis_output/analysis.md", + "analysis_output/perf_report.xlsx", + "analysis_output/category_data/category_manifest.json", + "analysis_output/priority_data.json", + "analysis_output/metadata/model_info.json", + "analysis_output/perf_improvement.png" + ], + "expected_behavior": [ + "Filled the Executive Summary of analysis.md with measured numbers, leaving no template placeholders such as 'X ms', 'Y%' or '' behind" ] }, { diff --git a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/hooks.py b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/hooks.py deleted file mode 100644 index a99d9e491..000000000 --- a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/hooks.py +++ /dev/null @@ -1,178 +0,0 @@ -############################################################################### -# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. -# -# See LICENSE for license information. -############################################################################### - -"""TraceLens setup and scoring for the `gemm-01-repeatability` behavior case. - -Prompts and expectations live in ``evals.json``; this file holds only what the -dataset format cannot express -- an external checkout, a virtualenv, and a -scoring script that lives in someone else's repo. - -The case mirrors what ``run_repeatability_parallel.sh`` schedules first: the -Phase-1 agent workflow on ``gemm_01_compute_few_tiles`` from -``combined_traces_standalone.csv``, then the first Phase-2 eval, -``workflow_scripted_evals.py``, over whatever the agent produced. - -The runner calls, in order: - - * ``setup_session(cache_dir)`` -- once per run. Clones and installs TraceLens - outside any agent workspace, and returns the paths the prompt interpolates. - * ``setup(workspace, case, ctx)`` -- per case. Creates the output directory - inside the agent's workspace and hands back its absolute path. - * ``check(run, case, ctx)`` -- per case, after grading. Runs TraceLens's - own scorer; anything it flags fails the case. - -Environment overrides: ``TRACELENS_REPO_URL`` and ``TRACELENS_REF``. -""" - -from __future__ import annotations - -import csv -import os -import subprocess -import sys -import tarfile -from pathlib import Path - -TRACELENS_REPO_URL = os.environ.get( - "TRACELENS_REPO_URL", "https://github.com/AMD-AGI/TraceLens.git" -) -TRACELENS_REF = os.environ.get("TRACELENS_REF", "").strip() -UNIT_TESTS_ARCHIVE = "unit_tests_standalone.tar.gz" -ANALYSIS_TESTS = "agent_evals/Analysis/analysis_tests" -COMBINED_TRACES_CSV = f"{ANALYSIS_TESTS}/combined_traces_standalone.csv" -WORKFLOW_EVAL_SCRIPT = "agent_evals/Analysis/eval_utils/workflow_scripted_evals.py" - -# The default repeatability order starts here. Asserted rather than assumed, so -# an upstream reordering surfaces as a clear failure instead of silently -# scoring a different case than the one this file documents. -EXPECTED_CASE_ID = "gemm_01_compute_few_tiles" - -# An analysis.md this short is a stub, not a report. -MIN_ANALYSIS_BYTES = 100 - - -def _run(cmd: list[str], *, cwd: Path | None = None) -> None: - proc = subprocess.run( - cmd, - cwd=str(cwd) if cwd else None, - capture_output=True, - text=True, - encoding="utf-8", - check=False, - ) - if proc.returncode != 0: - raise RuntimeError( - f"command failed ({proc.returncode}): {' '.join(cmd)}\n" - f"stdout:\n{proc.stdout}\nstderr:\n{proc.stderr}" - ) - - -def _clone_tracelens(cache_dir: Path) -> Path: - dest = cache_dir / "TraceLens" - if dest.exists(): - return dest - cmd = ["git", "clone", "--depth", "1"] - if TRACELENS_REF: - cmd += ["--branch", TRACELENS_REF] - _run([*cmd, TRACELENS_REPO_URL, str(dest)]) - return dest - - -def _extract_unit_tests(tracelens_dir: Path) -> None: - archive = tracelens_dir / ANALYSIS_TESTS / UNIT_TESTS_ARCHIVE - if not archive.is_file(): - raise FileNotFoundError(f"unit test archive not found: {archive}") - if (tracelens_dir / ANALYSIS_TESTS / "unit_tests_standalone").is_dir(): - return - # Archive members are rooted at agent_evals/Analysis/analysis_tests/... - # under the repo, so extract at tracelens_dir rather than at that subtree. - with tarfile.open(archive, "r:gz") as tar: - tar.extractall(path=tracelens_dir) - - -def _install_tracelens_venv(cache_dir: Path, tracelens_dir: Path) -> Path: - venv_dir = cache_dir / ".venv" - if not venv_dir.exists(): - _run([sys.executable, "-m", "venv", str(venv_dir)], cwd=cache_dir) - pip = venv_dir / "bin" / "pip" - python = venv_dir / "bin" / "python" - _run([str(pip), "install", "--upgrade", "pip"], cwd=cache_dir) - _run([str(pip), "install", "-e", str(tracelens_dir)], cwd=cache_dir) - _run([str(python), "-c", "import TraceLens"], cwd=cache_dir) - return venv_dir - - -def setup_session(cache_dir: Path) -> dict: - """Clone and install TraceLens once, outside any agent workspace.""" - print(" [setup] cloning and installing TraceLens (slow, once per run)", flush=True) - tracelens_dir = _clone_tracelens(cache_dir).resolve() - _extract_unit_tests(tracelens_dir) - - with (tracelens_dir / COMBINED_TRACES_CSV).open( - newline="", encoding="utf-8" - ) as handle: - row = next(csv.DictReader(handle)) - if row["id"] != EXPECTED_CASE_ID: - raise RuntimeError( - f"expected the first standalone repeatability case to be " - f"{EXPECTED_CASE_ID}, found {row['id']}; upstream reordered the CSV." - ) - - trace_path = (tracelens_dir / row["trace_path"]).resolve() - if not trace_path.is_file(): - raise FileNotFoundError(f"trace file missing after extract: {trace_path}") - - venv_dir = _install_tracelens_venv(cache_dir, tracelens_dir).resolve() - return { - "tracelens_dir": tracelens_dir, - "venv_path": venv_dir, - "trace_path": trace_path, - "platform": row["platform"], - } - - -def setup(workspace: Path, case, ctx: dict) -> dict: - """Create the output directory the prompt points the agent at.""" - output_dir = workspace / "analysis_output" - output_dir.mkdir(parents=True, exist_ok=True) - return {"output_dir": output_dir} - - -def check(run, case, ctx: dict) -> None: - """Score the agent's report with TraceLens's own Phase-2 eval.""" - output_dir = Path(ctx["output_dir"]) - tracelens_dir = Path(ctx["tracelens_dir"]) - venv_python = Path(ctx["venv_path"]) / "bin" / "python" - - analysis_md = output_dir / "analysis.md" - assert analysis_md.stat().st_size >= MIN_ANALYSIS_BYTES, ( - f"analysis.md is only {analysis_md.stat().st_size} bytes; expected at " - f"least {MIN_ANALYSIS_BYTES}" - ) - - results_csv = output_dir / "workflow_scripted_results.csv" - _run( - [ - str(venv_python), - str(tracelens_dir / WORKFLOW_EVAL_SCRIPT), - "--output-dir", - str(output_dir), - "--results", - str(results_csv), - "--comparison-scope", - "standalone", - ], - cwd=tracelens_dir, - ) - - with results_csv.open(newline="", encoding="utf-8") as handle: - rows = list(csv.DictReader(handle)) - assert rows, f"workflow eval produced no rows: {results_csv}" - - failures = [row for row in rows if row.get("result") != "PASS"] - assert not failures, "workflow_scripted_evals.py reported failures:\n" + "\n".join( - f" - {row.get('issue_summary')}: {row.get('details')}" for row in failures[:10] - ) From 6a0b5b00474cf93bbc41842217871e9a01655a71 Mon Sep 17 00:00:00 2001 From: Daniel Holanda Date: Sat, 29 Aug 2026 13:27:43 -0700 Subject: [PATCH 07/16] Run the analysis-orchestrator evals with skillscope (#3) * Match the skill's frontmatter name to its folder The Agent Skills format requires `name` to equal the directory name, and skillscope's structural check fails on the mismatch. Federation into amd/skills rewrites the name to the `as:` value, so the catalog copy stays `tracelens-analysis-orchestrator`. Co-authored-by: Daniel Holanda * Add skillscope skill-evals workflow Runs the structural, routing, and behavior evals for TraceLens/Agent/Analysis/skills against danielholanda/skillscope. Co-authored-by: Daniel Holanda * Name the skill in skill-evals instead of deriving it Drops the SKILL_GLOBS indirection and `--routing-skills all` in favour of naming analysis-orchestrator where the choice is made, which also makes the routing comment unnecessary. Co-authored-by: Daniel Holanda * Point skill-evals at the skill folder rather than a glob Naming the one skill makes the behavior job's --skill redundant, so it goes too; --routing-skills stays because routing has no default. Co-authored-by: Daniel Holanda --------- Co-authored-by: Cursor Agent --- .github/workflows/skill-evals.yml | 61 +++++++++++++++++++ .../skills/analysis-orchestrator/SKILL.md | 2 +- 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/skill-evals.yml diff --git a/.github/workflows/skill-evals.yml b/.github/workflows/skill-evals.yml new file mode 100644 index 000000000..d72b373ee --- /dev/null +++ b/.github/workflows/skill-evals.yml @@ -0,0 +1,61 @@ +############################################################################### +# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# +# See LICENSE for license information. +############################################################################### + +# Grades this repo's skills with https://github.com/danielholanda/skillscope: +# structure (free), then routing (which skill fires) and behavior (what the +# agent did once it fired). + +name: skill-evals + +on: + pull_request: + paths: + - "TraceLens/Agent/Analysis/skills/**" + - ".github/workflows/skill-evals.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + structural: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: danielholanda/skillscope@main + with: + command: structural + skills: TraceLens/Agent/Analysis/skills/analysis-orchestrator + + routing: + needs: structural + runs-on: ubuntu-latest + timeout-minutes: 40 + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + steps: + - uses: actions/checkout@v4 + - uses: danielholanda/skillscope@main + with: + command: run + args: --mode routing --routing-skills analysis-orchestrator + skills: TraceLens/Agent/Analysis/skills/analysis-orchestrator + install-claude: "true" + + behavior: + needs: structural + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + steps: + - uses: actions/checkout@v4 + - uses: danielholanda/skillscope@main + with: + command: run + args: --mode behavior + skills: TraceLens/Agent/Analysis/skills/analysis-orchestrator + install-claude: "true" diff --git a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/SKILL.md b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/SKILL.md index b297db9d4..870652231 100644 --- a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/SKILL.md +++ b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/SKILL.md @@ -1,5 +1,5 @@ --- -name: tracelens-analysis-orchestrator +name: analysis-orchestrator description: >- Orchestrates modular PyTorch profiler trace analysis with TraceLens: generates perf reports, prepares category data, runs system-level and compute-kernel subagents in From 3bf24f1db78563893d533a4570019553915e7591 Mon Sep 17 00:00:00 2001 From: Daniel Holanda Date: Sat, 29 Aug 2026 15:38:10 -0700 Subject: [PATCH 08/16] Point skill-evals at skillscope's first-class commands (#4) Routing and behavioral are commands now, not modes of `run`. A single-skill repo also no longer has to name its routing set, so the args lines go away. Co-authored-by: Cursor Agent --- .github/workflows/skill-evals.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/skill-evals.yml b/.github/workflows/skill-evals.yml index d72b373ee..934859af9 100644 --- a/.github/workflows/skill-evals.yml +++ b/.github/workflows/skill-evals.yml @@ -5,7 +5,7 @@ ############################################################################### # Grades this repo's skills with https://github.com/danielholanda/skillscope: -# structure (free), then routing (which skill fires) and behavior (what the +# structure (free), then routing (which skill fires) and behavioral (what the # agent did once it fired). name: skill-evals @@ -40,12 +40,11 @@ jobs: - uses: actions/checkout@v4 - uses: danielholanda/skillscope@main with: - command: run - args: --mode routing --routing-skills analysis-orchestrator + command: routing skills: TraceLens/Agent/Analysis/skills/analysis-orchestrator install-claude: "true" - behavior: + behavioral: needs: structural runs-on: ubuntu-latest timeout-minutes: 90 @@ -55,7 +54,6 @@ jobs: - uses: actions/checkout@v4 - uses: danielholanda/skillscope@main with: - command: run - args: --mode behavior + command: behavioral skills: TraceLens/Agent/Analysis/skills/analysis-orchestrator install-claude: "true" From 83c8949184c9f049421be4b322c5481882c6e4c3 Mon Sep 17 00:00:00 2001 From: Daniel Holanda Date: Mon, 31 Aug 2026 09:27:39 -0700 Subject: [PATCH 09/16] Use reusable workflow --- .github/workflows/skill-evals.yml | 51 ++++++++----------------------- 1 file changed, 12 insertions(+), 39 deletions(-) diff --git a/.github/workflows/skill-evals.yml b/.github/workflows/skill-evals.yml index 934859af9..6541ffaf0 100644 --- a/.github/workflows/skill-evals.yml +++ b/.github/workflows/skill-evals.yml @@ -4,9 +4,7 @@ # See LICENSE for license information. ############################################################################### -# Grades this repo's skills with https://github.com/danielholanda/skillscope: -# structure (free), then routing (which skill fires) and behavioral (what the -# agent did once it fired). +# Run skillscope on evals.json (same intake criteria as amd/skills) name: skill-evals @@ -21,39 +19,14 @@ permissions: contents: read jobs: - structural: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: danielholanda/skillscope@main - with: - command: structural - skills: TraceLens/Agent/Analysis/skills/analysis-orchestrator - - routing: - needs: structural - runs-on: ubuntu-latest - timeout-minutes: 40 - env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - steps: - - uses: actions/checkout@v4 - - uses: danielholanda/skillscope@main - with: - command: routing - skills: TraceLens/Agent/Analysis/skills/analysis-orchestrator - install-claude: "true" - - behavioral: - needs: structural - runs-on: ubuntu-latest - timeout-minutes: 90 - env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - steps: - - uses: actions/checkout@v4 - - uses: danielholanda/skillscope@main - with: - command: behavioral - skills: TraceLens/Agent/Analysis/skills/analysis-orchestrator - install-claude: "true" + evals: + uses: danielholanda/skillscope/.github/workflows/reusable.yml@main + secrets: + api_key: ${{ secrets.ANTHROPIC_API_KEY }} + with: + skills: TraceLens/Agent/Analysis/skills/analysis-orchestrator + # Each step can be set to `required`, `optional`, or `off`. + # On amd/skills, all steps are required. + structural: required + routing: required + behavioral: required \ No newline at end of file From 968ffee35c577f8f1a4b969d77de4aece2b303c4 Mon Sep 17 00:00:00 2001 From: Daniel Holanda Date: Mon, 31 Aug 2026 09:59:51 -0700 Subject: [PATCH 10/16] Better eval name --- .github/workflows/skill-evals.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/skill-evals.yml b/.github/workflows/skill-evals.yml index 6541ffaf0..ed43861d0 100644 --- a/.github/workflows/skill-evals.yml +++ b/.github/workflows/skill-evals.yml @@ -6,7 +6,7 @@ # Run skillscope on evals.json (same intake criteria as amd/skills) -name: skill-evals +name: AMD Skills Checks on: pull_request: From d2d37675424afe9115cef65120c5a209495f4de0 Mon Sep 17 00:00:00 2001 From: Daniel Holanda Date: Mon, 31 Aug 2026 10:18:53 -0700 Subject: [PATCH 11/16] Tergets --- .github/workflows/skill-evals.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/skill-evals.yml b/.github/workflows/skill-evals.yml index ed43861d0..7f05a9d55 100644 --- a/.github/workflows/skill-evals.yml +++ b/.github/workflows/skill-evals.yml @@ -20,6 +20,7 @@ permissions: jobs: evals: + name: AMD Skills Checks uses: danielholanda/skillscope/.github/workflows/reusable.yml@main secrets: api_key: ${{ secrets.ANTHROPIC_API_KEY }} From 5af9b4a384837bfcb51ed9292a75e337b1cb41ef Mon Sep 17 00:00:00 2001 From: Daniel Holanda Date: Wed, 2 Sep 2026 13:50:10 -0700 Subject: [PATCH 12/16] Point to upstream repo --- .github/workflows/skill-evals.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/skill-evals.yml b/.github/workflows/skill-evals.yml index 7f05a9d55..d126d59c4 100644 --- a/.github/workflows/skill-evals.yml +++ b/.github/workflows/skill-evals.yml @@ -21,7 +21,7 @@ permissions: jobs: evals: name: AMD Skills Checks - uses: danielholanda/skillscope/.github/workflows/reusable.yml@main + uses: amd/skillscope/.github/workflows/reusable.yml@main secrets: api_key: ${{ secrets.ANTHROPIC_API_KEY }} with: From 61fc0d7fc59b51baee2dea1eb60e08dc549fe52f Mon Sep 17 00:00:00 2001 From: Daniel Holanda Date: Thu, 3 Sep 2026 06:27:48 -0700 Subject: [PATCH 13/16] Add options to dispatchable workflow --- .github/workflows/skill-evals.yml | 29 ++++++++++++++++--- .../skills/analysis-orchestrator/SKILL.md | 2 +- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/.github/workflows/skill-evals.yml b/.github/workflows/skill-evals.yml index d126d59c4..09ebc2b98 100644 --- a/.github/workflows/skill-evals.yml +++ b/.github/workflows/skill-evals.yml @@ -14,6 +14,28 @@ on: - "TraceLens/Agent/Analysis/skills/**" - ".github/workflows/skill-evals.yml" workflow_dispatch: + inputs: + structural: + description: Structural evals + type: choice + options: + - required + - off + default: required + routing: + description: Routing evals + type: choice + options: + - required + - off + default: off + behavioral: + description: Behavioral evals + type: choice + options: + - required + - off + default: off permissions: contents: read @@ -27,7 +49,6 @@ jobs: with: skills: TraceLens/Agent/Analysis/skills/analysis-orchestrator # Each step can be set to `required`, `optional`, or `off`. - # On amd/skills, all steps are required. - structural: required - routing: required - behavioral: required \ No newline at end of file + structural: ${{ github.event.inputs.structural || 'required' }} + routing: ${{ github.event.inputs.routing || 'off' }} + behavioral: ${{ github.event.inputs.behavioral || 'off' }} \ No newline at end of file diff --git a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/SKILL.md b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/SKILL.md index 870652231..b297db9d4 100644 --- a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/SKILL.md +++ b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/SKILL.md @@ -1,5 +1,5 @@ --- -name: analysis-orchestrator +name: tracelens-analysis-orchestrator description: >- Orchestrates modular PyTorch profiler trace analysis with TraceLens: generates perf reports, prepares category data, runs system-level and compute-kernel subagents in From 1b6fcb3364d16771e5ea027a1988329ca51e0ca5 Mon Sep 17 00:00:00 2001 From: Daniel Holanda Date: Thu, 3 Sep 2026 06:31:43 -0700 Subject: [PATCH 14/16] Ensure folder and skill match names --- TraceLens/Agent/Analysis/skills/analysis-orchestrator/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/SKILL.md b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/SKILL.md index b297db9d4..870652231 100644 --- a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/SKILL.md +++ b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/SKILL.md @@ -1,5 +1,5 @@ --- -name: tracelens-analysis-orchestrator +name: analysis-orchestrator description: >- Orchestrates modular PyTorch profiler trace analysis with TraceLens: generates perf reports, prepares category data, runs system-level and compute-kernel subagents in From ac83bc819abaf313eb7f5d8c2e596d5e1531cf53 Mon Sep 17 00:00:00 2001 From: Daniel Holanda Date: Thu, 3 Sep 2026 06:38:41 -0700 Subject: [PATCH 15/16] Trigger workflows with checkboxes --- .github/workflows/skill-evals.yml | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/.github/workflows/skill-evals.yml b/.github/workflows/skill-evals.yml index 09ebc2b98..c8f8cf530 100644 --- a/.github/workflows/skill-evals.yml +++ b/.github/workflows/skill-evals.yml @@ -17,25 +17,16 @@ on: inputs: structural: description: Structural evals - type: choice - options: - - required - - off - default: required + type: boolean + default: true routing: description: Routing evals - type: choice - options: - - required - - off - default: off + type: boolean + default: false behavioral: description: Behavioral evals - type: choice - options: - - required - - off - default: off + type: boolean + default: false permissions: contents: read @@ -49,6 +40,6 @@ jobs: with: skills: TraceLens/Agent/Analysis/skills/analysis-orchestrator # Each step can be set to `required`, `optional`, or `off`. - structural: ${{ github.event.inputs.structural || 'required' }} - routing: ${{ github.event.inputs.routing || 'off' }} - behavioral: ${{ github.event.inputs.behavioral || 'off' }} \ No newline at end of file + structural: ${{ github.event_name != 'workflow_dispatch' && 'required' || inputs.structural && 'required' || 'off' }} + routing: ${{ github.event_name == 'workflow_dispatch' && inputs.routing && 'required' || 'off' }} + behavioral: ${{ github.event_name == 'workflow_dispatch' && inputs.behavioral && 'required' || 'off' }} \ No newline at end of file From 14252149f0b48b3178dcd7cc80927e81f580b351 Mon Sep 17 00:00:00 2001 From: Daniel Holanda Date: Thu, 3 Sep 2026 06:42:56 -0700 Subject: [PATCH 16/16] Fix naming --- .github/workflows/skill-evals.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/skill-evals.yml b/.github/workflows/skill-evals.yml index c8f8cf530..e631d6a34 100644 --- a/.github/workflows/skill-evals.yml +++ b/.github/workflows/skill-evals.yml @@ -16,15 +16,15 @@ on: workflow_dispatch: inputs: structural: - description: Structural evals + description: Structural Tests type: boolean default: true routing: - description: Routing evals + description: Routing Tests type: boolean default: false behavioral: - description: Behavioral evals + description: Behavioral Tests type: boolean default: false