diff --git a/.coveragerc b/.coveragerc index c4bb7d51c..496ab14dd 100644 --- a/.coveragerc +++ b/.coveragerc @@ -6,6 +6,7 @@ omit = TraceLens/PerfModel/benchmarking/* TraceLens/PerfModel/origami_helper.py TraceLens/PerfModel/run_perf_model.py + TraceLens/Agent/*/skills/*/evals/* [paths] source = diff --git a/.github/workflows/skill-evals.yml b/.github/workflows/skill-evals.yml new file mode 100644 index 000000000..e631d6a34 --- /dev/null +++ b/.github/workflows/skill-evals.yml @@ -0,0 +1,45 @@ +############################################################################### +# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# +# See LICENSE for license information. +############################################################################### + +# Run skillscope on evals.json (same intake criteria as amd/skills) + +name: AMD Skills Checks + +on: + pull_request: + paths: + - "TraceLens/Agent/Analysis/skills/**" + - ".github/workflows/skill-evals.yml" + workflow_dispatch: + inputs: + structural: + description: Structural Tests + type: boolean + default: true + routing: + description: Routing Tests + type: boolean + default: false + behavioral: + description: Behavioral Tests + type: boolean + default: false + +permissions: + contents: read + +jobs: + evals: + name: AMD Skills Checks + uses: amd/skillscope/.github/workflows/reusable.yml@main + secrets: + api_key: ${{ secrets.ANTHROPIC_API_KEY }} + with: + skills: TraceLens/Agent/Analysis/skills/analysis-orchestrator + # Each step can be set to `required`, `optional`, or `off`. + structural: ${{ github.event_name != 'workflow_dispatch' && 'required' || inputs.structural && 'required' || 'off' }} + routing: ${{ github.event_name == 'workflow_dispatch' && inputs.routing && 'required' || 'off' }} + behavioral: ${{ github.event_name == 'workflow_dispatch' && inputs.behavioral && 'required' || 'off' }} \ No newline at end of file diff --git a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/SKILL.md b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/SKILL.md index d94cdc650..2430e4ddf 100644 --- a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/SKILL.md +++ b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/SKILL.md @@ -1,5 +1,5 @@ --- -name: tracelens-analysis-orchestrator +name: analysis-orchestrator description: >- Orchestrates modular PyTorch profiler trace analysis with TraceLens: generates perf reports, prepares category data, runs system-level and compute-kernel subagents in diff --git a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/evals.json b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/evals.json new file mode 100644 index 000000000..efca33d38 --- /dev/null +++ b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/evals.json @@ -0,0 +1,50 @@ +{ + "evaluations": [ + { + "id": "gemm-01-repeatability", + "skill_should_trigger": true, + "prompt": "Run the full standalone TraceLens analysis workflow on locally with the default analysis mode, and write everything to analysis_output/ in the current directory. The trace ships inside https://github.com/AMD-AGI/TraceLens/raw/main/agent_evals/Analysis/analysis_tests/unit_tests_standalone.tar.gz. Unpack that and analyze agent_evals/Analysis/analysis_tests/unit_tests_standalone/gemm/gemm_01_compute_few_tiles_analysis_output/gemm_01_compute_few_tiles.json.", + "files_exist": [ + "analysis_output/analysis.md", + "analysis_output/perf_report.xlsx", + "analysis_output/category_data/category_manifest.json", + "analysis_output/priority_data.json", + "analysis_output/metadata/model_info.json", + "analysis_output/perf_improvement.png" + ], + "expected_behavior": [ + "Filled the Executive Summary of analysis.md with measured numbers, leaving no template placeholders such as 'X ms', 'Y%' or '' behind" + ] + }, + { + "id": "analyze-torch-trace", + "skill_should_trigger": true, + "prompt": "My training step is way slower than I expected. I ran it under the PyTorch profiler and the output is at https://github.com/AMD-AGI/TraceLens/raw/main/tests/traces/mi300/resnet_act_checkpoint.json.gz. Work out where the time is actually going and write it up so I know what to fix first." + }, + { + "id": "compare-two-traces", + "skill_should_trigger": true, + "prompt": "I profiled the same model on two different machines and one is clearly slower, but I can't tell why. The captures are at https://github.com/AMD-AGI/TraceLens/raw/main/tests/traces/mi300/facebook_timesformer-base-finetuned-k400__1016002.json.gz and https://github.com/AMD-AGI/TraceLens/raw/main/tests/traces/h100/facebook_timesformer-base-finetuned-k400__1016002.json.gz. Tell me which GPU kernels are worse on the slow one, worst first, and write it up." + }, + { + "id": "agentic-analysis-workflow", + "skill_should_trigger": true, + "prompt": "Run the agentic analysis workflow on the trace at https://github.com/AMD-AGI/TraceLens/raw/main/tests/traces/mi300/gaunernst_bert-small-uncased__1016001.json.gz and produce analysis.md." + }, + { + "id": "kernel-perf-report", + "skill_should_trigger": true, + "prompt": "I captured a GPU kernel trace from a distributed training run: https://github.com/AMD-AGI/TraceLens/raw/main/tests/traces/mi300/llama_70b_fsdp/rank0_trace_no_pyfn.json.gz. I need a prioritized report of the worst performance offenders that I can show stakeholders." + }, + { + "id": "node-event-loop", + "skill_should_trigger": false, + "prompt": "My Node.js server shows high event loop latency under load. Help me profile it and find the blocking call." + }, + { + "id": "cprofile-python-script", + "skill_should_trigger": false, + "prompt": "Profile my Python script with cProfile and tell me which function is eating the most time." + } + ] +} diff --git a/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/machine.yml b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/machine.yml new file mode 100644 index 000000000..189cf20c3 --- /dev/null +++ b/TraceLens/Agent/Analysis/skills/analysis-orchestrator/evals/machine.yml @@ -0,0 +1,7 @@ +############################################################################### +# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# +# See LICENSE for license information. +############################################################################### + +os: [Linux]