eval.yml

name: Skill Evals

on:
  workflow_dispatch:
    inputs:
      model:
        description: "Model to evaluate against"
        required: true
        default: "claude-sonnet-4.6"
        type: choice
        options:
          - claude-sonnet-4.6
          - claude-opus-4.6
          - gpt-4.1
          - gpt-5.2
      tasks:
        description: "Task filter glob (blank = all)"
        required: false
        type: string
      tags:
        description: "Tag filter (blank = all)"
        required: false
        type: string

jobs:
  eval:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Install waza
        run: curl -fsSL https://raw.githubusercontent.com/microsoft/waza/main/install.sh | bash

      - name: Run evals
        working-directory: evals/tfctl-evals
        env:
          # Uses the Actions-provided GITHUB_TOKEN if org has Copilot enabled.
          # Falls back to COPILOT_TOKEN secret if that doesn't work.
          COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_TOKEN || github.token }}
        run: |
          ARGS="--model ${{ inputs.model }}"
          if [ -n "${{ inputs.tasks }}" ]; then
            ARGS="$ARGS --task '${{ inputs.tasks }}'"
          fi
          if [ -n "${{ inputs.tags }}" ]; then
            ARGS="$ARGS --tags '${{ inputs.tags }}'"
          fi
          eval waza run evals/tfctl/eval.yaml $ARGS -o results.json

      - name: Upload results
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: eval-results-${{ inputs.model }}
          path: evals/tfctl-evals/results.json