evaluation.yml

name: Agent Evaluation

on:
  pull_request:
    paths:
      - 'src/**'
      - 'tests/evals/**'
  schedule:
    - cron: '0 2 * * *'  # Daily at 2 AM UTC
  workflow_dispatch:
    inputs:
      eval_scope:
        description: 'Evaluation scope'
        required: true
        default: 'basic'
        type: choice
        options:
          - basic
          - full
          - quick

env:
  PYTHON_VERSION: '3.12'
  EVAL_OUTPUT_DIR: 'eval_results'

jobs:
  # Determine evaluation scope based on trigger
  setup:
    runs-on: ubuntu-latest
    outputs:
      scope: ${{ steps.determine-scope.outputs.scope }}
    steps:
      - name: Determine evaluation scope
        id: determine-scope
        run: |
          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
            echo "scope=${{ github.event.inputs.eval_scope }}" >> $GITHUB_OUTPUT
          elif [ "${{ github.event_name }}" == "schedule" ]; then
            echo "scope=full" >> $GITHUB_OUTPUT
          else
            echo "scope=basic" >> $GITHUB_OUTPUT
          fi

  evaluate:
    needs: setup
    runs-on: ubuntu-latest
    timeout-minutes: 30

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: 'pip'

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -e ".[dev]"

      - name: Run evaluation
        id: evaluation
        env:
          EVAL_SCOPE: ${{ needs.setup.outputs.scope }}
        run: |
          python scripts/ci_eval_runner.py --scope $EVAL_SCOPE --output $EVAL_OUTPUT_DIR

          # Extract key metrics for summary
          if [ -f "$EVAL_OUTPUT_DIR/summary.json" ]; then
            SUCCESS_RATE=$(python -c "import json; d=json.load(open('$EVAL_OUTPUT_DIR/summary.json')); print(d.get('success_rate', 0))")
            echo "success_rate=$SUCCESS_RATE" >> $GITHUB_OUTPUT
          fi

      - name: Upload evaluation results
        uses: actions/upload-artifact@v4
        with:
          name: evaluation-results-${{ github.run_number }}
          path: ${{ env.EVAL_OUTPUT_DIR }}/
          retention-days: 30

      - name: Check for regression
        if: github.event_name == 'pull_request'
        run: |
          python scripts/ci_eval_runner.py --check-regression --baseline main

  # Post results as PR comment
  comment:
    needs: evaluate
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Download evaluation results
        uses: actions/download-artifact@v4
        with:
          name: evaluation-results-${{ github.run_number }}
          path: eval_results/

      - name: Generate PR comment
        id: generate-comment
        run: |
          python -c "
          import json
          import os

          summary_file = 'eval_results/summary.json'
          if os.path.exists(summary_file):
              with open(summary_file) as f:
                  data = json.load(f)

              comment = '''## 🤖 Agent Evaluation Results

          | Metric | Value |
          |--------|-------|
          | Success Rate | {success_rate:.1%} |
          | Total Tasks | {total_tasks} |
          | Avg Latency | {avg_latency:.2f}s |
          | Pass@1 | {pass_at_1:.3f} |

          <details>
          <summary>📊 Detailed Results</summary>

          \`\`\`json
          {details}
          \`\`\`
          </details>
          '''.format(
                  success_rate=data.get('success_rate', 0),
                  total_tasks=data.get('total_tasks', 0),
                  avg_latency=data.get('avg_latency', 0),
                  pass_at_1=data.get('pass_at_1', 0),
                  details=json.dumps(data, indent=2)[:1000]
              )
          else:
              comment = '## ⚠️ Evaluation results not found'

          # Write to file for the next step
          with open('pr_comment.md', 'w') as f:
              f.write(comment)
          "

      - name: Post PR comment
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const comment = fs.readFileSync('pr_comment.md', 'utf8');

            // Find existing comment
            const { data: comments } = await github.rest.issues.listComments({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
            });

            const botComment = comments.find(c =>
              c.user.type === 'Bot' && c.body.includes('Agent Evaluation Results')
            );

            if (botComment) {
              await github.rest.issues.updateComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: botComment.id,
                body: comment
              });
            } else {
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: context.issue.number,
                body: comment
              });
            }

  # Notify on failure
  notify:
    needs: evaluate
    if: failure() && github.event_name == 'schedule'
    runs-on: ubuntu-latest
    steps:
      - name: Create issue on failure
        uses: actions/github-script@v7
        with:
          script: |
            await github.rest.issues.create({
              owner: context.repo.owner,
              repo: context.repo.repo,
              title: '🚨 Daily Evaluation Failed',
              body: `The scheduled evaluation failed on ${new Date().toISOString()}.

              [View workflow run](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})`,
              labels: ['evaluation', 'automated']
            });