Compare Workshop evals on pull requests

.github/workflows/workshop-evals-pr.yml

@@ -0,0 +1,242 @@
+name: Workshop eval comparison
+
+# Same trust boundary as preview.yml: GitHub withholds repository secrets from fork PRs, and the
+# first job re-verifies that a non-bot same-repository author still has write access before any
+# secret-bearing job starts. Core maintainers' same-repository branch code is trusted.
+
+# Only changes that can move the agent's behaviour, or the machinery that measures it, are worth
+# a 50-minute inference run. Everything else (dependency bumps, UI, auth, sharing, storage) is not.
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      # Eval CI and harness
+      - .github/workflows/workshop-evals-pr.yml
+      - scripts/evals/**
+      - packages/workshop-evals/**
+      - packages/integration-tests/src/**
+      # Agent loop, prompts, tool descriptions, compaction, model routing
+      - packages/workshop-backend/src/agent*
+      - packages/workshop-backend/src/ai-*
+      # Tool implementations and the workspace the agent operates on
+      - packages/workshop-backend/src/overseer.ts
+      - packages/workshop-backend/src/worktree-*
+      - packages/workshop-backend/src/web-fetch.ts
+      # Output formats the prompt steers toward
+      - packages/workshop-backend/format-blueprints/**
+
+permissions: {}
+
+concurrency:
+  group: workshop-evals-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+env:
+  NODE_VERSION: "24.19.0"
+  EVAL_TRIALS: "3"
+
+jobs:
+  trust:
+    name: Verify pull request
+    if: >-
+      github.event.pull_request.head.repo.id == github.event.pull_request.base.repo.id &&
+      github.event.pull_request.user.type != 'Bot' &&
+      github.repository_owner == 'cloudflare'
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    steps:
+      - name: Verify the pull request is from a maintainer
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+          PR_AUTHOR: ${{ github.event.pull_request.user.login }}
+        run: |
+          permission=$(gh api "repos/$GH_REPO/collaborators/$PR_AUTHOR/permission" --jq '.permission')
+          if [[ ! "$permission" =~ ^(admin|maintain|write)$ ]]; then
+            echo "$PR_AUTHOR has '$permission' on $GH_REPO; Workshop evals require write access."
+            exit 1
+          fi
+
+  pr-evals:
+    name: Eval ${{ matrix.revision }}
+    needs: trust
+    runs-on: ubuntu-latest
+    timeout-minutes: 180
+    permissions:
+      actions: read
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - revision: baseline
+            sha: ${{ github.event.pull_request.base.sha }}
+          - revision: candidate
+            sha: ${{ github.event.pull_request.head.sha }}
+    steps:
+      # Baselines are cached per base commit, so a PR can only ever reuse a baseline measured at its
+      # own base. Any trusted PR run may fill the cache; the same maintainer trust that lets it run
+      # candidate code with the gateway token lets it store a result for its base.
+      - name: Find cached baseline
+        if: matrix.revision == 'baseline'
+        id: stored
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+          ARTIFACT_NAME: workshop-evals-baseline-${{ matrix.sha }}
+        run: |
+          artifact_id=$(gh api "repos/$GH_REPO/actions/artifacts?name=$ARTIFACT_NAME&per_page=100" \
+            --jq '[.artifacts[] | select(.expired == false)] | sort_by(.created_at) | last | .id // ""')
+          echo "artifact-id=$artifact_id" >> "$GITHUB_OUTPUT"
+
+      - name: Restore cached baseline
+        if: matrix.revision == 'baseline' && steps.stored.outputs.artifact-id != ''
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+          ARTIFACT_ID: ${{ steps.stored.outputs.artifact-id }}
+        run: |
+          mkdir -p packages/workshop-evals/.wrangler/evals
+          gh api "repos/$GH_REPO/actions/artifacts/$ARTIFACT_ID/zip" > /tmp/workshop-evals-baseline.zip
+          unzip -q /tmp/workshop-evals-baseline.zip -d packages/workshop-evals/.wrangler/evals
+
+      - name: Check out ${{ matrix.revision }}
+        if: matrix.revision == 'candidate' || steps.stored.outputs.artifact-id == ''
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ matrix.sha }}
+          persist-credentials: false
+
+      - name: Enable Corepack
+        if: matrix.revision == 'candidate' || steps.stored.outputs.artifact-id == ''
+        run: corepack enable
+
+      - name: Set up Vite+, Node.js and dependencies
+        if: matrix.revision == 'candidate' || steps.stored.outputs.artifact-id == ''
+        uses: voidzero-dev/setup-vp@313600b80b104eadebb9111787d37a2e83e014ca # v1.17.0
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: true
+          run-install: true
+
+      - name: Run ${{ matrix.revision }} evals
+        if: matrix.revision == 'candidate' || steps.stored.outputs.artifact-id == ''
+        continue-on-error: true
+        # The same AI Gateway and token the manual eval workflow and Bonk already use.
+        env:
+          CF_AI_GATEWAY: ${{ secrets.CF_AI_GATEWAY_NAME }}
+          CF_AI_GATEWAY_ACCOUNT_ID: ${{ secrets.CF_AI_GATEWAY_ACCOUNT_ID }}
+          CF_AI_GATEWAY_API_TOKEN: ${{ secrets.CF_AI_GATEWAY_TOKEN }}
+          WORKSHOP_EVAL_TRIALS: ${{ env.EVAL_TRIALS }}
+          WORKSHOP_EVAL_COMMIT: ${{ matrix.sha }}
+        run: pnpm evals
+
+      # Only a complete, infrastructure-clean baseline is worth sharing; a broken one is still
+      # compared against here (its cohorts read "baseline run errors") but the next PR on this base
+      # measures it again rather than inheriting it.
+      - name: Check whether the baseline is complete
+        if: matrix.revision == 'baseline' && steps.stored.outputs.artifact-id == ''
+        id: complete
+        continue-on-error: true
+        run: >-
+          node scripts/evals/validate-results.ts
+          packages/workshop-evals/.wrangler/evals/results.json "$EVAL_TRIALS"
+
+      - name: Cache the baseline for other pull requests on this base
+        if: steps.complete.outcome == 'success'
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: workshop-evals-baseline-${{ matrix.sha }}
+          path: packages/workshop-evals/.wrangler/evals/results.json
+          include-hidden-files: true
+          if-no-files-found: error
+          retention-days: 90
+
+      - name: Upload current-run ${{ matrix.revision }} trajectories
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: workshop-evals-${{ matrix.revision }}
+          path: packages/workshop-evals/.wrangler/evals/results.json
+          include-hidden-files: true
+          if-no-files-found: error
+          overwrite: true
+          retention-days: 30
+
+  compare:
+    name: Compare evals
+    needs: [trust, pr-evals]
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      actions: read
+      checks: write
+      contents: read
+    steps:
+      - name: Check out candidate
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
+          fetch-depth: 0
+
+
+      - name: Enable Corepack
+        run: corepack enable
+
+      - name: Set up Vite+, Node.js and dependencies
+        uses: voidzero-dev/setup-vp@313600b80b104eadebb9111787d37a2e83e014ca # v1.17.0
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: true
+          run-install: true
+
+      - name: Download baseline trajectories
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
+        with:
+          name: workshop-evals-baseline
+          path: artifacts/baseline
+
+      - name: Download candidate trajectories
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
+        with:
+          name: workshop-evals-candidate
+          path: artifacts/candidate
+
+      # Cohorts are non-comparable when the eval or harness code itself changed between the two
+      # commits: a scorer change moves the goalposts without touching the product under test.
+      - name: Compare eval results
+        run: |
+          node scripts/evals/compare-results.ts \
+            artifacts/baseline/results.json \
+            artifacts/candidate/results.json \
+            artifacts/comparison/comparison.json \
+            artifacts/comparison/comparison.md \
+            packages/integration-tests packages/workshop-evals
+          cat artifacts/comparison/comparison.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload comparison
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: workshop-evals-comparison
+          path: artifacts/comparison
+          if-no-files-found: error
+          overwrite: true
+          retention-days: 30
+
+      - name: Publish comparison check
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          DETAILS_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          jq -n \
+            --arg name "Workshop eval comparison" \
+            --arg head_sha "$HEAD_SHA" \
+            --arg details_url "$DETAILS_URL" \
+            --rawfile summary artifacts/comparison/comparison.md \
+            '{name: $name, head_sha: $head_sha, status: "completed", conclusion: "neutral", details_url: $details_url, output: {title: "Workshop eval comparison", summary: $summary}}' \
+            > /tmp/workshop-evals-check.json
+          gh api --method POST "repos/$GH_REPO/check-runs" --input /tmp/workshop-evals-check.json