From b20cf5d3a41e2ca5cb29f92d6a7cfe89b31527b0 Mon Sep 17 00:00:00 2001 From: Ashish Kumar Singh Date: Thu, 10 Sep 2026 17:04:21 -0500 Subject: [PATCH] Compare Workshop evals on pull requests --- .github/workflows/workshop-evals-pr.yml | 248 ++++++++++++++++ .../workshop-evals/src/comparison.test.ts | 211 +++++++++++++ packages/workshop-evals/src/comparison.ts | 279 ++++++++++++++++++ scripts/evals/compare-results.ts | 83 ++++++ scripts/evals/validate-results.ts | 40 +++ 5 files changed, 861 insertions(+) create mode 100644 .github/workflows/workshop-evals-pr.yml create mode 100644 packages/workshop-evals/src/comparison.test.ts create mode 100644 packages/workshop-evals/src/comparison.ts create mode 100644 scripts/evals/compare-results.ts create mode 100644 scripts/evals/validate-results.ts diff --git a/.github/workflows/workshop-evals-pr.yml b/.github/workflows/workshop-evals-pr.yml new file mode 100644 index 000000000..97ac48948 --- /dev/null +++ b/.github/workflows/workshop-evals-pr.yml @@ -0,0 +1,248 @@ +name: Workshop eval comparison + +# Same trust boundary as preview.yml: GitHub withholds repository secrets from fork PRs, and the +# first job re-verifies that a non-bot same-repository author still has write access before any +# secret-bearing job starts. Core maintainers' same-repository branch code is trusted. + +# Only changes that can move the agent's behaviour, or the machinery that measures it, are worth +# a 50-minute inference run. Everything else (dependency bumps, UI, auth, sharing, storage) is not. +on: + pull_request: + # `closed` starts a run that does nothing but, through the concurrency group, cancels an eval + # run still spending inference on a pull request nobody will look at. + types: [opened, synchronize, reopened, closed] + paths: + # Eval CI and harness + - .github/workflows/workshop-evals-pr.yml + - scripts/evals/** + - packages/workshop-evals/** + - packages/integration-tests/src/** + # Agent loop, prompts, tool descriptions, compaction, model routing; admin-config feeds the + # instance instructions and the format list into the system prompt + - packages/workshop-backend/src/agent* + - packages/workshop-backend/src/ai-* + - packages/workshop-backend/src/admin-config.ts + # Tool implementations and the workspace the agent operates on + - packages/workshop-backend/src/overseer.ts + - packages/workshop-backend/src/worktree-* + - packages/workshop-backend/src/web-fetch.ts + # Output formats the prompt steers toward + - packages/workshop-backend/format-blueprints/** + +permissions: {} + +concurrency: + group: workshop-evals-${{ github.event.pull_request.number }} + cancel-in-progress: true + +env: + NODE_VERSION: "24.19.0" + EVAL_TRIALS: "3" + +jobs: + trust: + name: Verify pull request + if: >- + github.event.action != 'closed' && + github.event.pull_request.head.repo.id == github.event.pull_request.base.repo.id && + github.event.pull_request.user.type != 'Bot' && + github.repository_owner == 'cloudflare' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + steps: + - name: Verify the pull request is from a maintainer + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PR_AUTHOR: ${{ github.event.pull_request.user.login }} + run: | + permission=$(gh api "repos/$GH_REPO/collaborators/$PR_AUTHOR/permission" --jq '.permission') + if [[ ! "$permission" =~ ^(admin|maintain|write)$ ]]; then + echo "$PR_AUTHOR has '$permission' on $GH_REPO; Workshop evals require write access." + exit 1 + fi + + pr-evals: + name: Eval ${{ matrix.revision }} + needs: trust + runs-on: ubuntu-latest + timeout-minutes: 180 + permissions: + actions: read + contents: read + strategy: + fail-fast: false + matrix: + include: + - revision: baseline + sha: ${{ github.event.pull_request.base.sha }} + - revision: candidate + sha: ${{ github.event.pull_request.head.sha }} + steps: + # Baselines are cached per base commit, so a PR can only ever reuse a baseline measured at its + # own base. Any trusted PR run may fill the cache; the same maintainer trust that lets it run + # candidate code with the gateway token lets it store a result for its base. + - name: Find cached baseline + if: matrix.revision == 'baseline' + id: stored + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + # The trial count comes from this workflow file, so it is part of what a baseline is. + ARTIFACT_NAME: workshop-evals-baseline-${{ matrix.sha }}-${{ env.EVAL_TRIALS }} + run: | + artifact_id=$(gh api "repos/$GH_REPO/actions/artifacts?name=$ARTIFACT_NAME&per_page=100" \ + --jq '[.artifacts[] | select(.expired == false)] | sort_by(.created_at) | last | .id // ""') + echo "artifact-id=$artifact_id" >> "$GITHUB_OUTPUT" + + - name: Restore cached baseline + if: matrix.revision == 'baseline' && steps.stored.outputs.artifact-id != '' + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + ARTIFACT_ID: ${{ steps.stored.outputs.artifact-id }} + run: | + mkdir -p packages/workshop-evals/.wrangler/evals + gh api "repos/$GH_REPO/actions/artifacts/$ARTIFACT_ID/zip" > /tmp/workshop-evals-baseline.zip + unzip -q /tmp/workshop-evals-baseline.zip -d packages/workshop-evals/.wrangler/evals + + - name: Check out ${{ matrix.revision }} + if: matrix.revision == 'candidate' || steps.stored.outputs.artifact-id == '' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ matrix.sha }} + persist-credentials: false + + - name: Enable Corepack + if: matrix.revision == 'candidate' || steps.stored.outputs.artifact-id == '' + run: corepack enable + + - name: Set up Vite+, Node.js and dependencies + if: matrix.revision == 'candidate' || steps.stored.outputs.artifact-id == '' + uses: voidzero-dev/setup-vp@313600b80b104eadebb9111787d37a2e83e014ca # v1.17.0 + with: + node-version: ${{ env.NODE_VERSION }} + cache: true + run-install: true + + - name: Run ${{ matrix.revision }} evals + if: matrix.revision == 'candidate' || steps.stored.outputs.artifact-id == '' + continue-on-error: true + # The same AI Gateway and token the manual eval workflow and Bonk already use. + env: + CF_AI_GATEWAY: ${{ secrets.CF_AI_GATEWAY_NAME }} + CF_AI_GATEWAY_ACCOUNT_ID: ${{ secrets.CF_AI_GATEWAY_ACCOUNT_ID }} + CF_AI_GATEWAY_API_TOKEN: ${{ secrets.CF_AI_GATEWAY_TOKEN }} + WORKSHOP_EVAL_TRIALS: ${{ env.EVAL_TRIALS }} + WORKSHOP_EVAL_COMMIT: ${{ matrix.sha }} + run: pnpm evals + + # Only a complete, infrastructure-clean baseline is worth sharing; a broken one is still + # compared against here (its cohorts read "baseline run errors") but the next PR on this base + # measures it again rather than inheriting it. + - name: Check whether the baseline is complete + if: matrix.revision == 'baseline' && steps.stored.outputs.artifact-id == '' + id: complete + continue-on-error: true + run: >- + node scripts/evals/validate-results.ts + packages/workshop-evals/.wrangler/evals/results.json "$EVAL_TRIALS" + + - name: Cache the baseline for other pull requests on this base + if: steps.complete.outcome == 'success' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: workshop-evals-baseline-${{ matrix.sha }}-${{ env.EVAL_TRIALS }} + path: packages/workshop-evals/.wrangler/evals/results.json + include-hidden-files: true + if-no-files-found: error + retention-days: 90 + + - name: Upload current-run ${{ matrix.revision }} trajectories + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: workshop-evals-${{ matrix.revision }} + path: packages/workshop-evals/.wrangler/evals/results.json + include-hidden-files: true + if-no-files-found: error + overwrite: true + retention-days: 30 + + compare: + name: Compare evals + needs: [trust, pr-evals] + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: read + checks: write + contents: read + steps: + - name: Check out candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + fetch-depth: 0 + + + - name: Enable Corepack + run: corepack enable + + - name: Set up Vite+, Node.js and dependencies + uses: voidzero-dev/setup-vp@313600b80b104eadebb9111787d37a2e83e014ca # v1.17.0 + with: + node-version: ${{ env.NODE_VERSION }} + cache: true + run-install: true + + - name: Download baseline trajectories + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + with: + name: workshop-evals-baseline + path: artifacts/baseline + + - name: Download candidate trajectories + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + with: + name: workshop-evals-candidate + path: artifacts/candidate + + # Cohorts are non-comparable when the eval or harness code itself changed between the two + # commits: a scorer change moves the goalposts without touching the product under test. + - name: Compare eval results + run: | + node scripts/evals/compare-results.ts \ + artifacts/baseline/results.json \ + artifacts/candidate/results.json \ + artifacts/comparison/comparison.json \ + artifacts/comparison/comparison.md \ + packages/integration-tests packages/workshop-evals + cat artifacts/comparison/comparison.md >> "$GITHUB_STEP_SUMMARY" + + - name: Upload comparison + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: workshop-evals-comparison + path: artifacts/comparison + if-no-files-found: error + overwrite: true + retention-days: 30 + + - name: Publish comparison check + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + DETAILS_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + jq -n \ + --arg name "Workshop eval comparison" \ + --arg head_sha "$HEAD_SHA" \ + --arg details_url "$DETAILS_URL" \ + --rawfile summary artifacts/comparison/comparison.md \ + '{name: $name, head_sha: $head_sha, status: "completed", conclusion: "neutral", details_url: $details_url, output: {title: "Workshop eval comparison", summary: $summary}}' \ + > /tmp/workshop-evals-check.json + gh api --method POST "repos/$GH_REPO/check-runs" --input /tmp/workshop-evals-check.json diff --git a/packages/workshop-evals/src/comparison.test.ts b/packages/workshop-evals/src/comparison.test.ts new file mode 100644 index 000000000..1305893b4 --- /dev/null +++ b/packages/workshop-evals/src/comparison.test.ts @@ -0,0 +1,211 @@ +import { expect, it } from "vitest"; +import { compareEvalResults, renderEvalComparison, validateEvalResults } from "./comparison.js"; + +const MODEL = "@cf/deepseek-ai/deepseek-v4-pro-0813"; +const BASE_SHA = "a".repeat(40); +const HEAD_SHA = "b".repeat(40); +const VERSION = "c".repeat(64); + +type TrialOptions = { + taskId?: string; + taskVersion?: string; + gitCommit?: string; + status?: "passed" | "failed"; + duration?: number; + modelTurns?: number; + toolCalls?: number; + toolErrors?: number; + cost?: number; + errors?: { name: string; message: string }[]; + outcomeStatus?: "completed" | "error" | "timedOut" | "cancelled"; +}; + +function trial(options: TrialOptions = {}) { + const { + taskId = "project-doc", + taskVersion = VERSION, + gitCommit = BASE_SHA, + status = "passed", + duration = 100, + modelTurns = 2, + toolCalls = 3, + toolErrors = 0, + cost, + errors = [], + outcomeStatus = "completed", + } = options; + return { + status, + duration, + meta: { + harness: { + run: { + session: { metadata: { taskId, taskVersion, gitCommit } }, + usage: { + model: MODEL, + metadata: cost === undefined ? {} : { observedCumulativeChatCostUsd: cost }, + }, + output: { + metrics: { modelTurns, toolCalls, toolErrors }, + turns: [{ outcome: { status: outcomeStatus } }], + }, + errors, + }, + }, + }, + }; +} + +function report( + assertions: ReturnType[], + ...emptyFiles: { name: string; message: string }[]): string { + return JSON.stringify({ testResults: [ + { name: "/evals/project-doc.eval.ts", assertionResults: assertions }, + ...emptyFiles.map(file => ({ ...file, assertionResults: [] })), + ] }); +} + +it("compares three-trial task cohorts", () => { + const baseline = report([ + trial({ status: "passed", duration: 100, cost: 0.1 }), + trial({ status: "failed", duration: 200, toolErrors: 1, cost: 0.2 }), + trial({ status: "passed", duration: 300, cost: 0.3 }), + ]); + const candidate = report([ + trial({ gitCommit: HEAD_SHA, duration: 200, cost: 0.2 }), + trial({ gitCommit: HEAD_SHA, duration: 300, cost: 0.3 }), + trial({ gitCommit: HEAD_SHA, duration: 400, cost: 0.4 }), + ]); + + const comparison = compareEvalResults(baseline, candidate); + + expect(comparison.baselineSha).toBe(BASE_SHA); + expect(comparison.candidateSha).toBe(HEAD_SHA); + expect(comparison.rows).toEqual([{ + taskId: "project-doc", + model: MODEL, + reason: null, + baseline: { + trials: 3, + passed: 2, + meanDurationMs: 200, + meanModelTurns: 2, + meanToolCalls: 3, + meanToolErrors: 1 / 3, + meanCostUsd: (0.1 + 0.2 + 0.3) / 3, + }, + candidate: { + trials: 3, + passed: 3, + meanDurationMs: 300, + meanModelTurns: 2, + meanToolCalls: 3, + meanToolErrors: 0, + meanCostUsd: (0.2 + 0.3 + 0.4) / 3, + }, + }]); + expect(renderEvalComparison(comparison)).toContain("+33.3 pp"); +}); + +it("does not compare costs from different trial populations", () => { + const baseline = report([trial({ cost: 0.1 }), trial(), trial({ cost: 0.3 })]); + const candidate = report([ + trial({ gitCommit: HEAD_SHA, cost: 0.2 }), + trial({ gitCommit: HEAD_SHA, cost: 0.3 }), + trial({ gitCommit: HEAD_SHA, cost: 0.4 }), + ]); + + const row = compareEvalResults(baseline, candidate).rows[0]; + expect(row.baseline?.meanCostUsd).toBeNull(); + expect(row.candidate?.meanCostUsd).toBeCloseTo(0.3); +}); + +it("separates infrastructure errors from failed agent outcomes", () => { + const baselineError = report([ + trial({ errors: [{ name: "EvalCleanupError", message: "Cleanup failed." }] }), + trial(), + trial(), + ]); + const baseline = report([trial(), trial(), trial()]); + const candidateInfrastructure = report([ + trial({ gitCommit: HEAD_SHA, status: "failed", errors: [{ + name: "EvalRunError", message: "Verifier failed.", + }] }), + trial({ gitCommit: HEAD_SHA }), + trial({ gitCommit: HEAD_SHA }), + ]); + const candidateAgentFailure = report([ + trial({ gitCommit: HEAD_SHA, status: "failed", errors: [{ + name: "AgentError", message: "Agent stopped.", + }] }), + trial({ gitCommit: HEAD_SHA, status: "failed", outcomeStatus: "timedOut", errors: [{ + name: "AgentTimeout", message: "Agent timed out.", + }, { + name: "EvalRunError", message: "Agent timed out.", + }] }), + trial({ gitCommit: HEAD_SHA }), + ]); + + expect(compareEvalResults(baselineError, candidateAgentFailure).rows[0].reason) + .toBe("baseline run errors"); + expect(compareEvalResults(baseline, candidateInfrastructure).rows[0].reason) + .toBe("candidate run errors"); + expect(compareEvalResults(baseline, candidateAgentFailure).rows[0]).toMatchObject({ + reason: null, + candidate: { trials: 3, passed: 1 }, + }); +}); + +it("does not compare changed tasks or unequal trial counts", () => { + const baseline = report([trial(), trial(), trial()]); + const changed = report([ + trial({ gitCommit: HEAD_SHA, taskVersion: "changed" }), + trial({ gitCommit: HEAD_SHA, taskVersion: "changed" }), + trial({ gitCommit: HEAD_SHA, taskVersion: "changed" }), + ]); + const shorter = report([ + trial({ gitCommit: HEAD_SHA }), + trial({ gitCommit: HEAD_SHA }), + ]); + + expect(compareEvalResults(baseline, changed).rows[0].reason).toBe("task version changed"); + const asked: string[][] = []; + expect(compareEvalResults(baseline, changed, (...shas) => { + asked.push(shas); + return true; + }).rows[0].reason).toBe("eval definition changed"); + expect(asked).toEqual([[BASE_SHA, HEAD_SHA]]); + expect(compareEvalResults(baseline, shorter).rows[0].reason).toBe("trial counts differ"); +}); + +it("accepts a complete baseline with agent failures but not infrastructure failures", () => { + const complete = report([ + trial(), + trial({ status: "failed", errors: [{ name: "AgentError", message: "Agent stopped." }] }), + trial({ taskId: "expense-ledger" }), + trial({ taskId: "expense-ledger" }), + ]); + const short = report([trial(), trial(), trial({ taskId: "expense-ledger" })]); + const infrastructure = report([ + trial(), + trial({ status: "failed", errors: [{ name: "EvalRunError", message: "Verifier failed." }] }), + ]); + const mixedCommits = report([trial(), trial({ gitCommit: HEAD_SHA })]); + const uncollected = report( + [trial(), trial()], + { name: "/evals/appointment-desk.eval.ts", message: "Cannot find module './verifier.js'" }); + + expect(() => validateEvalResults(complete, 2)).not.toThrow(); + expect(() => validateEvalResults(short, 2)).toThrow("expense-ledger on"); + expect(() => validateEvalResults(infrastructure, 2)).toThrow("infrastructure failures"); + expect(() => validateEvalResults(mixedCommits, 2)).toThrow("inconsistent commits"); + expect(() => validateEvalResults(uncollected, 2)) + .toThrow("appointment-desk.eval.ts ran no trials: Cannot find module"); +}); + +it("rejects malformed reports", () => { + expect(() => compareEvalResults("not json", report([trial()]))) + .toThrow("baseline results are not valid JSON"); + expect(() => compareEvalResults("{}", report([trial()]))) + .toThrow("baseline results are invalid"); +}); diff --git a/packages/workshop-evals/src/comparison.ts b/packages/workshop-evals/src/comparison.ts new file mode 100644 index 000000000..63912f367 --- /dev/null +++ b/packages/workshop-evals/src/comparison.ts @@ -0,0 +1,279 @@ +import { basename } from "node:path"; +import { z } from "zod"; +import type { JsonValue } from "vitest-evals"; + +const AssertionSchema = z.object({ + status: z.enum(["passed", "failed"]), + duration: z.number().nonnegative(), + meta: z.object({ + harness: z.object({ + run: z.object({ + session: z.object({ + metadata: z.object({ + taskId: z.string().min(1), + taskVersion: z.string().min(1), + gitCommit: z.string().min(1), + }).loose(), + }).loose(), + usage: z.object({ + model: z.string().min(1), + metadata: z.object({ + observedCumulativeChatCostUsd: z.number().nonnegative().optional(), + }).loose(), + }).loose(), + output: z.object({ + metrics: z.object({ + modelTurns: z.number().int().nonnegative(), + toolCalls: z.number().int().nonnegative(), + toolErrors: z.number().int().nonnegative(), + }), + turns: z.array(z.object({ + outcome: z.object({ status: z.string() }).loose(), + }).loose()), + }).loose(), + errors: z.array(z.object({ + name: z.string(), + message: z.string(), + }).loose()), + }).loose(), + }).loose(), + }).loose(), +}).loose(); + +// One entry per eval file. A file that fails before its first trial (a collection error) is still +// listed, with no assertions and the error in `message`. +const FileSchema = z.object({ + name: z.string(), + message: z.string().optional(), + assertionResults: z.array(AssertionSchema), +}).loose(); + +const ResultsSchema = z.object({ testResults: z.array(FileSchema) }).loose(); + +type Assertion = z.infer; +type EvalFile = z.infer; + +export type EvalStats = { + trials: number; + passed: number; + meanDurationMs: number; + meanModelTurns: number; + meanToolCalls: number; + meanToolErrors: number; + /** Null when any trial lacks a cost: a mean over a subset would not compare across sides. */ + meanCostUsd: number | null; +}; + +/** One task/model cohort. `reason` is null exactly when the two sides can be compared. */ +export type EvalComparisonRow = { taskId: string; model: string } & ( + | { reason: null; baseline: EvalStats; candidate: EvalStats } + | { reason: string; baseline: EvalStats | null; candidate: EvalStats | null } +); + +export type EvalComparison = { + baselineSha: string; + candidateSha: string; + rows: EvalComparisonRow[]; +}; + +type Cohort = { + taskId: string; + model: string; + taskVersion: string; + assertions: Assertion[]; +}; + +function parseResults(name: string, text: string): EvalFile[] { + let raw: JsonValue; + try { + raw = JSON.parse(text); + } catch (error) { + throw new Error(`${name} results are not valid JSON`, { cause: error }); + } + const parsed = ResultsSchema.safeParse(raw); + if (!parsed.success) { + throw new Error(`${name} results are invalid: ${z.prettifyError(parsed.error)}`); + } + if (trials(parsed.data.testResults).length === 0) { + throw new Error(`${name} results contain no evals`); + } + return parsed.data.testResults; +} + +function trials(files: EvalFile[]): Assertion[] { + return files.flatMap(file => file.assertionResults); +} + +function cohortKey(taskId: string, model: string): string { + return JSON.stringify([taskId, model]); +} + +function group(assertions: Assertion[]): Map { + const cohorts = new Map(); + for (const assertion of assertions) { + const run = assertion.meta.harness.run; + const { taskId, taskVersion } = run.session.metadata; + const { model } = run.usage; + const key = cohortKey(taskId, model); + const cohort = cohorts.get(key); + if (cohort === undefined) { + cohorts.set(key, { taskId, model, taskVersion, assertions: [assertion] }); + } else { + if (cohort.taskVersion !== taskVersion) { + throw new Error(`${taskId} has inconsistent task versions`); + } + cohort.assertions.push(assertion); + } + } + return cohorts; +} + +function singleCommit(name: string, assertions: Assertion[]): string { + const commits = new Set(assertions.map( + assertion => assertion.meta.harness.run.session.metadata.gitCommit)); + if (commits.size !== 1) throw new Error(`${name} results have inconsistent commits`); + const commit = commits.values().next().value; + if (commit === undefined) throw new Error(`${name} results have no commit`); + return commit; +} + +function mean(values: number[]): number { + return values.reduce((sum, value) => sum + value, 0) / values.length; +} + +function stats({ assertions }: Cohort): EvalStats { + const costs = assertions.flatMap(assertion => { + const cost = assertion.meta.harness.run.usage.metadata.observedCumulativeChatCostUsd; + return cost === undefined ? [] : [cost]; + }); + const metrics = assertions.map(assertion => assertion.meta.harness.run.output.metrics); + return { + trials: assertions.length, + passed: assertions.filter(assertion => assertion.status === "passed").length, + meanDurationMs: mean(assertions.map(assertion => assertion.duration)), + meanModelTurns: mean(metrics.map(value => value.modelTurns)), + meanToolCalls: mean(metrics.map(value => value.toolCalls)), + meanToolErrors: mean(metrics.map(value => value.toolErrors)), + meanCostUsd: costs.length === assertions.length ? mean(costs) : null, + }; +} + +function hasInfrastructureFailure(assertion: Assertion): boolean { + const run = assertion.meta.harness.run; + if (run.output.turns.some(turn => + turn.outcome.status === "error" || turn.outcome.status === "cancelled")) return true; + const names = new Set(run.errors.map(error => error.name)); + if (names.has("EvalCleanupError")) return true; + const hasAgentOutcome = names.has("AgentError") || names.has("AgentTimeout"); + return names.has("EvalRunError") && !hasAgentOutcome; +} + +/** + * Reject a report that cannot serve as a shared baseline: every eval file must have run, every + * task/model cohort must hold exactly `expectedTrials` trials, and no trial may have failed for + * infrastructure reasons. Agent failures are legitimate baseline data and pass. + */ +export function validateEvalResults(text: string, expectedTrials: number): void { + const files = parseResults("baseline", text); + for (const file of files) { + if (file.assertionResults.length === 0) { + throw new Error(`${basename(file.name)} ran no trials${file.message ? `: ${file.message}` : ""}`); + } + } + const assertions = trials(files); + singleCommit("baseline", assertions); + for (const cohort of group(assertions).values()) { + if (cohort.assertions.length !== expectedTrials) { + throw new Error( + `${cohort.taskId} on ${cohort.model} has ${cohort.assertions.length} trials, ` + + `expected ${expectedTrials}`); + } + if (cohort.assertions.some(hasInfrastructureFailure)) { + throw new Error(`${cohort.taskId} on ${cohort.model} has infrastructure failures`); + } + } +} + +/** + * Compare baseline and candidate Vitest eval reports. `definitionsChanged` is asked, with both + * reports' commits, whether the code that defines or scores a trial differs between them; when it + * does, no cohort is comparable. + */ +export function compareEvalResults( + baselineText: string, candidateText: string, + definitionsChanged: (baselineSha: string, candidateSha: string) => boolean = () => false, +): EvalComparison { + const baselineAssertions = trials(parseResults("baseline", baselineText)); + const candidateAssertions = trials(parseResults("candidate", candidateText)); + const baselineSha = singleCommit("baseline", baselineAssertions); + const candidateSha = singleCommit("candidate", candidateAssertions); + const changed = definitionsChanged(baselineSha, candidateSha); + const baseline = group(baselineAssertions); + const candidate = group(candidateAssertions); + // Either side's cohort carries the identity; both do when the key is shared. + const rows = [...new Map([...baseline, ...candidate])].map(([key, cohort]): EvalComparisonRow => { + const identity = { taskId: cohort.taskId, model: cohort.model }; + const base = baseline.get(key); + const next = candidate.get(key); + if (base === undefined) { + return { ...identity, reason: "missing baseline", baseline: null, candidate: stats(cohort) }; + } + if (next === undefined) { + return { ...identity, reason: "missing candidate", baseline: stats(base), candidate: null }; + } + const reason = changed ? "eval definition changed" + : base.taskVersion !== next.taskVersion ? "task version changed" + : base.assertions.length !== next.assertions.length ? "trial counts differ" + : base.assertions.some(hasInfrastructureFailure) ? "baseline run errors" + : next.assertions.some(hasInfrastructureFailure) ? "candidate run errors" + : null; + return { ...identity, reason, baseline: stats(base), candidate: stats(next) }; + }).toSorted((left, right) => + left.taskId.localeCompare(right.taskId) || left.model.localeCompare(right.model)); + return { baselineSha, candidateSha, rows }; +} + +function passRate(stats: EvalStats): number { + return stats.passed / stats.trials; +} + +function side(value: EvalStats | null): string { + return value === null + ? "—" + : `${value.passed}/${value.trials} (${(passRate(value) * 100).toFixed(1)}%)`; +} + +function signed(value: number, suffix: string): string { + return `${value >= 0 ? "+" : ""}${value.toFixed(1)}${suffix}`; +} + +/** Render a concise GitHub Check summary. */ +export function renderEvalComparison(comparison: EvalComparison): string { + const lines = [ + "# Workshop eval comparison", + "", + `Baseline \`${comparison.baselineSha}\` vs candidate \`${comparison.candidateSha}\`.`, + "", + "| Task | Model | Baseline | Candidate | Pass-rate delta | Duration delta | Tool-error delta | Cost delta |", + "| --- | --- | --- | --- | --- | --- | --- | --- |", + ]; + for (const row of comparison.rows) { + const cells = [row.taskId, row.model, side(row.baseline), side(row.candidate)]; + if (row.reason !== null) { + cells.push(row.reason, "—", "—", "—"); + } else { + const { baseline, candidate } = row; + const costDelta = baseline.meanCostUsd === null || candidate.meanCostUsd === null + ? "—" + : `${candidate.meanCostUsd >= baseline.meanCostUsd ? "+" : "-"}$${ + Math.abs(candidate.meanCostUsd - baseline.meanCostUsd).toFixed(4)}`; + cells.push( + signed((passRate(candidate) - passRate(baseline)) * 100, " pp"), + signed(candidate.meanDurationMs - baseline.meanDurationMs, " ms"), + signed(candidate.meanToolErrors - baseline.meanToolErrors, ""), + costDelta); + } + lines.push(`| ${cells.join(" | ")} |`); + } + return `${lines.join("\n")}\n`; +} diff --git a/scripts/evals/compare-results.ts b/scripts/evals/compare-results.ts new file mode 100644 index 000000000..e90bc411f --- /dev/null +++ b/scripts/evals/compare-results.ts @@ -0,0 +1,83 @@ +// Compare two Workshop eval result files and write the comparison JSON and Markdown: +// node scripts/evals/compare-results.ts \ +// \ +// ... +// Cohorts are non-comparable when any differs between the two reports' commits, +// since a change to the eval code moves the goalposts without touching the product under test. +// This file runs under Node's native TypeScript stripping, so imports name real .ts files and only +// erasable syntax may appear here. +import { spawnSync } from "node:child_process"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { + compareEvalResults, renderEvalComparison, +} from "../../packages/workshop-evals/src/comparison.ts"; + +const USAGE = "Usage: node scripts/evals/compare-results.ts " + + " " + + "..."; + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +async function readResults(side: string, path: string): Promise { + try { + return await readFile(path, "utf8"); + } catch (error) { + throw new Error( + `cannot read ${side} results at ${path}: ${errorMessage(error)}`, { cause: error }); + } +} + +function git(args: string[]): number { + const result = spawnSync("git", args, { stdio: ["ignore", "ignore", "inherit"] }); + if (result.error) throw result.error; + return result.status ?? 1; +} + +/** A stored baseline may predate a shallow checkout; fetch its commit on demand. */ +function ensureCommit(sha: string): void { + if (git(["cat-file", "-e", `${sha}^{commit}`]) === 0) return; + if (git(["fetch", "--no-tags", "--depth=1", "origin", sha]) !== 0) { + throw new Error(`cannot fetch baseline commit ${sha}`); + } +} + +function definitionsChanged(paths: string[]): (baselineSha: string, candidateSha: string) => boolean { + return (baselineSha, candidateSha) => { + ensureCommit(baselineSha); + const status = git(["diff", "--quiet", baselineSha, candidateSha, "--", ...paths]); + if (status === 0) return false; + if (status === 1) return true; + throw new Error(`git diff ${baselineSha} ${candidateSha} exited with ${status}`); + }; +} + +async function main(argv: string[]): Promise { + if (argv.includes("--help") || argv.includes("-h")) { + console.log(USAGE); + return; + } + if (argv.length < 5) { + throw new Error(`expected at least 5 arguments but received ${argv.length}\n${USAGE}`); + } + const [baselinePath, candidatePath, jsonPath, markdownPath, ...definitionPaths] = argv; + const report = compareEvalResults( + await readResults("baseline", baselinePath), + await readResults("candidate", candidatePath), + definitionsChanged(definitionPaths)); + const markdown = renderEvalComparison(report); + await mkdir(dirname(resolve(jsonPath)), { recursive: true }); + await mkdir(dirname(resolve(markdownPath)), { recursive: true }); + await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, "utf8"); + await writeFile(markdownPath, markdown, "utf8"); + console.log(`Compared ${report.rows.length} task/model cohorts.`); + console.log(`Wrote ${jsonPath}`); + console.log(`Wrote ${markdownPath}`); +} + +main(process.argv.slice(2)).catch((error: unknown) => { + console.error(`compare-results: ${errorMessage(error)}`); + process.exitCode = 1; +}); diff --git a/scripts/evals/validate-results.ts b/scripts/evals/validate-results.ts new file mode 100644 index 000000000..229f14e0e --- /dev/null +++ b/scripts/evals/validate-results.ts @@ -0,0 +1,40 @@ +// Reject a Workshop eval result file that cannot be stored as the main baseline: +// node scripts/evals/validate-results.ts +// This file runs under Node's native TypeScript stripping, so imports name real .ts files and only +// erasable syntax may appear here. +import { readFile } from "node:fs/promises"; +import { validateEvalResults } from "../../packages/workshop-evals/src/comparison.ts"; + +const USAGE = "Usage: node scripts/evals/validate-results.ts "; + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +async function main(argv: string[]): Promise { + if (argv.includes("--help") || argv.includes("-h")) { + console.log(USAGE); + return; + } + if (argv.length !== 2) { + throw new Error(`expected 2 arguments but received ${argv.length}\n${USAGE}`); + } + const [resultsPath, rawTrials] = argv; + const trials = Number(rawTrials); + if (!Number.isInteger(trials) || trials < 1) { + throw new Error(`trials must be a positive integer\n${USAGE}`); + } + let text: string; + try { + text = await readFile(resultsPath, "utf8"); + } catch (error) { + throw new Error(`cannot read results at ${resultsPath}: ${errorMessage(error)}`, { cause: error }); + } + validateEvalResults(text, trials); + console.log(`${resultsPath} is a complete ${trials}-trial baseline.`); +} + +main(process.argv.slice(2)).catch((error: unknown) => { + console.error(`validate-results: ${errorMessage(error)}`); + process.exitCode = 1; +});