From ab7e69f5ceb317b60d644fbff6665ef31a885192 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 24 Jul 2026 11:42:44 -0600 Subject: [PATCH 1/3] fix(optimization): run official benchmark reproducibly --- .github/workflows/empirical-gate.yml | 75 ----------- .../official-optimizer-benchmark.yml | 121 ++++++++++++++++++ .../compare-optimization-methods/index.ts | 17 ++- ...icial-optimizer-benchmark-workflow.test.ts | 45 +++++++ 4 files changed, 177 insertions(+), 81 deletions(-) delete mode 100644 .github/workflows/empirical-gate.yml create mode 100644 .github/workflows/official-optimizer-benchmark.yml create mode 100644 tests/official-optimizer-benchmark-workflow.test.ts diff --git a/.github/workflows/empirical-gate.yml b/.github/workflows/empirical-gate.yml deleted file mode 100644 index c8aab2c3..00000000 --- a/.github/workflows/empirical-gate.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: Empirical Lift (scheduled) - -# The live companion to deterministic tests in ci.yml. Unit tests check ranking -# behavior without a model. This job periodically measures the methods with a -# real model and stores the result. It does not block pull requests because live -# model results vary and cost money. -# -# Permissive: never blocks a deploy. Neutral-skips when no LLM key secret is -# provisioned (forks, pre-secret), so it is never a spurious red. - -on: - schedule: - # Weekly, Monday 06:17 UTC (off the top-of-hour to avoid the scheduler herd). - - cron: '17 6 * * 1' - workflow_dispatch: - inputs: - population: - description: 'GEPA population per generation' - default: '2' - generations: - description: 'GEPA generations' - default: '2' - epochs: - description: 'SkillOpt epochs' - default: '3' - -jobs: - empirical-lift: - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 22 - cache: pnpm - - name: Install JS deps - run: pnpm install --frozen-lockfile - - # Skip cleanly when no model key is configured. - - name: Check for an LLM key - id: guard - env: - KEY: ${{ secrets.LLM_API_KEY != '' && secrets.LLM_API_KEY || secrets.TANGLE_API_KEY }} - run: | - if [ -n "$KEY" ]; then - echo "run=true" >> "$GITHUB_OUTPUT" - else - echo "::notice::No LLM_API_KEY / TANGLE_API_KEY secret set. Skipping the live method comparison; deterministic tests still run in ci.yml." - echo "run=false" >> "$GITHUB_OUTPUT" - fi - - - name: Compare GEPA reflection, GEPA Pareto, and SkillOpt - if: steps.guard.outputs.run == 'true' - env: - # Defaults to the Tangle router; override the base/model via repo vars - # to benchmark on any OpenAI-compatible endpoint. - LLM_API_KEY: ${{ secrets.LLM_API_KEY != '' && secrets.LLM_API_KEY || secrets.TANGLE_API_KEY }} - LLM_BASE_URL: ${{ vars.LLM_BASE_URL }} - LLM_MODEL: ${{ vars.LLM_MODEL }} - PRICE_IN_PER_M: ${{ vars.PRICE_IN_PER_M }} - PRICE_OUT_PER_M: ${{ vars.PRICE_OUT_PER_M }} - POPULATION: ${{ github.event.inputs.population || '2' }} - GENERATIONS: ${{ github.event.inputs.generations || '2' }} - EPOCHS: ${{ github.event.inputs.epochs || '3' }} - run: pnpm tsx examples/compare-optimization-methods/index.ts - - - name: Upload method comparison artifact - if: steps.guard.outputs.run == 'true' - uses: actions/upload-artifact@v4 - with: - name: empirical-lift-${{ github.run_number }} - path: .evolve/compare-optimization-methods/**/comparison.json - if-no-files-found: warn diff --git a/.github/workflows/official-optimizer-benchmark.yml b/.github/workflows/official-optimizer-benchmark.yml new file mode 100644 index 00000000..fff7b177 --- /dev/null +++ b/.github/workflows/official-optimizer-benchmark.yml @@ -0,0 +1,121 @@ +name: Official Optimizer Benchmark + +# This scheduled run compares the pinned upstream GEPA and SkillOpt projects +# against the same task, data split, model, and candidate-case evaluation limit. +# It records the full result without blocking pull requests on model variance. + +on: + schedule: + - cron: '17 6 * * 1' + workflow_dispatch: + inputs: + optimizers: + description: 'Comma-separated methods: gepa,skillopt' + default: 'gepa,skillopt' + type: string + model: + description: 'Exact worker and optimizer model; falls back to the LLM_MODEL repository variable' + required: false + type: string + base_url: + description: 'OpenAI-compatible endpoint; falls back to LLM_BASE_URL or TANGLE_ROUTER_URL' + required: false + type: string + price_in_per_m: + description: 'Exact input-token USD per million; falls back to PRICE_IN_PER_M' + required: false + type: string + price_out_per_m: + description: 'Exact output-token USD per million; falls back to PRICE_OUT_PER_M' + required: false + type: string + skillopt_epochs: + description: 'SkillOpt epochs' + default: '2' + type: string + skillopt_batch_size: + description: 'SkillOpt train cases per batch' + default: '2' + type: string + max_optimizer_model_cost_usd: + description: 'Equal optimizer-model spend limit per method' + default: '5' + type: string + max_total_cost_usd: + description: 'Stop the complete comparison before this total spend' + default: '20' + type: string + +jobs: + benchmark: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: clients/python/uv.lock + - name: Install JS deps + run: pnpm install --frozen-lockfile + - name: Install pinned official optimizers + working-directory: clients/python + run: uv sync --frozen --group skillopt-source --group gepa-source + + - name: Check benchmark configuration + id: config + env: + KEY: ${{ secrets.LLM_API_KEY != '' && secrets.LLM_API_KEY || secrets.TANGLE_API_KEY }} + OPENAI_KEY: ${{ secrets.LLM_API_KEY }} + BASE_URL: ${{ github.event.inputs.base_url != '' && github.event.inputs.base_url || vars.LLM_BASE_URL != '' && vars.LLM_BASE_URL || vars.TANGLE_ROUTER_URL }} + MODEL: ${{ github.event.inputs.model != '' && github.event.inputs.model || vars.LLM_MODEL }} + PRICE_IN: ${{ github.event.inputs.price_in_per_m != '' && github.event.inputs.price_in_per_m || vars.PRICE_IN_PER_M }} + PRICE_OUT: ${{ github.event.inputs.price_out_per_m != '' && github.event.inputs.price_out_per_m || vars.PRICE_OUT_PER_M }} + run: | + if [ -n "$KEY" ]; then + if [ -z "$OPENAI_KEY" ] && [ -z "$BASE_URL" ]; then + echo "::error::TANGLE_API_KEY requires LLM_BASE_URL or TANGLE_ROUTER_URL." + exit 1 + fi + if [ -z "$MODEL" ] || [ -z "$PRICE_IN" ] || [ -z "$PRICE_OUT" ]; then + echo "::error::Set the exact model and input/output token prices as workflow inputs or repository variables." + exit 1 + fi + echo "run=true" >> "$GITHUB_OUTPUT" + else + echo "::notice::No LLM_API_KEY or TANGLE_API_KEY secret is available. Skipping the paid benchmark." + echo "run=false" >> "$GITHUB_OUTPUT" + fi + + - name: Compare official GEPA and SkillOpt + if: steps.config.outputs.run == 'true' + env: + LLM_API_KEY: ${{ secrets.LLM_API_KEY != '' && secrets.LLM_API_KEY || secrets.TANGLE_API_KEY }} + LLM_BASE_URL: ${{ github.event.inputs.base_url != '' && github.event.inputs.base_url || vars.LLM_BASE_URL != '' && vars.LLM_BASE_URL || vars.TANGLE_ROUTER_URL }} + LLM_MODEL: ${{ github.event.inputs.model != '' && github.event.inputs.model || vars.LLM_MODEL }} + PRICE_IN_PER_M: ${{ github.event.inputs.price_in_per_m != '' && github.event.inputs.price_in_per_m || vars.PRICE_IN_PER_M }} + PRICE_OUT_PER_M: ${{ github.event.inputs.price_out_per_m != '' && github.event.inputs.price_out_per_m || vars.PRICE_OUT_PER_M }} + PRICE_SOURCE: 'Workflow inputs or repository variables recorded with the run.' + OPTIMIZERS: ${{ github.event.inputs.optimizers || 'gepa,skillopt' }} + SKILLOPT_EPOCHS: ${{ github.event.inputs.skillopt_epochs || '2' }} + SKILLOPT_BATCH_SIZE: ${{ github.event.inputs.skillopt_batch_size || '2' }} + MAX_OPTIMIZER_MODEL_COST_USD: ${{ github.event.inputs.max_optimizer_model_cost_usd || '5' }} + MAX_TOTAL_COST_USD: ${{ github.event.inputs.max_total_cost_usd || '20' }} + OPTIMIZER_PYTHON: ${{ github.workspace }}/clients/python/.venv/bin/python + run: pnpm tsx examples/compare-optimization-methods/index.ts + + - name: Upload method comparison artifact + if: always() && steps.config.outputs.run == 'true' + uses: actions/upload-artifact@v4 + with: + name: official-optimizer-benchmark-${{ github.run_number }} + path: .evolve/compare-optimization-methods/**/comparison.json + if-no-files-found: error diff --git a/examples/compare-optimization-methods/index.ts b/examples/compare-optimization-methods/index.ts index 16b39d64..627c4456 100644 --- a/examples/compare-optimization-methods/index.ts +++ b/examples/compare-optimization-methods/index.ts @@ -52,11 +52,15 @@ const PRICE_CACHED_IN_PER_M = optionalNonNegativeNumberEnv('PRICE_CACHED_IN_PER_ const PRICE_CACHE_WRITE_IN_PER_M = optionalNonNegativeNumberEnv('PRICE_CACHE_WRITE_IN_PER_M') const PRICE_OUT_PER_M = optionalNonNegativeNumberEnv('PRICE_OUT_PER_M') const CALL_TIMEOUT_MS = positiveIntegerEnv('CALL_TIMEOUT_MS', 30_000) -const GEPA_MAX_PROPOSER_COST_USD = positiveNumberEnv('GEPA_MAX_PROPOSER_COST_USD', 5) +const MAX_OPTIMIZER_MODEL_COST_USD = positiveNumberEnv('MAX_OPTIMIZER_MODEL_COST_USD', 5) +const MAX_TOTAL_COST_USD = positiveNumberEnv('MAX_TOTAL_COST_USD', 20) +const GEPA_MAX_PROPOSER_COST_USD = positiveNumberEnv( + 'GEPA_MAX_PROPOSER_COST_USD', + MAX_OPTIMIZER_MODEL_COST_USD, +) const SKILLOPT_EPOCHS = positiveIntegerEnv('SKILLOPT_EPOCHS', 2) const SKILLOPT_BATCH_SIZE = positiveIntegerEnv('SKILLOPT_BATCH_SIZE', 2) const OPTIMIZATION_CONCURRENCY = positiveIntegerEnv('OPTIMIZATION_CONCURRENCY', 1) -const MAX_RUN_COST_USD = positiveNumberEnv('MAX_RUN_COST_USD', 12) const SELECTION_N = 3 const TRAIN = SEARCH.slice(0, -SELECTION_N) const SELECTION = SEARCH.slice(-SELECTION_N) @@ -134,10 +138,10 @@ const PRICE_SOURCE = ? 'Caller-supplied PRICE_* environment variables.' : 'Provider usage cost or Agent Eval package model pricing.') const gepaModelBudget = selectedNames.includes('gepa') - ? optimizerModelBudgetFromEnv('GEPA', MAX_RUN_COST_USD, customTokenPricing) + ? optimizerModelBudgetFromEnv('GEPA', MAX_OPTIMIZER_MODEL_COST_USD, customTokenPricing) : undefined const skillOptModelBudget = selectedNames.includes('skillopt') - ? optimizerModelBudgetFromEnv('SKILLOPT', MAX_RUN_COST_USD, customTokenPricing) + ? optimizerModelBudgetFromEnv('SKILLOPT', MAX_OPTIMIZER_MODEL_COST_USD, customTokenPricing) : undefined const llm: LlmClientOptions = { @@ -264,7 +268,7 @@ async function main() { dispatchTimeoutMs: CALL_TIMEOUT_MS, expectUsage: 'assert', }, - costCeiling: MAX_RUN_COST_USD, + costCeiling: MAX_TOTAL_COST_USD, dispatchTimeoutMs: CALL_TIMEOUT_MS, expectUsage: 'assert', }) @@ -324,7 +328,8 @@ async function main() { gepa: selectedNames.includes('gepa') ? GEPA_MAX_EVALUATIONS : null, skillopt: selectedNames.includes('skillopt') ? SKILLOPT_MAX_EVALUATIONS : null, }, - allRunCostUsd: MAX_RUN_COST_USD, + optimizerModelCostUsdPerMethod: MAX_OPTIMIZER_MODEL_COST_USD, + allRunCostUsd: MAX_TOTAL_COST_USD, optimizerModels: { gepa: gepaModelBudget ?? null, skillopt: skillOptModelBudget ?? null, diff --git a/tests/official-optimizer-benchmark-workflow.test.ts b/tests/official-optimizer-benchmark-workflow.test.ts new file mode 100644 index 00000000..5f95cdc7 --- /dev/null +++ b/tests/official-optimizer-benchmark-workflow.test.ts @@ -0,0 +1,45 @@ +import { readFileSync } from 'node:fs' +import { resolve } from 'node:path' +import { describe, expect, it } from 'vitest' + +const workflow = readFileSync( + resolve(process.cwd(), '.github/workflows/official-optimizer-benchmark.yml'), + 'utf8', +) +const benchmark = readFileSync( + resolve(process.cwd(), 'examples/compare-optimization-methods/index.ts'), + 'utf8', +) + +describe('official optimizer benchmark workflow', () => { + it('installs and runs both pinned upstream optimizers', () => { + expect(workflow).toContain('uv sync --frozen --group skillopt-source --group gepa-source') + expect(workflow).toContain('github.event.inputs.optimizers') + expect(workflow).toContain("'gepa,skillopt'") + expect(workflow).toContain('github.workspace') + expect(workflow).toContain('/clients/python/.venv/bin/python') + expect(workflow).toContain('pnpm tsx examples/compare-optimization-methods/index.ts') + }) + + it('uses the controls read by the current benchmark', () => { + for (const name of [ + 'SKILLOPT_EPOCHS', + 'SKILLOPT_BATCH_SIZE', + 'MAX_OPTIMIZER_MODEL_COST_USD', + 'MAX_TOTAL_COST_USD', + ]) { + expect(workflow).toContain(`${name}:`) + expect(benchmark).toContain(`'${name}'`) + } + for (const staleName of ['POPULATION', 'GENERATIONS', 'EPOCHS']) { + expect(workflow).not.toMatch(new RegExp(`^\\s+${staleName}:`, 'm')) + } + }) + + it('requires reproducible model and price metadata before paid work', () => { + expect(workflow).toContain('[ -z "$MODEL" ]') + expect(workflow).toContain('[ -z "$PRICE_IN" ]') + expect(workflow).toContain('[ -z "$PRICE_OUT" ]') + expect(workflow).toContain('if-no-files-found: error') + }) +}) From 7c288087665e90a4d8905b5b9edfbb41f20db29a Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 24 Jul 2026 11:51:08 -0600 Subject: [PATCH 2/3] fix(benchmarks): align official method budgets --- examples/benchmarks/appworld/README.md | 7 +-- examples/benchmarks/appworld/run-bench.ts | 17 ++++--- .../gsm8k/compare-optimization-methods.ts | 17 ++++--- .../compare-optimization-methods/README.md | 7 +-- package.json | 3 +- pnpm-lock.yaml | 15 ++---- ...icial-optimizer-benchmark-workflow.test.ts | 51 ++++++++++++++----- 7 files changed, 76 insertions(+), 41 deletions(-) diff --git a/examples/benchmarks/appworld/README.md b/examples/benchmarks/appworld/README.md index ea5f4818..323e0ec9 100644 --- a/examples/benchmarks/appworld/README.md +++ b/examples/benchmarks/appworld/README.md @@ -53,14 +53,14 @@ Common settings: | `GEPA_MAX_PROPOSER_COST_USD` | `5` | GEPA model spend limit for one engine stage | | `GEPA_PRICE_IN_PER_M` | required | Exact GEPA input rate per million tokens | | `GEPA_PRICE_OUT_PER_M` | required | Exact GEPA output rate per million tokens | -| `GEPA_MAX_MODEL_COST_USD` | `MAX_RUN_COST_USD` | Shared GEPA model spend limit | +| `GEPA_MAX_MODEL_COST_USD` | `MAX_OPTIMIZER_MODEL_COST_USD` | GEPA model spend limit | | `SKILLOPT_MODEL` | `BENCH_MODEL` | Model used by SkillOpt | | `SKILLOPT_EPOCHS` | `1` | SkillOpt training epochs | | `SKILLOPT_BATCH_SIZE` | `2` | SkillOpt train tasks per step | | `SKILLOPT_MAX_EVALUATIONS` | core plan size | Maximum SkillOpt candidate-task calls | | `SKILLOPT_PRICE_IN_PER_M` | required | Exact optimizer-model input rate per million tokens | | `SKILLOPT_PRICE_OUT_PER_M` | required | Exact optimizer-model output rate per million tokens | -| `SKILLOPT_MAX_MODEL_COST_USD` | `MAX_RUN_COST_USD` | SkillOpt optimizer-model spend limit | +| `SKILLOPT_MAX_MODEL_COST_USD` | `MAX_OPTIMIZER_MODEL_COST_USD` | SkillOpt optimizer-model spend limit | | `TRAIN_N` | `3` | Train tasks per method | | `SELECTION_N` | `3` | Tasks used to select one prompt per method | | `TEST_N` | `5` | Final comparison tasks | @@ -72,7 +72,8 @@ Common settings: | `CALL_TIMEOUT` | `120` | Seconds per model request | | `MAX_TOKENS` | `6000` | Maximum output tokens per model request | | `RATE_LIMIT_BUDGET` | `240` | Seconds spent retrying rate limits per request | -| `MAX_RUN_COST_USD` | `125` | Shared limit for all optimization and final-task spend | +| `MAX_OPTIMIZER_MODEL_COST_USD` | `5` | Equal optimizer-model spend limit per method | +| `MAX_TOTAL_COST_USD` | `125` | Shared limit for all optimization and final-task spend | | `OUT_DIR` | `/tmp/appworld-bench` | Output directory | The defaults are sized to check the workflow. diff --git a/examples/benchmarks/appworld/run-bench.ts b/examples/benchmarks/appworld/run-bench.ts index a72e63fe..e295e21d 100644 --- a/examples/benchmarks/appworld/run-bench.ts +++ b/examples/benchmarks/appworld/run-bench.ts @@ -55,7 +55,12 @@ const API_KEY = process.env.OPENAI_API_KEY?.trim() ?? '' const TRAIN_N = positiveIntegerEnv('TRAIN_N', 3) const SELECTION_N = positiveIntegerEnv('SELECTION_N', 3) const TEST_N = positiveIntegerEnv('TEST_N', 5) -const GEPA_MAX_PROPOSER_COST_USD = positiveNumberEnv('GEPA_MAX_PROPOSER_COST_USD', 5) +const MAX_OPTIMIZER_MODEL_COST_USD = positiveNumberEnv('MAX_OPTIMIZER_MODEL_COST_USD', 5) +const MAX_TOTAL_COST_USD = positiveNumberEnv('MAX_TOTAL_COST_USD', 125) +const GEPA_MAX_PROPOSER_COST_USD = positiveNumberEnv( + 'GEPA_MAX_PROPOSER_COST_USD', + MAX_OPTIMIZER_MODEL_COST_USD, +) const SKILLOPT_EPOCHS = positiveIntegerEnv('SKILLOPT_EPOCHS', 1) const SKILLOPT_BATCH_SIZE = positiveIntegerEnv('SKILLOPT_BATCH_SIZE', 2) const SKILLOPT_CORE_EVALUATIONS = @@ -75,7 +80,6 @@ const OPTIMIZATION_CONCURRENCY = positiveIntegerEnv('OPTIMIZATION_CONCURRENCY', const CALL_TIMEOUT = positiveNumberEnv('CALL_TIMEOUT', 120) const MAX_TOKENS = positiveIntegerEnv('MAX_TOKENS', 6000) const RATE_LIMIT_BUDGET = positiveNumberEnv('RATE_LIMIT_BUDGET', 240) -const MAX_RUN_COST_USD = positiveNumberEnv('MAX_RUN_COST_USD', 125) const OUT_DIR = stringEnv('OUT_DIR', join(tmpdir(), 'appworld-bench')) const SEED = safeIntegerEnv('SEED', 42) if (TEMPERATURE > 2) throw new Error('TEMPERATURE must be between 0 and 2') @@ -365,10 +369,10 @@ async function main(): Promise { ) const budgets = { ...(selected.has('gepa') - ? { gepa: optimizerModelBudgetFromEnv('GEPA', MAX_RUN_COST_USD) } + ? { gepa: optimizerModelBudgetFromEnv('GEPA', MAX_OPTIMIZER_MODEL_COST_USD) } : {}), ...(selected.has('skillopt') - ? { skillopt: optimizerModelBudgetFromEnv('SKILLOPT', MAX_RUN_COST_USD) } + ? { skillopt: optimizerModelBudgetFromEnv('SKILLOPT', MAX_OPTIMIZER_MODEL_COST_USD) } : {}), } const methods = officialMethods(selected, budgets) @@ -397,7 +401,7 @@ async function main(): Promise { maxConcurrency: MAXCONC, expectUsage: 'assert', }, - costCeiling: MAX_RUN_COST_USD, + costCeiling: MAX_TOTAL_COST_USD, expectUsage: 'assert', }) @@ -426,7 +430,8 @@ async function main(): Promise { skillopt: selected.has('skillopt') ? SKILLOPT_MAX_EVALUATIONS : null, }, gepaMaxProposerCostUsd: selected.has('gepa') ? GEPA_MAX_PROPOSER_COST_USD : null, - allRunCostUsd: MAX_RUN_COST_USD, + optimizerModelCostUsdPerMethod: MAX_OPTIMIZER_MODEL_COST_USD, + allRunCostUsd: MAX_TOTAL_COST_USD, repetitionsPerFinalTask: REPS, optimizationConcurrency: OPTIMIZATION_CONCURRENCY, taskConcurrency: MAXCONC, diff --git a/examples/benchmarks/gsm8k/compare-optimization-methods.ts b/examples/benchmarks/gsm8k/compare-optimization-methods.ts index 85b925dd..cdc3600c 100644 --- a/examples/benchmarks/gsm8k/compare-optimization-methods.ts +++ b/examples/benchmarks/gsm8k/compare-optimization-methods.ts @@ -64,7 +64,12 @@ const OPTIMIZER_PYTHON = process.env.OPTIMIZER_PYTHON?.trim() || 'python' const OPTIMIZER_API_KEY = (process.env.OPTIMIZER_API_KEY || API_KEY)?.trim() const OPTIMIZER_BASE_URL = (process.env.OPTIMIZER_BASE_URL || BASE_URL).trim() const GEPA_MODEL = process.env.GEPA_MODEL || MODEL -const GEPA_MAX_PROPOSER_COST_USD = positiveNumberEnv('GEPA_MAX_PROPOSER_COST_USD', 10) +const MAX_OPTIMIZER_MODEL_COST_USD = positiveNumberEnv('MAX_OPTIMIZER_MODEL_COST_USD', 10) +const MAX_TOTAL_COST_USD = positiveNumberEnv('MAX_TOTAL_COST_USD', 25) +const GEPA_MAX_PROPOSER_COST_USD = positiveNumberEnv( + 'GEPA_MAX_PROPOSER_COST_USD', + MAX_OPTIMIZER_MODEL_COST_USD, +) const SKILLOPT_MODEL = process.env.SKILLOPT_MODEL || MODEL const SKILLOPT_EPOCHS = positiveIntegerEnv('SKILLOPT_EPOCHS', 2) const SKILLOPT_BATCH_SIZE = positiveIntegerEnv('SKILLOPT_BATCH_SIZE', 4) @@ -80,7 +85,6 @@ const OPTIMIZATION_CONCURRENCY = positiveIntegerEnv('OPTIMIZATION_CONCURRENCY', const TASK_CONCURRENCY = positiveIntegerEnv('TASK_CONCURRENCY', 2) const REPS = positiveIntegerEnv('REPS', 1) const MAX_SMOKE_COST_USD = positiveNumberEnv('MAX_SMOKE_COST_USD', 2) -const MAX_RUN_COST_USD = positiveNumberEnv('MAX_RUN_COST_USD', 25) const SMOKE = process.env.SMOKE === '1' const SEED = 42 const RESAMPLES = 4_000 @@ -284,12 +288,12 @@ async function main() { const gepaModelBudget = optimizerModelBudgetFromEnv( 'GEPA', - MAX_RUN_COST_USD, + MAX_OPTIMIZER_MODEL_COST_USD, CUSTOM_TOKEN_PRICING, ) const skillOptModelBudget = optimizerModelBudgetFromEnv( 'SKILLOPT', - MAX_RUN_COST_USD, + MAX_OPTIMIZER_MODEL_COST_USD, CUSTOM_TOKEN_PRICING, ) const runner = { @@ -368,7 +372,7 @@ async function main() { maxConcurrency: TASK_CONCURRENCY, expectUsage: 'assert', }, - costCeiling: MAX_RUN_COST_USD, + costCeiling: MAX_TOTAL_COST_USD, dispatchTimeoutMs: CALL_TIMEOUT_MS, expectUsage: 'assert', }) @@ -416,7 +420,8 @@ async function main() { coreEvaluations: SKILLOPT_CORE_EVALUATIONS, }, smokeCostUsd: MAX_SMOKE_COST_USD, - allRunCostUsd: MAX_RUN_COST_USD, + optimizerModelCostUsdPerMethod: MAX_OPTIMIZER_MODEL_COST_USD, + allRunCostUsd: MAX_TOTAL_COST_USD, worker: { requestTimeoutMs: CALL_TIMEOUT_MS, maxOutputTokens: WORKER_MAX_TOKENS, diff --git a/examples/compare-optimization-methods/README.md b/examples/compare-optimization-methods/README.md index 2419e667..867abdfe 100644 --- a/examples/compare-optimization-methods/README.md +++ b/examples/compare-optimization-methods/README.md @@ -94,7 +94,7 @@ Replace all four example rates with the exact rates charged by that endpoint. | `GEPA_MAX_PROPOSER_COST_USD` | `5` | Maximum GEPA model spend inside one engine stage. | | `GEPA_PRICE_IN_PER_M` | required | Exact GEPA input rate per million tokens. | | `GEPA_PRICE_OUT_PER_M` | required | Exact GEPA output rate per million tokens. | -| `GEPA_MAX_MODEL_COST_USD` | `MAX_RUN_COST_USD` | Shared GEPA model spend limit. | +| `GEPA_MAX_MODEL_COST_USD` | `MAX_OPTIMIZER_MODEL_COST_USD` | GEPA model spend limit. | | `GEPA_MAX_MODEL_REQUESTS` | `100` | Shared GEPA model request limit. | | `SKILLOPT_MODEL` | `LLM_MODEL` | Model used by SkillOpt reflection and editing. | | `SKILLOPT_EPOCHS` | `2` | SkillOpt training epochs. | @@ -102,9 +102,10 @@ Replace all four example rates with the exact rates charged by that endpoint. | `SKILLOPT_MAX_EVALUATIONS` | core plan size | Maximum SkillOpt candidate-case calls. | | `SKILLOPT_PRICE_IN_PER_M` | required | Exact optimizer-model input rate per million tokens. | | `SKILLOPT_PRICE_OUT_PER_M` | required | Exact optimizer-model output rate per million tokens. | -| `SKILLOPT_MAX_MODEL_COST_USD` | `MAX_RUN_COST_USD` | SkillOpt optimizer-model spend limit. | +| `SKILLOPT_MAX_MODEL_COST_USD` | `MAX_OPTIMIZER_MODEL_COST_USD` | SkillOpt optimizer-model spend limit. | | `SKILLOPT_MAX_MODEL_REQUESTS` | `100` | SkillOpt optimizer-model request limit. | -| `MAX_RUN_COST_USD` | `12` | Shared limit for all optimization and final-case spend. | +| `MAX_OPTIMIZER_MODEL_COST_USD` | `5` | Equal optimizer-model spend limit per method. | +| `MAX_TOTAL_COST_USD` | `20` | Shared limit for all optimization and final-case spend. | | `OPTIMIZATION_CONCURRENCY` | `1` | Methods allowed to optimize concurrently. | | `BILLING_NOTE` | inferred | Billing context saved with the result. | | `PRICE_SOURCE` | inferred | Source of the token prices saved with the result. | diff --git a/package.json b/package.json index 18599ae7..0d7ce8d9 100644 --- a/package.json +++ b/package.json @@ -177,7 +177,8 @@ "openapi3-ts": "^4.5.0", "tsup": "^8.0.0", "typescript": "^5.7.0", - "vitest": "^3.0.0" + "vitest": "^3.0.0", + "yaml": "2.9.0" }, "pnpm": { "minimumReleaseAge": 4320, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e2123f3c..81028ce8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -61,6 +61,9 @@ importers: vitest: specifier: ^3.0.0 version: 3.2.4(@types/node@25.6.0)(yaml@2.9.0) + yaml: + specifier: 2.9.0 + version: 2.9.0 packages: @@ -1087,11 +1090,6 @@ packages: utf-8-validate: optional: true - yaml@2.8.3: - resolution: {integrity: sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==} - engines: {node: '>= 14.6'} - hasBin: true - yaml@2.9.0: resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==} engines: {node: '>= 14.6'} @@ -1674,7 +1672,7 @@ snapshots: openapi3-ts@4.5.0: dependencies: - yaml: 2.8.3 + yaml: 2.9.0 ox@0.14.20(typescript@5.9.3)(zod@4.3.6): dependencies: @@ -1989,10 +1987,7 @@ snapshots: ws@8.21.0: {} - yaml@2.8.3: {} - - yaml@2.9.0: - optional: true + yaml@2.9.0: {} zod@4.3.6: {} diff --git a/tests/official-optimizer-benchmark-workflow.test.ts b/tests/official-optimizer-benchmark-workflow.test.ts index 5f95cdc7..d40b212c 100644 --- a/tests/official-optimizer-benchmark-workflow.test.ts +++ b/tests/official-optimizer-benchmark-workflow.test.ts @@ -1,6 +1,7 @@ import { readFileSync } from 'node:fs' import { resolve } from 'node:path' import { describe, expect, it } from 'vitest' +import { parse } from 'yaml' const workflow = readFileSync( resolve(process.cwd(), '.github/workflows/official-optimizer-benchmark.yml'), @@ -10,36 +11,62 @@ const benchmark = readFileSync( resolve(process.cwd(), 'examples/compare-optimization-methods/index.ts'), 'utf8', ) +type WorkflowStep = { + name?: string + uses?: string + run?: string + if?: string + env?: Record + with?: Record +} +const document = parse(workflow) as { + on: { workflow_dispatch: { inputs: Record } } + jobs: { benchmark: { steps: WorkflowStep[] } } +} +const steps = document.jobs.benchmark.steps +const stepNamed = (name: string) => { + const step = steps.find((candidate) => candidate.name === name) + if (!step) throw new Error(`Missing workflow step: ${name}`) + return step +} describe('official optimizer benchmark workflow', () => { it('installs and runs both pinned upstream optimizers', () => { - expect(workflow).toContain('uv sync --frozen --group skillopt-source --group gepa-source') - expect(workflow).toContain('github.event.inputs.optimizers') - expect(workflow).toContain("'gepa,skillopt'") - expect(workflow).toContain('github.workspace') - expect(workflow).toContain('/clients/python/.venv/bin/python') - expect(workflow).toContain('pnpm tsx examples/compare-optimization-methods/index.ts') + expect(stepNamed('Install pinned official optimizers').run).toBe( + 'uv sync --frozen --group skillopt-source --group gepa-source', + ) + const compare = stepNamed('Compare official GEPA and SkillOpt') + expect(compare.run).toBe('pnpm tsx examples/compare-optimization-methods/index.ts') + expect(compare.env?.OPTIMIZERS).toContain('gepa,skillopt') + expect(compare.env?.OPTIMIZER_PYTHON).toContain('/clients/python/.venv/bin/python') }) it('uses the controls read by the current benchmark', () => { + const compareEnv = stepNamed('Compare official GEPA and SkillOpt').env ?? {} for (const name of [ 'SKILLOPT_EPOCHS', 'SKILLOPT_BATCH_SIZE', 'MAX_OPTIMIZER_MODEL_COST_USD', 'MAX_TOTAL_COST_USD', ]) { - expect(workflow).toContain(`${name}:`) + expect(compareEnv).toHaveProperty(name) expect(benchmark).toContain(`'${name}'`) } for (const staleName of ['POPULATION', 'GENERATIONS', 'EPOCHS']) { - expect(workflow).not.toMatch(new RegExp(`^\\s+${staleName}:`, 'm')) + expect(compareEnv).not.toHaveProperty(staleName) } }) it('requires reproducible model and price metadata before paid work', () => { - expect(workflow).toContain('[ -z "$MODEL" ]') - expect(workflow).toContain('[ -z "$PRICE_IN" ]') - expect(workflow).toContain('[ -z "$PRICE_OUT" ]') - expect(workflow).toContain('if-no-files-found: error') + const config = stepNamed('Check benchmark configuration') + expect(config.run).toContain('[ -z "$MODEL" ]') + expect(config.run).toContain('[ -z "$PRICE_IN" ]') + expect(config.run).toContain('[ -z "$PRICE_OUT" ]') + const upload = stepNamed('Upload method comparison artifact') + expect(upload.if).toBe("always() && steps.config.outputs.run == 'true'") + expect(upload.with?.['if-no-files-found']).toBe('error') + expect(Object.keys(document.on.workflow_dispatch.inputs)).toEqual( + expect.arrayContaining(['model', 'price_in_per_m', 'price_out_per_m']), + ) }) }) From c714bf63c9ef97b9c990a9b20228a2eb74240e6b Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 24 Jul 2026 12:03:35 -0600 Subject: [PATCH 3/3] fix(benchmarks): serialize paid optimizer runs --- .github/workflows/official-optimizer-benchmark.yml | 7 +++++++ tests/official-optimizer-benchmark-workflow.test.ts | 10 ++++++++++ 2 files changed, 17 insertions(+) diff --git a/.github/workflows/official-optimizer-benchmark.yml b/.github/workflows/official-optimizer-benchmark.yml index fff7b177..c2533ac5 100644 --- a/.github/workflows/official-optimizer-benchmark.yml +++ b/.github/workflows/official-optimizer-benchmark.yml @@ -46,6 +46,13 @@ on: default: '20' type: string +permissions: + contents: read + +concurrency: + group: official-optimizer-benchmark + cancel-in-progress: false + jobs: benchmark: runs-on: ubuntu-latest diff --git a/tests/official-optimizer-benchmark-workflow.test.ts b/tests/official-optimizer-benchmark-workflow.test.ts index d40b212c..893a5c9b 100644 --- a/tests/official-optimizer-benchmark-workflow.test.ts +++ b/tests/official-optimizer-benchmark-workflow.test.ts @@ -21,6 +21,8 @@ type WorkflowStep = { } const document = parse(workflow) as { on: { workflow_dispatch: { inputs: Record } } + permissions: { contents: string } + concurrency: { group: string; 'cancel-in-progress': boolean } jobs: { benchmark: { steps: WorkflowStep[] } } } const steps = document.jobs.benchmark.steps @@ -69,4 +71,12 @@ describe('official optimizer benchmark workflow', () => { expect.arrayContaining(['model', 'price_in_per_m', 'price_out_per_m']), ) }) + + it('serializes paid runs with read-only repository access', () => { + expect(document.permissions).toEqual({ contents: 'read' }) + expect(document.concurrency).toEqual({ + group: 'official-optimizer-benchmark', + 'cancel-in-progress': false, + }) + }) })