fix(optimization): run official benchmark reproducibly

.github/workflows/official-optimizer-benchmark.yml

@@ -0,0 +1,128 @@
+name: Official Optimizer Benchmark
+
+# This scheduled run compares the pinned upstream GEPA and SkillOpt projects
+# against the same task, data split, model, and candidate-case evaluation limit.
+# It records the full result without blocking pull requests on model variance.
+
+on:
+  schedule:
+    - cron: '17 6 * * 1'
+  workflow_dispatch:
+    inputs:
+      optimizers:
+        description: 'Comma-separated methods: gepa,skillopt'
+        default: 'gepa,skillopt'
+        type: string
+      model:
+        description: 'Exact worker and optimizer model; falls back to the LLM_MODEL repository variable'
+        required: false
+        type: string
+      base_url:
+        description: 'OpenAI-compatible endpoint; falls back to LLM_BASE_URL or TANGLE_ROUTER_URL'
+        required: false
+        type: string
+      price_in_per_m:
+        description: 'Exact input-token USD per million; falls back to PRICE_IN_PER_M'
+        required: false
+        type: string
+      price_out_per_m:
+        description: 'Exact output-token USD per million; falls back to PRICE_OUT_PER_M'
+        required: false
+        type: string
+      skillopt_epochs:
+        description: 'SkillOpt epochs'
+        default: '2'
+        type: string
+      skillopt_batch_size:
+        description: 'SkillOpt train cases per batch'
+        default: '2'
+        type: string
+      max_optimizer_model_cost_usd:
+        description: 'Equal optimizer-model spend limit per method'
+        default: '5'
+        type: string
+      max_total_cost_usd:
+        description: 'Stop the complete comparison before this total spend'
+        default: '20'
+        type: string
+
+permissions:
+  contents: read
+
+concurrency:
+  group: official-optimizer-benchmark
+  cancel-in-progress: false
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: pnpm
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+          cache-dependency-glob: clients/python/uv.lock
+      - name: Install JS deps
+        run: pnpm install --frozen-lockfile
+      - name: Install pinned official optimizers
+        working-directory: clients/python
+        run: uv sync --frozen --group skillopt-source --group gepa-source
+
+      - name: Check benchmark configuration
+        id: config
+        env:
+          KEY: ${{ secrets.LLM_API_KEY != '' && secrets.LLM_API_KEY || secrets.TANGLE_API_KEY }}
+          OPENAI_KEY: ${{ secrets.LLM_API_KEY }}
+          BASE_URL: ${{ github.event.inputs.base_url != '' && github.event.inputs.base_url || vars.LLM_BASE_URL != '' && vars.LLM_BASE_URL || vars.TANGLE_ROUTER_URL }}
+          MODEL: ${{ github.event.inputs.model != '' && github.event.inputs.model || vars.LLM_MODEL }}
+          PRICE_IN: ${{ github.event.inputs.price_in_per_m != '' && github.event.inputs.price_in_per_m || vars.PRICE_IN_PER_M }}
+          PRICE_OUT: ${{ github.event.inputs.price_out_per_m != '' && github.event.inputs.price_out_per_m || vars.PRICE_OUT_PER_M }}
+        run: |
+          if [ -n "$KEY" ]; then
+            if [ -z "$OPENAI_KEY" ] && [ -z "$BASE_URL" ]; then
+              echo "::error::TANGLE_API_KEY requires LLM_BASE_URL or TANGLE_ROUTER_URL."
+              exit 1
+            fi
+            if [ -z "$MODEL" ] || [ -z "$PRICE_IN" ] || [ -z "$PRICE_OUT" ]; then
+              echo "::error::Set the exact model and input/output token prices as workflow inputs or repository variables."
+              exit 1
+            fi
+            echo "run=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "::notice::No LLM_API_KEY or TANGLE_API_KEY secret is available. Skipping the paid benchmark."
+            echo "run=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Compare official GEPA and SkillOpt
+        if: steps.config.outputs.run == 'true'
+        env:
+          LLM_API_KEY: ${{ secrets.LLM_API_KEY != '' && secrets.LLM_API_KEY || secrets.TANGLE_API_KEY }}
+          LLM_BASE_URL: ${{ github.event.inputs.base_url != '' && github.event.inputs.base_url || vars.LLM_BASE_URL != '' && vars.LLM_BASE_URL || vars.TANGLE_ROUTER_URL }}
+          LLM_MODEL: ${{ github.event.inputs.model != '' && github.event.inputs.model || vars.LLM_MODEL }}
+          PRICE_IN_PER_M: ${{ github.event.inputs.price_in_per_m != '' && github.event.inputs.price_in_per_m || vars.PRICE_IN_PER_M }}
+          PRICE_OUT_PER_M: ${{ github.event.inputs.price_out_per_m != '' && github.event.inputs.price_out_per_m || vars.PRICE_OUT_PER_M }}
+          PRICE_SOURCE: 'Workflow inputs or repository variables recorded with the run.'
+          OPTIMIZERS: ${{ github.event.inputs.optimizers || 'gepa,skillopt' }}
+          SKILLOPT_EPOCHS: ${{ github.event.inputs.skillopt_epochs || '2' }}
+          SKILLOPT_BATCH_SIZE: ${{ github.event.inputs.skillopt_batch_size || '2' }}
+          MAX_OPTIMIZER_MODEL_COST_USD: ${{ github.event.inputs.max_optimizer_model_cost_usd || '5' }}
+          MAX_TOTAL_COST_USD: ${{ github.event.inputs.max_total_cost_usd || '20' }}
+          OPTIMIZER_PYTHON: ${{ github.workspace }}/clients/python/.venv/bin/python
+        run: pnpm tsx examples/compare-optimization-methods/index.ts
+
+      - name: Upload method comparison artifact
+        if: always() && steps.config.outputs.run == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: official-optimizer-benchmark-${{ github.run_number }}
+          path: .evolve/compare-optimization-methods/**/comparison.json
+          if-no-files-found: error

examples/benchmarks/appworld/README.md

@@ -53,14 +53,14 @@ Common settings:
 | `GEPA_MAX_PROPOSER_COST_USD` | `5` | GEPA model spend limit for one engine stage |
 | `GEPA_PRICE_IN_PER_M` | required | Exact GEPA input rate per million tokens |
 | `GEPA_PRICE_OUT_PER_M` | required | Exact GEPA output rate per million tokens |
-| `GEPA_MAX_MODEL_COST_USD` | `MAX_RUN_COST_USD` | Shared GEPA model spend limit |
+| `GEPA_MAX_MODEL_COST_USD` | `MAX_OPTIMIZER_MODEL_COST_USD` | GEPA model spend limit |
 | `SKILLOPT_MODEL` | `BENCH_MODEL` | Model used by SkillOpt |
 | `SKILLOPT_EPOCHS` | `1` | SkillOpt training epochs |
 | `SKILLOPT_BATCH_SIZE` | `2` | SkillOpt train tasks per step |
 | `SKILLOPT_MAX_EVALUATIONS` | core plan size | Maximum SkillOpt candidate-task calls |
 | `SKILLOPT_PRICE_IN_PER_M` | required | Exact optimizer-model input rate per million tokens |
 | `SKILLOPT_PRICE_OUT_PER_M` | required | Exact optimizer-model output rate per million tokens |
-| `SKILLOPT_MAX_MODEL_COST_USD` | `MAX_RUN_COST_USD` | SkillOpt optimizer-model spend limit |
+| `SKILLOPT_MAX_MODEL_COST_USD` | `MAX_OPTIMIZER_MODEL_COST_USD` | SkillOpt optimizer-model spend limit |
 | `TRAIN_N` | `3` | Train tasks per method |
 | `SELECTION_N` | `3` | Tasks used to select one prompt per method |
 | `TEST_N` | `5` | Final comparison tasks |
@@ -72,7 +72,8 @@ Common settings:
 | `CALL_TIMEOUT` | `120` | Seconds per model request |
 | `MAX_TOKENS` | `6000` | Maximum output tokens per model request |
 | `RATE_LIMIT_BUDGET` | `240` | Seconds spent retrying rate limits per request |
-| `MAX_RUN_COST_USD` | `125` | Shared limit for all optimization and final-task spend |
+| `MAX_OPTIMIZER_MODEL_COST_USD` | `5` | Equal optimizer-model spend limit per method |
+| `MAX_TOTAL_COST_USD` | `125` | Shared limit for all optimization and final-task spend |
 | `OUT_DIR` | `/tmp/appworld-bench` | Output directory |
 
 The defaults are sized to check the workflow.

examples/benchmarks/appworld/run-bench.ts

@@ -55,7 +55,12 @@ const API_KEY = process.env.OPENAI_API_KEY?.trim() ?? ''
 const TRAIN_N = positiveIntegerEnv('TRAIN_N', 3)
 const SELECTION_N = positiveIntegerEnv('SELECTION_N', 3)
 const TEST_N = positiveIntegerEnv('TEST_N', 5)
-const GEPA_MAX_PROPOSER_COST_USD = positiveNumberEnv('GEPA_MAX_PROPOSER_COST_USD', 5)
+const MAX_OPTIMIZER_MODEL_COST_USD = positiveNumberEnv('MAX_OPTIMIZER_MODEL_COST_USD', 5)
+const MAX_TOTAL_COST_USD = positiveNumberEnv('MAX_TOTAL_COST_USD', 125)
+const GEPA_MAX_PROPOSER_COST_USD = positiveNumberEnv(
+  'GEPA_MAX_PROPOSER_COST_USD',
+  MAX_OPTIMIZER_MODEL_COST_USD,
+)
 const SKILLOPT_EPOCHS = positiveIntegerEnv('SKILLOPT_EPOCHS', 1)
 const SKILLOPT_BATCH_SIZE = positiveIntegerEnv('SKILLOPT_BATCH_SIZE', 2)
 const SKILLOPT_CORE_EVALUATIONS =
@@ -75,7 +80,6 @@ const OPTIMIZATION_CONCURRENCY = positiveIntegerEnv('OPTIMIZATION_CONCURRENCY',
 const CALL_TIMEOUT = positiveNumberEnv('CALL_TIMEOUT', 120)
 const MAX_TOKENS = positiveIntegerEnv('MAX_TOKENS', 6000)
 const RATE_LIMIT_BUDGET = positiveNumberEnv('RATE_LIMIT_BUDGET', 240)
-const MAX_RUN_COST_USD = positiveNumberEnv('MAX_RUN_COST_USD', 125)
 const OUT_DIR = stringEnv('OUT_DIR', join(tmpdir(), 'appworld-bench'))
 const SEED = safeIntegerEnv('SEED', 42)
 if (TEMPERATURE > 2) throw new Error('TEMPERATURE must be between 0 and 2')
@@ -365,10 +369,10 @@ async function main(): Promise<void> {
   )
   const budgets = {
     ...(selected.has('gepa')
-      ? { gepa: optimizerModelBudgetFromEnv('GEPA', MAX_RUN_COST_USD) }
+      ? { gepa: optimizerModelBudgetFromEnv('GEPA', MAX_OPTIMIZER_MODEL_COST_USD) }
       : {}),
     ...(selected.has('skillopt')
-      ? { skillopt: optimizerModelBudgetFromEnv('SKILLOPT', MAX_RUN_COST_USD) }
+      ? { skillopt: optimizerModelBudgetFromEnv('SKILLOPT', MAX_OPTIMIZER_MODEL_COST_USD) }
       : {}),
   }
   const methods = officialMethods(selected, budgets)
@@ -397,7 +401,7 @@ async function main(): Promise<void> {
       maxConcurrency: MAXCONC,
       expectUsage: 'assert',
     },
-    costCeiling: MAX_RUN_COST_USD,
+    costCeiling: MAX_TOTAL_COST_USD,
     expectUsage: 'assert',
   })
 
@@ -426,7 +430,8 @@ async function main(): Promise<void> {
         skillopt: selected.has('skillopt') ? SKILLOPT_MAX_EVALUATIONS : null,
       },
       gepaMaxProposerCostUsd: selected.has('gepa') ? GEPA_MAX_PROPOSER_COST_USD : null,
-      allRunCostUsd: MAX_RUN_COST_USD,
+      optimizerModelCostUsdPerMethod: MAX_OPTIMIZER_MODEL_COST_USD,
+      allRunCostUsd: MAX_TOTAL_COST_USD,
       repetitionsPerFinalTask: REPS,
       optimizationConcurrency: OPTIMIZATION_CONCURRENCY,
       taskConcurrency: MAXCONC,

examples/benchmarks/gsm8k/compare-optimization-methods.ts

@@ -64,7 +64,12 @@ const OPTIMIZER_PYTHON = process.env.OPTIMIZER_PYTHON?.trim() || 'python'
 const OPTIMIZER_API_KEY = (process.env.OPTIMIZER_API_KEY || API_KEY)?.trim()
 const OPTIMIZER_BASE_URL = (process.env.OPTIMIZER_BASE_URL || BASE_URL).trim()
 const GEPA_MODEL = process.env.GEPA_MODEL || MODEL
-const GEPA_MAX_PROPOSER_COST_USD = positiveNumberEnv('GEPA_MAX_PROPOSER_COST_USD', 10)
+const MAX_OPTIMIZER_MODEL_COST_USD = positiveNumberEnv('MAX_OPTIMIZER_MODEL_COST_USD', 10)
+const MAX_TOTAL_COST_USD = positiveNumberEnv('MAX_TOTAL_COST_USD', 25)
+const GEPA_MAX_PROPOSER_COST_USD = positiveNumberEnv(
+  'GEPA_MAX_PROPOSER_COST_USD',
+  MAX_OPTIMIZER_MODEL_COST_USD,
+)
 const SKILLOPT_MODEL = process.env.SKILLOPT_MODEL || MODEL
 const SKILLOPT_EPOCHS = positiveIntegerEnv('SKILLOPT_EPOCHS', 2)
 const SKILLOPT_BATCH_SIZE = positiveIntegerEnv('SKILLOPT_BATCH_SIZE', 4)
@@ -80,7 +85,6 @@ const OPTIMIZATION_CONCURRENCY = positiveIntegerEnv('OPTIMIZATION_CONCURRENCY',
 const TASK_CONCURRENCY = positiveIntegerEnv('TASK_CONCURRENCY', 2)
 const REPS = positiveIntegerEnv('REPS', 1)
 const MAX_SMOKE_COST_USD = positiveNumberEnv('MAX_SMOKE_COST_USD', 2)
-const MAX_RUN_COST_USD = positiveNumberEnv('MAX_RUN_COST_USD', 25)
 const SMOKE = process.env.SMOKE === '1'
 const SEED = 42
 const RESAMPLES = 4_000
@@ -284,12 +288,12 @@ async function main() {
 
   const gepaModelBudget = optimizerModelBudgetFromEnv(
     'GEPA',
-    MAX_RUN_COST_USD,
+    MAX_OPTIMIZER_MODEL_COST_USD,
     CUSTOM_TOKEN_PRICING,
   )
   const skillOptModelBudget = optimizerModelBudgetFromEnv(
     'SKILLOPT',
-    MAX_RUN_COST_USD,
+    MAX_OPTIMIZER_MODEL_COST_USD,
     CUSTOM_TOKEN_PRICING,
   )
   const runner = {
@@ -368,7 +372,7 @@ async function main() {
       maxConcurrency: TASK_CONCURRENCY,
       expectUsage: 'assert',
     },
-    costCeiling: MAX_RUN_COST_USD,
+    costCeiling: MAX_TOTAL_COST_USD,
     dispatchTimeoutMs: CALL_TIMEOUT_MS,
     expectUsage: 'assert',
   })
@@ -416,7 +420,8 @@ async function main() {
         coreEvaluations: SKILLOPT_CORE_EVALUATIONS,
       },
       smokeCostUsd: MAX_SMOKE_COST_USD,
-      allRunCostUsd: MAX_RUN_COST_USD,
+      optimizerModelCostUsdPerMethod: MAX_OPTIMIZER_MODEL_COST_USD,
+      allRunCostUsd: MAX_TOTAL_COST_USD,
       worker: {
         requestTimeoutMs: CALL_TIMEOUT_MS,
         maxOutputTokens: WORKER_MAX_TOKENS,

examples/compare-optimization-methods/README.md

@@ -94,17 +94,18 @@ Replace all four example rates with the exact rates charged by that endpoint.
 | `GEPA_MAX_PROPOSER_COST_USD` | `5` | Maximum GEPA model spend inside one engine stage. |
 | `GEPA_PRICE_IN_PER_M` | required | Exact GEPA input rate per million tokens. |
 | `GEPA_PRICE_OUT_PER_M` | required | Exact GEPA output rate per million tokens. |
-| `GEPA_MAX_MODEL_COST_USD` | `MAX_RUN_COST_USD` | Shared GEPA model spend limit. |
+| `GEPA_MAX_MODEL_COST_USD` | `MAX_OPTIMIZER_MODEL_COST_USD` | GEPA model spend limit. |
 | `GEPA_MAX_MODEL_REQUESTS` | `100` | Shared GEPA model request limit. |
 | `SKILLOPT_MODEL` | `LLM_MODEL` | Model used by SkillOpt reflection and editing. |
 | `SKILLOPT_EPOCHS` | `2` | SkillOpt training epochs. |
 | `SKILLOPT_BATCH_SIZE` | `2` | SkillOpt train cases per step. |
 | `SKILLOPT_MAX_EVALUATIONS` | core plan size | Maximum SkillOpt candidate-case calls. |
 | `SKILLOPT_PRICE_IN_PER_M` | required | Exact optimizer-model input rate per million tokens. |
 | `SKILLOPT_PRICE_OUT_PER_M` | required | Exact optimizer-model output rate per million tokens. |
-| `SKILLOPT_MAX_MODEL_COST_USD` | `MAX_RUN_COST_USD` | SkillOpt optimizer-model spend limit. |
+| `SKILLOPT_MAX_MODEL_COST_USD` | `MAX_OPTIMIZER_MODEL_COST_USD` | SkillOpt optimizer-model spend limit. |
 | `SKILLOPT_MAX_MODEL_REQUESTS` | `100` | SkillOpt optimizer-model request limit. |
-| `MAX_RUN_COST_USD` | `12` | Shared limit for all optimization and final-case spend. |
+| `MAX_OPTIMIZER_MODEL_COST_USD` | `5` | Equal optimizer-model spend limit per method. |
+| `MAX_TOTAL_COST_USD` | `20` | Shared limit for all optimization and final-case spend. |
 | `OPTIMIZATION_CONCURRENCY` | `1` | Methods allowed to optimize concurrently. |
 | `BILLING_NOTE` | inferred | Billing context saved with the result. |
 | `PRICE_SOURCE` | inferred | Source of the token prices saved with the result. |