tangle-network · drewstone · Jul 24, 2026 · Jul 24, 2026 · Jul 24, 2026 · Jul 24, 2026
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -61,3 +61,26 @@ jobs:
 
       - name: Verify agent-bench against current agent-runtime
         run: pnpm run verify:bench
+
+  official-optimizers:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: pnpm/action-setup@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: pnpm
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install package set
+        run: pnpm install --frozen-lockfile
+
+      - name: Verify packed Runtime with official GEPA and SkillOpt
+        run: pnpm run verify:official-optimizers
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -22,6 +22,10 @@ jobs:
           cache: pnpm
           registry-url: https://registry.npmjs.org
 
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
       - name: Install deps
         run: pnpm install --frozen-lockfile
 
@@ -40,6 +44,9 @@ jobs:
       - name: Verify packed package exports
         run: pnpm run verify:package
 
+      - name: Verify packed Runtime with official GEPA and SkillOpt
+        run: pnpm run verify:official-optimizers
+
       - name: Verify agent-bench against this release
         run: pnpm run verify:bench
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,16 @@
 # Changelog
 
+## 0.105.0
+
+- Add `officialGepa(...)` and `officialSkillOpt(...)` as Runtime adapters over the upstream GEPA and Microsoft SkillOpt implementations in `@tangle-network/agent-eval`.
+- Require one complete `OptimizationMethod` for profile improvement and keep final-test scenarios outside optimizer input.
+- Authorize every exact execution-capable profile candidate before it reaches an agent.
+- Preserve resumed optimizer spend, model identity, package provenance, and separate optimization versus final-test costs in activation receipts.
+- Verify released Python packages, pinned source revisions, resume behavior, concurrency, and packed external installs in CI.
+- Keep code improvement on Runtime-owned isolated Git worktrees.
+- Remove the retired local prompt, profile-diff, campaign OTLP, and record-only optimizer paths.
+- Require `@tangle-network/agent-eval` 0.126.x.
+
 ## 0.104.0
 
 - Add the Tangle Sandbox exact-process environment provider for verified candidate execution.

diff --git a/README.md b/README.md
diff --git a/bench/CHANGELOG.md b/bench/CHANGELOG.md
@@ -1,5 +1,12 @@
 # Changelog
 
+## 0.4.0
+
+- Add the resumable SWE improvement loop backed by the official GEPA engine.
+- Isolate every concurrent candidate in its own Git worktree and authorize the exact candidate before execution.
+- Include the complete Runtime, Bench, and installed SWE judge implementations in resume identity.
+- Consume `@tangle-network/agent-runtime` 0.105.x, `@tangle-network/agent-eval` 0.126.6, and `@tangle-network/agent-knowledge` 5.0.1.
+
 ## 0.3.8
 
 - Resolve package-owned fixtures and scripts from both source and compiled installs, and prove ToolLLM fixture loading from the packed package.

diff --git a/bench/README.md b/bench/README.md
@@ -26,6 +26,13 @@ pnpm install                                              # tsx + link parent
 
 The judge needs only Docker; workers need a model key (Tangle router `TANGLE_API_KEY`, or a direct provider).
 
+Live optimizer scripts require explicit token prices so cost records cannot be guessed.
+Use the `REFLECT_` prefix for the general benchmark scripts and `GEPA_OPTIMIZER_` for the SWE arena seat.
+For either prefix, set `INPUT_USD_PER_MILLION`, `CACHED_INPUT_USD_PER_MILLION`, `CACHE_WRITE_USD_PER_MILLION`, and `OUTPUT_USD_PER_MILLION`.
+The SWE arena seat reads its model from `GEPA_OPTIMIZER_MODEL`, its URL from `GEPA_OPTIMIZER_BASE_URL`, and its key from `GEPA_OPTIMIZER_API_KEY`.
+It falls back to the configured driver model, `ROUTER_BASE`, and `TANGLE_API_KEY`.
+Missing prices fail before an optimizer model call.
+
 Retain every official per-test log and report before the temporary evaluator directory is removed:
 
 ```ts

diff --git a/bench/gen6-config.json b/bench/gen6-config.json
diff --git a/bench/package.json b/bench/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.3.8",
+  "version": "0.4.0",
   "type": "module",
   "description": "The unified benchmark suite for agent-runtime agents: 31 adapters (commit0, enterpriseops-gym, ragbench, crag, nomiracl, open-rag-bench, t2-ragbench, tau3-banking, bfcl, finresearchbench, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.",
   "repository": {
@@ -40,9 +40,9 @@
     "verify:pier": "tsx scripts/verify-pier-pair.mts"
   },
   "dependencies": {
-    "@tangle-network/agent-eval": "0.125.0",
+    "@tangle-network/agent-eval": "0.126.6",
     "@tangle-network/agent-interface": "0.32.0",
-    "@tangle-network/agent-knowledge": "^4.1.0",
+    "@tangle-network/agent-knowledge": "5.0.1",
     "@tangle-network/agent-runtime": "workspace:*",
     "@tangle-network/sandbox": "^0.12.0"
   },

diff --git a/bench/scripts/trata-hedge/README.md b/bench/scripts/trata-hedge/README.md
@@ -13,9 +13,9 @@ sparse (1 iff all themes). No deployable ground-truth checker — it's an **orac
 - **NOT** the verifier-grounded selector gate (our HumanEval/commit0 headline). The only
   checker is the judge itself, so selecting by it = selecting by the eval metric (an
   oracle, a Goodhart trap). See `docs/results.md`.
-- **IS** admissible: (1) more-compute headroom (pass@k), (2) the **improvement loop**
-  (`selfImprove`/GEPA optimizing the analyst directive *against this judge*, held-out
-  gated — the legitimate use of a judge domain), with the honest caveat that an
+- **IS** admissible: (1) more-compute headroom (pass@k), (2) prompt optimization
+  (`improve` with agent-eval's official GEPA method, using disjoint train, selection,
+  and final-test partitions), with the honest caveat that an
   LLM-judge can be *gamed*, so a held-out lift means "higher judge score," which without
   ground truth we can't independently certify as "better analysis."
 
@@ -52,5 +52,6 @@ dotenvx run -f ~/company/devops/secrets/.env.keys -f ~/company/devops/secrets/ag
 ## Next steps
 
 1. Agentic solver: our sandbox runtime browses the corpus + cites (fair baseline).
-2. The improvement loop: `selfImprove` on the analyst directive vs this judge, held-out
-   gated, across the 102 tasks — with the judge-gaming caveat stated.
+2. Prompt optimization: run `bench/src/trata-gepa.mts` across all 102 tasks with
+   explicit train, selection, and final-test partitions, and state the judge-gaming
+   caveat with every result.
diff --git a/bench/src/gate.ts b/bench/src/gate.ts
@@ -427,7 +427,7 @@ export async function runGate(opts: RunGateOptions): Promise<GateReport> {
       root: a.label,
       nodes: [],
       total: acc.get(a.label)!.spend,
-      statusCounts: { done: 0, failed: 0, cancelled: 0, pending: 0 },
+      statusCounts: { done: 0, failed: 0, cancelled: 0, pending: 0, waiting: 0 },
     },
   }))
   const equalK = equalKOnCost(equalKArms)

diff --git a/bench/src/hev-eval.mts b/bench/src/hev-eval.mts
@@ -27,6 +27,7 @@ async function complete(base: string, key: string, model: string, prompt: string
 async function main(): Promise<void> {
   const key = process.env.TANGLE_API_KEY
   if (!key) throw new Error('TANGLE_API_KEY required')
+  const apiKey: string = key
   const base = process.env.ROUTER_BASE ?? 'https://api.together.xyz/v1'
   const model = process.env.WORKER_MODEL ?? 'meta-llama/Meta-Llama-3-8B-Instruct-Lite'
   const instruction = process.env.INSTRUCTION_FILE
@@ -51,8 +52,10 @@ async function main(): Promise<void> {
   let i = 0
   async function worker(): Promise<void> {
     while (i < tasks.length) {
-      const t = tasks[i++]
-      const reply = await complete(base, key, model, `${instruction}\n\n\`\`\`python\n${t.prompt}\`\`\``, maxTokens)
+      const t = tasks[i]
+      i += 1
+      if (!t) continue
+      const reply = await complete(base, apiKey, model, `${instruction}\n\n\`\`\`python\n${t.prompt}\`\`\``, maxTokens)
       const { pass: p } = await runChecker(t, extractCode(reply))
       if (p === 1) pass += 1
       else fails.push(t.taskId)