feat(optimization): adopt official methods across knowledge surfaces

.github/workflows/ci.yml

@@ -38,3 +38,38 @@ jobs:
 
       - name: Verify packed package lifecycle
         run: pnpm verify:package
+
+  official-optimizers:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: pnpm/action-setup@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: pnpm
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install JavaScript dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Install official Python optimizers
+        run: |
+          python -m venv .venv-official
+          .venv-official/bin/python -m pip install --upgrade pip
+          RPC_VERSION=$(node -e "const v=require('./package.json').dependencies['@tangle-network/agent-eval']; if (!/^\\d+\\.\\d+\\.\\d+$/.test(v)) throw new Error('agent-eval dependency must be exact'); console.log(v)")
+          .venv-official/bin/python -m pip install \
+            "agent-eval-rpc==$RPC_VERSION" \
+            "gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f" \
+            "skillopt @ git+https://github.com/microsoft/SkillOpt.git@61735e3922efc2b90c6d6cab561e62e98452ca90"
+
+      - name: Run packed package through official GEPA and SkillOpt
+        run: pnpm verify:official-optimizers
+        env:
+          AGENT_EVAL_TEST_PYTHON: ${{ github.workspace }}/.venv-official/bin/python

.github/workflows/publish.yml

@@ -20,6 +20,10 @@ jobs:
           cache: pnpm
           registry-url: https://registry.npmjs.org
 
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
       - name: Install deps
         run: pnpm install --frozen-lockfile
 
@@ -38,6 +42,21 @@ jobs:
       - name: Verify packed package lifecycle
         run: pnpm run verify:package
 
+      - name: Install official Python optimizers
+        run: |
+          python -m venv .venv-official
+          .venv-official/bin/python -m pip install --upgrade pip
+          RPC_VERSION=$(node -e "const v=require('./package.json').dependencies['@tangle-network/agent-eval']; if (!/^\\d+\\.\\d+\\.\\d+$/.test(v)) throw new Error('agent-eval dependency must be exact'); console.log(v)")
+          .venv-official/bin/python -m pip install \
+            "agent-eval-rpc==$RPC_VERSION" \
+            "gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f" \
+            "skillopt @ git+https://github.com/microsoft/SkillOpt.git@61735e3922efc2b90c6d6cab561e62e98452ca90"
+
+      - name: Run packed package through official GEPA and SkillOpt
+        run: pnpm run verify:official-optimizers
+        env:
+          AGENT_EVAL_TEST_PYTHON: ${{ github.workspace }}/.venv-official/bin/python
+
       - name: Verify tag/version lock
         run: |
           NPM_VERSION=$(node -p "require('./package.json').version")

CHANGELOG.md

@@ -1,23 +1,32 @@
 # Changelog
 
-## Unreleased
+## 5.0.0
 
 ### Breaking Changes
 
 - Retrieval improvement now requires independent train, selection, and final scenarios plus an explicit complete `OptimizationMethod`.
+- Serialized retrieval and RAG optimization now requires an immutable `executionRef` covering candidate execution and scoring behavior.
 - Memory configuration improvement now requires a baseline configuration, a complete `OptimizationMethod`, and independent train, selection, and final histories.
+- The RAG lifecycle promotion callback is now `decidePromotion` and runs only after final evidence passes regression, provenance, and cost checks.
+- Knowledge improvement requires an immutable `implementationRef`, separates repeatable development evaluation from single-use final evaluation, and refuses to resume after interrupted final scoring.
+- Memory candidate factories no longer receive scenario, repetition, or seed identity and must report observed external charges through `recordExternalCost()`.
+- Answer-quality hooks require immutable evaluator identity, final scenario identity, and complete cost evidence.
 - Removed the public retrieval and memory proposer-search options; candidate generation and selection now belong to `agent-eval` methods.
 
 ### Added
 
 - Added a shared serialized-candidate adapter for running complete `agent-eval` optimization methods with canonical candidate identity and untouched final comparison.
 - Added full RAG configuration optimization and KB maintenance policy optimization.
-- Added an explicit `OptimizationMethod` factory for bounded retrieval configuration enumeration over small finite spaces.
+- Added direct support for official GEPA and SkillOpt methods through the shared `OptimizationMethod` contract.
+- Added durable per-configuration memory candidate identities to prevent stale result reuse.
+- Added live activation verification so resumed memory runs reject configuration drift.
+- Added private execution contexts that expose memory operations, cancellation, and cost metering without evaluation labels.
 
 ### Changed
 
-- Updated `@tangle-network/agent-eval` to `0.123.8`.
+- Updated `@tangle-network/agent-eval` to `0.126.3` and `@tangle-network/agent-interface` to `0.32.0`.
 - Kept memory provider evaluations resumable and branch-isolated while moving search ownership to the supplied method.
+- Restricted immutable references to lowercase SHA-256 and full Git commit identities.
 
 ## 4.1.0
 

README.md

@@ -9,7 +9,7 @@ Supply application callbacks for those decisions, or use `@tangle-network/agent-
 ## Install
 
 ```bash
-pnpm add @tangle-network/agent-knowledge
+pnpm add @tangle-network/agent-knowledge @tangle-network/agent-eval@0.126.3
 ```
 
 Requires Node.js 20.19 or later.
@@ -121,6 +121,7 @@ import {
 const result = await improveKnowledgeBase({
   root: './support-kb',
   goal: 'Answer refund questions using the current policy',
+  implementationRef: 'git:0123456789abcdef0123456789abcdef01234567',
   runId: 'refund-policy-2026-07',
   maxCandidates: 3,
   updateKnowledge: async ({ candidateRoot }) => {
@@ -142,9 +143,15 @@ if (result.candidate && result.evaluation?.passed) {
 The update can come from a research agent, a coding agent, a source connector, or deterministic code.
 The comparison must return the product measures that decide whether the change helped.
 
-Use the same `runId` to resume an interrupted run.
+Use the same `runId` and `implementationRef` to resume an interrupted run.
+Change `implementationRef` whenever callbacks, evaluation policy, models, indexes, or external configuration change.
+Reusing a run ID with a different implementation reference fails before cached work or callbacks run.
 Different run IDs create separate candidate workspaces, so workers can explore in parallel.
 Promotion checks the original base hash and rejects a stale candidate instead of replacing newer work.
+Candidate retries use `evaluateDevelopment` when provided, otherwise they use deterministic validation, readiness, and KB quality checks.
+Development evaluation must use only train or selection data.
+The configured `evaluate` callback and final RAG phases run once, on the first candidate that passes those development checks.
+A failed final evaluation ends the run instead of selecting another candidate against final data.
 
 Candidate promotion currently requires Linux because it relies on Linux directory descriptors for exact file identity.
 
@@ -155,7 +162,6 @@ Use the narrowest API that matches the job:
 | API | What it does |
 |---|---|
 | `runRetrievalImprovementLoop` | Runs one complete `OptimizationMethod` over serialized retrieval configuration. |
-| `boundedRetrievalConfigMethod` | Builds a complete method that enumerates a small finite retrieval grid, limited to 128 configurations by default. |
 | `runRagOptimization` | Optimizes retrieval and answer behavior as one serialized RAG configuration. |
 | `optimizeKnowledgeBasePolicy` | Optimizes a KB maintenance policy, then applies only the selected policy to an isolated candidate. |
 | `scoreKnowledgeBaseIndex` | Measures KB structure, citations, source freshness, and configured quality thresholds. |
@@ -164,36 +170,56 @@ Use the narrowest API that matches the job:
 | `improveKnowledgeBase` | Adds resumable state, isolated candidates, exact promotion, and conflict detection around that process. |
 
 ```ts
-const method = boundedRetrievalConfigMethod({
-  searchSpace: { k: [5, 10], reranker: [false, true] },
-})
+import type {
+  OptimizationMethod,
+} from '@tangle-network/agent-eval/campaign'
+import {
+  runRetrievalImprovementLoop,
+  type RetrievalEvalArtifact,
+  type RetrievalEvalScenario,
+} from '@tangle-network/agent-knowledge'
 
-const result = await runRetrievalImprovementLoop({
-  baseline: { k: 5, reranker: false },
-  method,
-  trainScenarios,
-  selectionScenarios,
-  finalScenarios,
-  retrieve: ({ scenario, config, k }) => search(scenario.query, { ...config, k }),
-  runDir: 'refund-retrieval-v1',
-  expectUsage: 'off', // Local search does not make a billable model call.
-})
+async function tuneRetrieval(
+  method: OptimizationMethod<RetrievalEvalScenario, RetrievalEvalArtifact>,
+) {
+  return runRetrievalImprovementLoop({
+    executionRef: 'git:0123456789abcdef0123456789abcdef01234567',
+    baseline: { k: 5, reranker: false },
+    method,
+    trainScenarios,
+    selectionScenarios,
+    finalScenarios,
+    retrieve: ({ scenario, config, k }) => search(scenario.query, { ...config, k }),
+    runDir: 'refund-retrieval',
+    expectUsage: 'off', // Local search does not make a billable model call.
+  })
+}
 ```
 
 The method receives train and selection cases.
 `agent-eval` keeps final cases out of the search and measures the exact selected configuration on them afterward.
 Every optimization call requires an explicit complete method.
-Pass an applicable official method, a custom method, or `boundedRetrievalConfigMethod()` for a small finite retrieval grid.
+Create an official method with `gepaOptimizationMethod()` or `skillOptOptimizationMethod()` from `@tangle-network/agent-eval/campaign`, or pass another complete public `OptimizationMethod`.
 Reuse the run directory only with the method's compatible resume mode.
 Use separate run directories to explore branches in parallel.
 Optimizer-specific identity and resume settings stay on the supplied method; this package does not reinterpret them.
-The supplied method owns `evaluationVersion`, engine or skill inputs, and resume compatibility.
-`agent-eval` component surfaces stay outside this adapter; each knowledge candidate has one canonical serialized identity.
-See the [`agent-eval` method guide](https://github.com/tangle-network/agent-eval/blob/main/docs/campaign-proposers.md) for official GEPA and Omni methods.
+The supplied method owns its engine inputs and resume state.
+`executionRef` owns retrieval, index, judge, model, and external-service identity.
+Use `git:<40 lowercase hex>` or `sha256:<64 lowercase hex>` and change it whenever any candidate execution or scoring behavior changes.
+Each candidate has one canonical serialized identity.
+See the [`agent-eval` method guide](https://github.com/tangle-network/agent-eval/blob/main/docs/campaign-proposers.md) for official GEPA and SkillOpt methods.
 SkillOpt is skill-only; use it here only when the serialized candidate is itself a skill.
+Read reported spend from `result.comparison.totalCost` and upstream source and run identity from `result.comparison.best.provenance`.
+Official external methods must report observed package identity.
+Custom in-process methods have no external package identity, so their behavior must be covered by `executionRef`.
+Treat `accountingComplete: false` as incomplete evidence for activation.
 
 Retrieval and answer generation remain callbacks.
 This lets the same evaluation code work with local search, vector databases, hybrid search, rerankers, and hosted RAG services.
+Adaptive diagnosis, acquisition, and update callbacks finish before retrieval or RAG final scoring starts.
+Only answer evaluation, the terminal promotion decision, and the returned result can observe selected configurations.
+Answer-quality evidence must name at least two final scenario IDs, immutable dataset and evaluator references, non-empty finite metrics, and observed cost accounting.
+Promotion also requires `answerQualityCostCeiling`.
 
 ## Integrate memory systems
 
@@ -203,8 +229,15 @@ Install the provider you use, create its client, and pass that client to the ada
 
 The memory APIs support scoped reads and writes, isolated branches, ordered histories, independent train, selection, and final comparisons, and adapter experiments.
 Use them to compare a provider against no memory or another provider on the same tasks before changing production behavior.
-`runAgentMemoryImprovement` accepts a complete `OptimizationMethod`, evaluates each serialized configuration in an isolated provider branch, and activates only a final-data winner through compare-and-set.
-Paid memory improvement defaults to zero-dollar optimization and final limits; set both limits and a per-evaluation maximum before enabling paid work.
+`runAgentMemoryImprovement` accepts a complete `OptimizationMethod`, evaluates each serialized configuration in an isolated provider branch, and activates only a winner that passes a separate final comparison.
+Set `implementationRef` to `git:<40 lowercase hex>` or `sha256:<64 lowercase hex>` covering the installed implementation, method configuration, candidate construction, execution behavior, and external configuration so incompatible state cannot resume.
+The run records one immutable candidate reference for each memory configuration and refuses cached results if that reference changes.
+Each improvement candidate declares a maximum for one sequence and one recovery attempt.
+The adapter must enforce that maximum with its provider before starting external work.
+The adapter callback must call `recordExternalCost()` with each observed charge.
+Positive external work without a receipt is recorded as incomplete cost accounting, not as the configured maximum.
+Use `0` only for a free local path.
+Paid memory improvement defaults to a zero-dollar total limit; set `maxTotalCostUsd` and `maximumEvaluationCostUsd` before enabling paid work.
 
 ## Run benchmarks
 

docs/eval/rag-eval-roadmap.md

@@ -21,12 +21,12 @@ Done:
 
 - `runRetrievalImprovementLoop()` runs a complete `agent-eval` optimization method over retrieval configs.
 - `runRagOptimization()` does the same for a serialized retrieval and answer configuration.
-- `boundedRetrievalConfigMethod()` builds an explicitly supplied method for finite retrieval grids of at most 128 configurations by default.
 - `runRagKnowledgeImprovementLoop()` exposes the whole RAG lifecycle as typed phases:
   retrieval tuning, gap diagnosis, knowledge acquisition, knowledge update, answer-quality eval, and promotion.
 - Retrieval scenarios can label pages, page paths, sources, source anchors, and source spans.
 - The retrieval judge reports recall, MRR, nDCG, and precision@k; `agent-eval` reports cost separately.
 - Selection and final data remain independent, and the optimization method never receives final cases.
+- Every reusable optimization requires an immutable execution reference covering retrieval, index, model, judge, and external-service behavior.
 - The integration is tested with complete methods for retrieval and full RAG configuration.
 - The lifecycle loop is tested both with pluggable phase hooks and with a real local KB update through `runKnowledgeResearchLoop()`.
 - `ragAnswerQualityJudge()` and `createRagAnswerQualityHook()` score context precision/recall/relevance/sufficiency, faithfulness, answer relevance/correctness, citation support, abstention, and unsupported-answer rate.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-knowledge",
-  "version": "4.1.0",
+  "version": "5.0.0",
   "description": "Build, search, evaluate, and improve source-backed knowledge bases.",
   "homepage": "https://github.com/tangle-network/agent-knowledge#readme",
   "repository": {
@@ -69,11 +69,12 @@
     "lint": "biome check src tests",
     "format": "biome format --write src tests",
     "check:skills": "node scripts/check-skills.mjs",
-    "verify:package": "pnpm run check:skills && node scripts/verify-package.mjs"
+    "verify:package": "pnpm run check:skills && node scripts/verify-package.mjs",
+    "verify:official-optimizers": "node scripts/verify-official-optimizers.mjs"
   },
   "dependencies": {
-    "@tangle-network/agent-eval": "^0.123.8",
-    "@tangle-network/agent-interface": "^0.31.0",
+    "@tangle-network/agent-eval": "0.126.3",
+    "@tangle-network/agent-interface": "^0.32.0",
     "proper-lockfile": "4.1.2",
     "zod": "^4.4.3"
   },