feat(optimization): run knowledge searches through complete methods

AGENTS.md

@@ -75,7 +75,9 @@ The parser rejects absolute paths, `..`, control characters, and writes outside
 
 ## Eval Boundary
 
-Compare candidate knowledge bases on an actual task corpus by running an `@tangle-network/agent-eval` improvement loop (`runImprovementLoop`) over the variants; each run is scored into a `RunRecord`.
+Use a complete `OptimizationMethod` from `@tangle-network/agent-eval` with `runRetrievalImprovementLoop()`, `runRagOptimization()`, `optimizeKnowledgeBasePolicy()`, or `runAgentMemoryImprovement()`.
+The method owns candidate search and resume compatibility.
+This package owns serialized knowledge candidates, real KB or memory adapters, isolated data partitions, and safe activation.
 
 Use `knowledgeReleaseReport()` before promotion. It folds the candidate and baseline `RunRecord[]` (plus optional traces and the gate decision) into `agent-eval` release confidence evidence.
 

CHANGELOG.md

@@ -1,5 +1,24 @@
 # Changelog
 
+## Unreleased
+
+### Breaking Changes
+
+- Retrieval improvement now requires independent train, selection, and final scenarios plus either a complete `OptimizationMethod` or a bounded finite configuration space.
+- Memory configuration improvement now requires a baseline configuration, a complete `OptimizationMethod`, and independent train, selection, and final histories.
+- Removed the public retrieval and memory proposer-search options; candidate generation and selection now belong to `agent-eval` methods.
+
+### Added
+
+- Added a shared serialized-candidate adapter for running complete `agent-eval` optimization methods with canonical candidate identity and untouched final comparison.
+- Added full RAG configuration optimization and KB maintenance policy optimization.
+- Added bounded retrieval configuration enumeration for small finite spaces.
+
+### Changed
+
+- Updated `@tangle-network/agent-eval` to `0.123.8`.
+- Kept memory provider evaluations resumable and branch-isolated while moving search ownership to the supplied method.
+
 ## 4.1.0
 
 ### Added

README.md

@@ -21,9 +21,10 @@ Requires Node.js 20.19 or later.
 | Create a file-backed knowledge base | `initKnowledgeBase`, `addSourceText`, `applyKnowledgeWriteBlocks` | package root |
 | Search an existing package knowledge base | `createFileSystemSearchProvider` | package root |
 | Improve a live knowledge base without editing it in place | `improveKnowledgeBase` | package root |
-| Tune retrieval against labeled questions | `runRetrievalImprovementLoop` | package root |
+| Optimize retrieval or a complete RAG configuration | `runRetrievalImprovementLoop`, `runRagOptimization` | package root |
+| Optimize a KB maintenance policy | `optimizeKnowledgeBasePolicy` | package root |
 | Run retrieval, research, answer checks, and promotion as one process | `runRagKnowledgeImprovementLoop` | package root |
-| Compare or integrate agent memory systems | `AgentMemoryAdapter` and provider adapters | `/memory` |
+| Compare providers or optimize memory configuration | `AgentMemoryAdapter`, `runAgentMemoryImprovement` | `/memory` |
 | Read from external authorities | `KnowledgeSource` and source adapters | `/sources` |
 | Run retrieval, answer, KB, or memory benchmark cases | `runKnowledgeBenchmarkSuite` | `/benchmarks` |
 | Use live research or coding agents | `runKnowledgeImprovementJob` | `@tangle-network/agent-runtime/knowledge` |
@@ -153,12 +154,38 @@ Use the narrowest API that matches the job:
 
 | API | What it does |
 |---|---|
-| `runRetrievalImprovementLoop` | Searches retrieval configurations against labeled train and holdout questions, with run and cost limits. |
+| `runRetrievalImprovementLoop` | Runs one complete `OptimizationMethod` over serialized retrieval configuration. |
+| `boundedRetrievalConfigMethod` | Enumerates a small finite retrieval grid, limited to 128 configurations by default. |
+| `runRagOptimization` | Optimizes retrieval and answer behavior as one serialized RAG configuration. |
+| `optimizeKnowledgeBasePolicy` | Optimizes a KB maintenance policy, then applies only the selected policy to an isolated candidate. |
 | `scoreKnowledgeBaseIndex` | Measures KB structure, citations, source freshness, and configured quality thresholds. |
 | `createRagAnswerQualityHook` | Adapts answer-quality checks such as support, relevance, citations, and abstention. |
 | `runRagKnowledgeImprovementLoop` | Connects retrieval tuning, gap diagnosis, source acquisition, KB updates, answer checks, and a promotion decision. |
 | `improveKnowledgeBase` | Adds resumable state, isolated candidates, exact promotion, and conflict detection around that process. |
 
+```ts
+const result = await runRetrievalImprovementLoop({
+  baseline: { k: 5, reranker: false },
+  method, // Any OptimizationMethod that returns this serialized JSON surface.
+  trainScenarios,
+  selectionScenarios,
+  finalScenarios,
+  retrieve: ({ scenario, config, k }) => search(scenario.query, { ...config, k }),
+  runDir: 'refund-retrieval-v1',
+  expectUsage: 'off', // Local search does not make a billable model call.
+})
+```
+
+The method receives train and selection cases.
+`agent-eval` keeps final cases out of the search and measures the exact selected configuration on them afterward.
+Reuse the run directory only with the method's compatible resume mode.
+Use separate run directories to explore branches in parallel.
+Optimizer-specific identity and resume settings stay on the supplied method; this package does not reinterpret them.
+The supplied method owns `evaluationVersion`, engine or skill inputs, and resume compatibility.
+`agent-eval` component surfaces stay outside this adapter; each knowledge candidate has one canonical serialized identity.
+See the [`agent-eval` method guide](https://github.com/tangle-network/agent-eval/blob/main/docs/campaign-proposers.md) for official GEPA and Omni methods.
+SkillOpt is skill-only; use it here only when the serialized candidate is itself a skill.
+
 Retrieval and answer generation remain callbacks.
 This lets the same evaluation code work with local search, vector databases, hybrid search, rerankers, and hosted RAG services.
 
@@ -168,8 +195,10 @@ This lets the same evaluation code work with local search, vector databases, hyb
 The package is not a memory database.
 Install the provider you use, create its client, and pass that client to the adapter.
 
-The memory APIs support scoped reads and writes, isolated branches, ordered histories, holdout comparisons, and adapter experiments.
+The memory APIs support scoped reads and writes, isolated branches, ordered histories, independent train, selection, and final comparisons, and adapter experiments.
 Use them to compare a provider against no memory or another provider on the same tasks before changing production behavior.
+`runAgentMemoryImprovement` accepts a complete `OptimizationMethod`, evaluates each serialized configuration in an isolated provider branch, and activates only a final-data winner through compare-and-set.
+Paid memory improvement defaults to zero-dollar optimization and final limits; set both limits and a per-evaluation maximum before enabling paid work.
 
 ## Run benchmarks
 

docs/eval/rag-eval-roadmap.md

@@ -1,6 +1,6 @@
 # RAG Eval Completion Roadmap
 
-Verdict: `runRetrievalImprovementLoop()` is the right first loop, but it is only the retrieval layer.
+Verdict: use retrieval-only optimization when the retriever is the only changing component, and full RAG optimization when retrieval and answer behavior must move together.
 SOTA RAG evaluation requires retrieval quality, context quality, generated-answer quality, abstention behavior, robustness, and operating budgets.
 
 ## Research Basis
@@ -19,12 +19,15 @@ SOTA RAG evaluation requires retrieval quality, context quality, generated-answe
 
 Done:
 
-- `runRetrievalImprovementLoop()` auto-searches retrieval configs through `agent-eval`.
+- `runRetrievalImprovementLoop()` runs a complete `agent-eval` optimization method over retrieval configs.
+- `runRagOptimization()` does the same for a serialized retrieval and answer configuration.
+- `boundedRetrievalConfigMethod()` remains available for finite retrieval grids of at most 128 configurations by default.
 - `runRagKnowledgeImprovementLoop()` exposes the whole RAG lifecycle as typed phases:
   retrieval tuning, gap diagnosis, knowledge acquisition, knowledge update, answer-quality eval, and promotion.
 - Retrieval scenarios can label pages, page paths, sources, source anchors, and source spans.
-- The retrieval judge reports recall, MRR, nDCG, precision@k, cost, and held-out promotion.
-- The loop is tested with a real `agent-eval` run where `{ k: 2 }` beats `{ k: 1 }`.
+- The retrieval judge reports recall, MRR, nDCG, and precision@k; `agent-eval` reports cost separately.
+- Selection and final data remain independent, and the optimization method never receives final cases.
+- The integration is tested with complete methods for retrieval and full RAG configuration.
 - The lifecycle loop is tested both with pluggable phase hooks and with a real local KB update through `runKnowledgeResearchLoop()`.
 - `ragAnswerQualityJudge()` and `createRagAnswerQualityHook()` score context precision/recall/relevance/sufficiency, faithfulness, answer relevance/correctness, citation support, abstention, and unsupported-answer rate.
 - `normalizeExternalRagScores()` and the row exporters make Ragas, DeepEval, TruLens, RAGChecker, and custom evaluator outputs pluggable instead of hard dependencies.
@@ -56,9 +59,9 @@ Required slices:
 
 Ship criteria:
 
-- Holdout source-span Recall@5 is at least 0.90.
-- Holdout nDCG@5 is at least 0.80.
-- Train-to-holdout recall gap is at most 0.08.
+- Final source-span Recall@5 is at least 0.90.
+- Final nDCG@5 is at least 0.80.
+- Train-to-final recall gap is at most 0.08.
 - Stale or forbidden source hit rate is at most 0.02.
 - p95 retrieval latency and cost do not regress by more than 10 percent versus baseline.
 
@@ -69,9 +72,9 @@ Score it with deterministic checks first and LLM judges only for semantic qualit
 
 Ship criteria:
 
-- Faithfulness or groundedness is at least 0.95 on holdout.
-- Answer relevance is at least 0.90 on holdout.
-- Answer correctness is at least 0.85 on human-labeled holdout.
+- Faithfulness or groundedness is at least 0.95 on final cases.
+- Answer relevance is at least 0.90 on final cases.
+- Answer correctness is at least 0.85 on human-labeled final cases.
 - Citation support is at least 0.95 for claims that cite sources.
 - Unsupported-answer rate on unanswerable questions is at most 0.05.
 
@@ -101,8 +104,8 @@ Ship criteria:
 ### Phase 4: Production Loop
 
 Run the same eval pack on every retrieval or prompt change.
-Keep train/dev/holdout isolated.
-Never tune on holdout.
+Keep train, selection, and final data isolated.
+Never tune on final data.
 
 Ship criteria:
 

package.json

@@ -72,7 +72,7 @@
     "verify:package": "pnpm run check:skills && node scripts/verify-package.mjs"
   },
   "dependencies": {
-    "@tangle-network/agent-eval": "^0.122.8",
+    "@tangle-network/agent-eval": "^0.123.8",
     "@tangle-network/agent-interface": "^0.31.0",
     "proper-lockfile": "4.1.2",
     "zod": "^4.4.3"

src/index.ts

@@ -31,15 +31,18 @@ export {
   inspectPendingKnowledgeMutation,
   recoverPendingKnowledgeMutation,
 } from './mutation-lock'
+export * from './optimization'
 export * from './proposals'
 export * from './propose-from-finding'
 export * from './rag-eval'
 export * from './rag-improvement-loop'
+export * from './rag-optimization'
 export * from './readiness-check'
 export * from './release'
 export * from './research-driving-driver'
 export * from './research-loop'
 export * from './retrieval-eval'
+export * from './retrieval-optimization'
 export * from './schemas'
 export * from './search'
 export * from './sources'

src/kb-improvement.ts

@@ -11,6 +11,8 @@ export type {
   KnowledgeImprovementMutationReceipt,
   KnowledgeImprovementMutationResult,
   KnowledgeImprovementOptions,
+  KnowledgeImprovementRagOptimizationOptions,
+  KnowledgeImprovementRagOptimizationRunInput,
   KnowledgeImprovementResult,
   KnowledgeImprovementRetrievalOptions,
   KnowledgeImprovementRunState,
@@ -31,6 +33,12 @@ export {
   KnowledgeImprovementEvidenceSchema,
   KnowledgeImprovementRunStateSchema,
 } from './kb-improvement/contracts'
+export type {
+  KnowledgePolicyDispatch,
+  OptimizeKnowledgeBasePolicyOptions,
+  OptimizeKnowledgeBasePolicyResult,
+} from './kb-improvement/optimization'
+export { optimizeKnowledgeBasePolicy } from './kb-improvement/optimization'
 export { improveKnowledgeBase } from './kb-improvement/run'
 export type { KnowledgeImprovementEvent } from './kb-improvement/state'
 export {

src/kb-improvement/contracts.ts

@@ -10,7 +10,11 @@ import type {
   EvalKnowledgeBundleBuildResult,
   KnowledgeReadinessSpec,
 } from '../eval-readiness'
-import type { KnowledgeBaseQualityOptions, KnowledgeBaseQualityReport } from '../rag-eval'
+import type {
+  KnowledgeBaseQualityOptions,
+  KnowledgeBaseQualityReport,
+  RagAnswerEvalArtifact,
+} from '../rag-eval'
 import type {
   RagKnowledgeImprovementPhase,
   RagKnowledgeResearchOptions,
@@ -19,8 +23,9 @@ import type {
   RunRagKnowledgeImprovementLoopOptions,
   RunRagKnowledgeImprovementLoopResult,
 } from '../rag-improvement-loop'
+import type { RunRagOptimizationOptions } from '../rag-optimization'
 import type { RunKnowledgeResearchLoopOptions } from '../research-loop'
-import type { RunRetrievalImprovementLoopOptions } from '../retrieval-eval'
+import type { RunRetrievalImprovementLoopOptions } from '../retrieval-optimization'
 import type { KnowledgeIndex } from '../types'
 import type { ValidateKnowledgeOptions, ValidateKnowledgeResult } from '../validate'
 
@@ -455,6 +460,25 @@ export interface KnowledgeImprovementRetrievalOptions
   runDir?: RunRetrievalImprovementLoopOptions['runDir']
 }
 
+export type KnowledgeImprovementRagOptimizationRunInput = Parameters<
+  RunRagOptimizationOptions['run']
+>[0] & {
+  runId: string
+  iteration: number
+  candidateId: string
+  root: string
+  baselineRoot: string
+  candidateRoot: string
+  candidateIndex: KnowledgeIndex
+  baseHash: string
+}
+
+export interface KnowledgeImprovementRagOptimizationOptions
+  extends Omit<RunRagOptimizationOptions, 'run' | 'runDir'> {
+  runDir?: RunRagOptimizationOptions['runDir']
+  run(input: KnowledgeImprovementRagOptimizationRunInput): Promise<RagAnswerEvalArtifact>
+}
+
 export interface KnowledgeImprovementUpdateInput extends RagKnowledgeUpdateInput {
   runId: string
   iteration: number
@@ -485,6 +509,7 @@ export interface KnowledgeImprovementOptions {
   kbQuality?: KnowledgeBaseQualityOptions
   step?: RunKnowledgeResearchLoopOptions['step']
   knowledgeResearch?: Omit<RagKnowledgeResearchOptions, 'root'>
+  ragOptimization?: KnowledgeImprovementRagOptimizationOptions
   retrieval?: KnowledgeImprovementRetrievalOptions
   diagnose?: NonNullable<RunRagKnowledgeImprovementLoopOptions['diagnose']>
   acquireKnowledge?: NonNullable<RunRagKnowledgeImprovementLoopOptions['acquireKnowledge']>
@@ -513,6 +538,7 @@ export const UPDATE_PHASES: readonly RagKnowledgeImprovementPhase[] = [
 ]
 
 export const EVALUATION_PHASES: readonly RagKnowledgeImprovementPhase[] = [
+  'rag-optimization',
   'retrieval-tuning',
   'gap-diagnosis',
   'answer-quality',