fix(optimization): require explicit retrieval methods

CHANGELOG.md

@@ -4,15 +4,15 @@
 
 ### Breaking Changes
 
-- Retrieval improvement now requires independent train, selection, and final scenarios plus either a complete `OptimizationMethod` or a bounded finite configuration space.
+- Retrieval improvement now requires independent train, selection, and final scenarios plus an explicit complete `OptimizationMethod`.
 - Memory configuration improvement now requires a baseline configuration, a complete `OptimizationMethod`, and independent train, selection, and final histories.
 - Removed the public retrieval and memory proposer-search options; candidate generation and selection now belong to `agent-eval` methods.
 
 ### Added
 
 - Added a shared serialized-candidate adapter for running complete `agent-eval` optimization methods with canonical candidate identity and untouched final comparison.
 - Added full RAG configuration optimization and KB maintenance policy optimization.
-- Added bounded retrieval configuration enumeration for small finite spaces.
+- Added an explicit `OptimizationMethod` factory for bounded retrieval configuration enumeration over small finite spaces.
 
 ### Changed
 

README.md

@@ -155,7 +155,7 @@ Use the narrowest API that matches the job:
 | API | What it does |
 |---|---|
 | `runRetrievalImprovementLoop` | Runs one complete `OptimizationMethod` over serialized retrieval configuration. |
-| `boundedRetrievalConfigMethod` | Enumerates a small finite retrieval grid, limited to 128 configurations by default. |
+| `boundedRetrievalConfigMethod` | Builds a complete method that enumerates a small finite retrieval grid, limited to 128 configurations by default. |
 | `runRagOptimization` | Optimizes retrieval and answer behavior as one serialized RAG configuration. |
 | `optimizeKnowledgeBasePolicy` | Optimizes a KB maintenance policy, then applies only the selected policy to an isolated candidate. |
 | `scoreKnowledgeBaseIndex` | Measures KB structure, citations, source freshness, and configured quality thresholds. |
@@ -164,9 +164,13 @@ Use the narrowest API that matches the job:
 | `improveKnowledgeBase` | Adds resumable state, isolated candidates, exact promotion, and conflict detection around that process. |
 
 ```ts
+const method = boundedRetrievalConfigMethod({
+  searchSpace: { k: [5, 10], reranker: [false, true] },
+})
+
 const result = await runRetrievalImprovementLoop({
   baseline: { k: 5, reranker: false },
-  method, // Any OptimizationMethod that returns this serialized JSON surface.
+  method,
   trainScenarios,
   selectionScenarios,
   finalScenarios,
@@ -178,6 +182,8 @@ const result = await runRetrievalImprovementLoop({
 
 The method receives train and selection cases.
 `agent-eval` keeps final cases out of the search and measures the exact selected configuration on them afterward.
+Every optimization call requires an explicit complete method.
+Pass an applicable official method, a custom method, or `boundedRetrievalConfigMethod()` for a small finite retrieval grid.
 Reuse the run directory only with the method's compatible resume mode.
 Use separate run directories to explore branches in parallel.
 Optimizer-specific identity and resume settings stay on the supplied method; this package does not reinterpret them.

docs/eval/rag-eval-roadmap.md

@@ -21,7 +21,7 @@ Done:
 
 - `runRetrievalImprovementLoop()` runs a complete `agent-eval` optimization method over retrieval configs.
 - `runRagOptimization()` does the same for a serialized retrieval and answer configuration.
-- `boundedRetrievalConfigMethod()` remains available for finite retrieval grids of at most 128 configurations by default.
+- `boundedRetrievalConfigMethod()` builds an explicitly supplied method for finite retrieval grids of at most 128 configurations by default.
 - `runRagKnowledgeImprovementLoop()` exposes the whole RAG lifecycle as typed phases:
   retrieval tuning, gap diagnosis, knowledge acquisition, knowledge update, answer-quality eval, and promotion.
 - Retrieval scenarios can label pages, page paths, sources, source anchors, and source spans.

src/optimization.ts

@@ -72,6 +72,7 @@ export async function runSerializedKnowledgeOptimization<
 >(
   options: RunSerializedKnowledgeOptimizationOptions<TCandidate, TScenario, TArtifact>,
 ): Promise<RunSerializedKnowledgeOptimizationResult<TCandidate>> {
+  assertCompleteOptimizationMethod(options.method)
   const codec = options.codec ?? jsonCandidateCodec<TCandidate>()
   const baseline = normalizeCandidate(options.baseline, codec, 'baseline')
   assertPartitionContent(
@@ -159,6 +160,20 @@ export async function runSerializedKnowledgeOptimization<
   }
 }
 
+function assertCompleteOptimizationMethod(method: unknown): void {
+  const candidate = method as { name?: unknown; optimize?: unknown } | null | undefined
+  if (
+    !candidate ||
+    typeof candidate.name !== 'string' ||
+    candidate.name.trim().length === 0 ||
+    typeof candidate.optimize !== 'function'
+  ) {
+    throw new Error(
+      'knowledge optimization requires a complete OptimizationMethod with a non-empty name and optimize()',
+    )
+  }
+}
+
 export function jsonCandidateCodec<
   TCandidate extends JsonValue,
 >(): SerializedCandidateCodec<TCandidate> {

src/retrieval-optimization.ts

@@ -71,13 +71,10 @@ export interface RunRetrievalImprovementLoopOptions extends RetrievalOptimizatio
   trainScenarios: readonly RetrievalEvalScenario[]
   selectionScenarios: readonly RetrievalEvalScenario[]
   finalScenarios: readonly RetrievalEvalScenario[]
-  method?: OptimizationMethod<RetrievalEvalScenario, RetrievalEvalArtifact>
+  method: OptimizationMethod<RetrievalEvalScenario, RetrievalEvalArtifact>
   index?: KnowledgeIndex
   defaultK?: number
   retrieve?: RetrievalEvalRetriever
-  configurations?: readonly RetrievalConfig[]
-  searchSpace?: RetrievalParameterSearchSpace
-  boundedSearch?: Omit<BoundedRetrievalConfigMethodOptions, 'configurations' | 'searchSpace'>
   judges?: readonly JudgeConfig<RetrievalEvalArtifact, RetrievalEvalScenario>[]
   metricWeights?: RetrievalMetricWeights
 }
@@ -89,7 +86,6 @@ export interface RunRetrievalImprovementLoopResult
   trainScenarios: readonly RetrievalEvalScenario[]
   selectionScenarios: readonly RetrievalEvalScenario[]
   finalScenarios: readonly RetrievalEvalScenario[]
-  boundedConfigurations?: readonly RetrievalConfig[]
 }
 
 export function buildBoundedRetrievalConfigs(
@@ -135,7 +131,8 @@ export function buildBoundedRetrievalConfigs(
 
 /**
  * Exhaustively checks a small, finite retrieval grid through agent-eval.
- * Larger or generative spaces should supply an official OptimizationMethod.
+ * Pass the returned complete method explicitly to runRetrievalImprovementLoop().
+ * Larger or generative spaces should use an official OptimizationMethod.
  */
 export function boundedRetrievalConfigMethod(
   options: BoundedRetrievalConfigMethodOptions,
@@ -146,7 +143,7 @@ export function boundedRetrievalConfigMethod(
   if (!options.configurations && !options.searchSpace) {
     throw new Error('bounded retrieval method requires configurations or searchSpace')
   }
-  const name = options.name ?? 'bounded-retrieval-config-search'
+  const name = options.name ?? 'bounded-retrieval-config'
   return {
     name,
     async optimize(input) {
@@ -219,20 +216,6 @@ export function boundedRetrievalConfigMethod(
 export async function runRetrievalImprovementLoop(
   options: RunRetrievalImprovementLoopOptions,
 ): Promise<RunRetrievalImprovementLoopResult> {
-  if (options.method && (options.configurations || options.searchSpace)) {
-    throw new Error(
-      'runRetrievalImprovementLoop accepts method or bounded configurations, not both',
-    )
-  }
-  const boundedConfigurations = options.method ? undefined : resolveBoundedConfigurations(options)
-  const method =
-    options.method ??
-    boundedRetrievalConfigMethod({
-      ...(options.boundedSearch ?? {}),
-      ...(options.configurations
-        ? { configurations: options.configurations }
-        : { searchSpace: options.searchSpace! }),
-    })
   const dispatch = buildRetrievalEvalDispatch({
     index: options.index,
     defaultK: options.defaultK,
@@ -243,13 +226,10 @@ export async function runRetrievalImprovementLoop(
     trainScenarios,
     selectionScenarios,
     finalScenarios,
-    method: _method,
+    method,
     index: _index,
     defaultK: _defaultK,
     retrieve: _retrieve,
-    configurations: _configurations,
-    searchSpace: _searchSpace,
-    boundedSearch: _boundedSearch,
     judges,
     metricWeights,
     ...runOptions
@@ -274,30 +254,7 @@ export async function runRetrievalImprovementLoop(
     trainScenarios: [...trainScenarios],
     selectionScenarios: [...selectionScenarios],
     finalScenarios: [...finalScenarios],
-    ...(boundedConfigurations ? { boundedConfigurations } : {}),
-  }
-}
-
-function resolveBoundedConfigurations(
-  options: RunRetrievalImprovementLoopOptions,
-): RetrievalConfig[] {
-  if (options.configurations && options.searchSpace) {
-    throw new Error('runRetrievalImprovementLoop accepts configurations or searchSpace, not both')
-  }
-  if (options.configurations) {
-    return normalizeBoundedConfigurations(
-      options.configurations,
-      options.baseline,
-      options.boundedSearch?.maxConfigurations ?? 128,
-    )
-  }
-  if (options.searchSpace) {
-    return buildBoundedRetrievalConfigs(options.searchSpace, {
-      baseline: options.baseline,
-      maxConfigurations: options.boundedSearch?.maxConfigurations,
-    })
   }
-  throw new Error('runRetrievalImprovementLoop requires method, configurations, or searchSpace')
 }
 
 interface BoundedRetrievalMeasurement {

tests/kb-improvement/candidate.test.ts

@@ -2,6 +2,7 @@ import { mkdir, readFile, stat, writeFile } from 'node:fs/promises'
 import { dirname, join } from 'node:path'
 import { describe, expect, it } from 'vitest'
 import {
+  boundedRetrievalConfigMethod,
   buildEvalKnowledgeBundle,
   buildKnowledgeIndex,
   evaluateKnowledgeBaseReadiness,
@@ -365,7 +366,11 @@ describe('improveKnowledgeBase', () => {
               expected: { kind: 'page', pageId: 'refund-policy' },
             },
           ],
-          searchSpace: { k: [1, 2] },
+          method: boundedRetrievalConfigMethod({
+            searchSpace: { k: [1, 2] },
+            targetRecall: 1,
+            configurationConcurrency: 1,
+          }),
           retrieve: async ({ k }) => ({
             hits: [
               { pageId: 'distractor', path: 'knowledge/distractor.md', rank: 1 },
@@ -374,7 +379,6 @@ describe('improveKnowledgeBase', () => {
                 : []),
             ],
           }),
-          boundedSearch: { targetRecall: 1, configurationConcurrency: 1 },
           expectUsage: 'off',
           resamples: 200,
         },

tests/rag-improvement-loop.test.ts

@@ -7,6 +7,7 @@ import {
 } from '@tangle-network/agent-eval/campaign'
 import { afterEach, describe, expect, it } from 'vitest'
 import {
+  boundedRetrievalConfigMethod,
   type RagAnswerEvalArtifact,
   type RagAnswerEvalScenario,
   type RetrievalEvalScenario,
@@ -127,14 +128,17 @@ describe('RAG knowledge improvement loop', () => {
           makeScenario('q-selection-c'),
         ],
         finalScenarios: [makeScenario('q-final-a'), makeScenario('q-final-b')],
-        searchSpace: { k: [1, 2] },
+        method: boundedRetrievalConfigMethod({
+          searchSpace: { k: [1, 2] },
+          targetRecall: 1,
+          configurationConcurrency: 1,
+        }),
         retrieve: async ({ k }) => ({
           hits: [
             { pageId: 'distractor', path: 'knowledge/distractor.md', rank: 1 },
             ...(k >= 2 ? [{ pageId: 'gold', path: 'knowledge/gold.md', rank: 2 }] : []),
           ],
         }),
-        boundedSearch: { targetRecall: 1, configurationConcurrency: 1 },
         runDir: 'memory://rag-lifecycle-retrieval-test',
         storage: inMemoryCampaignStorage(),
         expectUsage: 'off',

tests/retrieval-eval.test.ts

@@ -5,6 +5,7 @@ import {
 } from '@tangle-network/agent-eval/campaign'
 import { describe, expect, it } from 'vitest'
 import {
+  boundedRetrievalConfigMethod,
   buildBoundedRetrievalConfigs,
   buildRetrievalEvalDispatch,
   type KnowledgeIndex,
@@ -256,6 +257,26 @@ describe('retrieval eval', () => {
     ])
   })
 
+  it('requires the caller to supply a complete OptimizationMethod', async () => {
+    const options = {
+      baseline: { k: 1 },
+      trainScenarios: [retrievalScenario('missing-method-train', 'train query')],
+      selectionScenarios: [retrievalScenario('missing-method-selection', 'selection query')],
+      finalScenarios: [
+        retrievalScenario('missing-method-final-a', 'final query a'),
+        retrievalScenario('missing-method-final-b', 'final query b'),
+      ],
+      retrieve: retrievalFixture,
+      runDir: '/runs/retrieval-missing-method-test',
+      storage: inMemoryCampaignStorage(),
+      expectUsage: 'off',
+    } as unknown as Parameters<typeof runRetrievalImprovementLoop>[0]
+
+    await expect(runRetrievalImprovementLoop(options)).rejects.toThrow(
+      'knowledge optimization requires a complete OptimizationMethod',
+    )
+  })
+
   it('rejects renamed duplicate scenarios before starting the method', async () => {
     let methodCalled = false
     const method: OptimizationMethod<RetrievalEvalScenario, RetrievalEvalArtifact> = {
@@ -340,11 +361,15 @@ describe('retrieval eval', () => {
       retrievalScenario('final-a', 'final query a'),
       retrievalScenario('final-b', 'final query b'),
     ]
+    const method = boundedRetrievalConfigMethod({
+      configurations: [{ k: 2 }, { k: 3 }],
+      configurationConcurrency: 1,
+      targetRecall: 1,
+    })
     const run = () =>
       runRetrievalImprovementLoop({
         baseline: { k: 1 },
-        configurations: [{ k: 2 }, { k: 3 }],
-        boundedSearch: { configurationConcurrency: 1, targetRecall: 1 },
+        method,
         trainScenarios,
         selectionScenarios,
         finalScenarios,
@@ -360,7 +385,7 @@ describe('retrieval eval', () => {
     const result = await run()
 
     expect(result.winnerConfig).toMatchObject({ k: 2 })
-    expect(result.boundedConfigurations).toEqual([{ k: 2 }, { k: 3 }])
+    expect(result.methodName).toBe('bounded-retrieval-config')
     expect(retrievedK).not.toContain(3)
     expect(result.trainScenarios).toHaveLength(1)
     expect(result.selectionScenarios).toHaveLength(3)