From 88763bf38924740f2ad696ffad929a1790ddd0ac Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 23 Jul 2026 19:46:08 -0600 Subject: [PATCH] fix(optimization): require explicit retrieval methods --- CHANGELOG.md | 4 +- README.md | 10 ++++- docs/eval/rag-eval-roadmap.md | 2 +- src/optimization.ts | 15 ++++++++ src/retrieval-optimization.ts | 53 +++----------------------- tests/kb-improvement/candidate.test.ts | 8 +++- tests/rag-improvement-loop.test.ts | 8 +++- tests/retrieval-eval.test.ts | 31 +++++++++++++-- 8 files changed, 71 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb3c120..f6f1afa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Breaking Changes -- Retrieval improvement now requires independent train, selection, and final scenarios plus either a complete `OptimizationMethod` or a bounded finite configuration space. +- Retrieval improvement now requires independent train, selection, and final scenarios plus an explicit complete `OptimizationMethod`. - Memory configuration improvement now requires a baseline configuration, a complete `OptimizationMethod`, and independent train, selection, and final histories. - Removed the public retrieval and memory proposer-search options; candidate generation and selection now belong to `agent-eval` methods. @@ -12,7 +12,7 @@ - Added a shared serialized-candidate adapter for running complete `agent-eval` optimization methods with canonical candidate identity and untouched final comparison. - Added full RAG configuration optimization and KB maintenance policy optimization. -- Added bounded retrieval configuration enumeration for small finite spaces. +- Added an explicit `OptimizationMethod` factory for bounded retrieval configuration enumeration over small finite spaces. ### Changed diff --git a/README.md b/README.md index 13f46cb..0b5c6c8 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ Use the narrowest API that matches the job: | API | What it does | |---|---| | `runRetrievalImprovementLoop` | Runs one complete `OptimizationMethod` over serialized retrieval configuration. | -| `boundedRetrievalConfigMethod` | Enumerates a small finite retrieval grid, limited to 128 configurations by default. | +| `boundedRetrievalConfigMethod` | Builds a complete method that enumerates a small finite retrieval grid, limited to 128 configurations by default. | | `runRagOptimization` | Optimizes retrieval and answer behavior as one serialized RAG configuration. | | `optimizeKnowledgeBasePolicy` | Optimizes a KB maintenance policy, then applies only the selected policy to an isolated candidate. | | `scoreKnowledgeBaseIndex` | Measures KB structure, citations, source freshness, and configured quality thresholds. | @@ -164,9 +164,13 @@ Use the narrowest API that matches the job: | `improveKnowledgeBase` | Adds resumable state, isolated candidates, exact promotion, and conflict detection around that process. | ```ts +const method = boundedRetrievalConfigMethod({ + searchSpace: { k: [5, 10], reranker: [false, true] }, +}) + const result = await runRetrievalImprovementLoop({ baseline: { k: 5, reranker: false }, - method, // Any OptimizationMethod that returns this serialized JSON surface. + method, trainScenarios, selectionScenarios, finalScenarios, @@ -178,6 +182,8 @@ const result = await runRetrievalImprovementLoop({ The method receives train and selection cases. `agent-eval` keeps final cases out of the search and measures the exact selected configuration on them afterward. +Every optimization call requires an explicit complete method. +Pass an applicable official method, a custom method, or `boundedRetrievalConfigMethod()` for a small finite retrieval grid. Reuse the run directory only with the method's compatible resume mode. Use separate run directories to explore branches in parallel. Optimizer-specific identity and resume settings stay on the supplied method; this package does not reinterpret them. diff --git a/docs/eval/rag-eval-roadmap.md b/docs/eval/rag-eval-roadmap.md index 247b551..0ff5a7e 100644 --- a/docs/eval/rag-eval-roadmap.md +++ b/docs/eval/rag-eval-roadmap.md @@ -21,7 +21,7 @@ Done: - `runRetrievalImprovementLoop()` runs a complete `agent-eval` optimization method over retrieval configs. - `runRagOptimization()` does the same for a serialized retrieval and answer configuration. -- `boundedRetrievalConfigMethod()` remains available for finite retrieval grids of at most 128 configurations by default. +- `boundedRetrievalConfigMethod()` builds an explicitly supplied method for finite retrieval grids of at most 128 configurations by default. - `runRagKnowledgeImprovementLoop()` exposes the whole RAG lifecycle as typed phases: retrieval tuning, gap diagnosis, knowledge acquisition, knowledge update, answer-quality eval, and promotion. - Retrieval scenarios can label pages, page paths, sources, source anchors, and source spans. diff --git a/src/optimization.ts b/src/optimization.ts index 74890e3..614b0f2 100644 --- a/src/optimization.ts +++ b/src/optimization.ts @@ -72,6 +72,7 @@ export async function runSerializedKnowledgeOptimization< >( options: RunSerializedKnowledgeOptimizationOptions, ): Promise> { + assertCompleteOptimizationMethod(options.method) const codec = options.codec ?? jsonCandidateCodec() const baseline = normalizeCandidate(options.baseline, codec, 'baseline') assertPartitionContent( @@ -159,6 +160,20 @@ export async function runSerializedKnowledgeOptimization< } } +function assertCompleteOptimizationMethod(method: unknown): void { + const candidate = method as { name?: unknown; optimize?: unknown } | null | undefined + if ( + !candidate || + typeof candidate.name !== 'string' || + candidate.name.trim().length === 0 || + typeof candidate.optimize !== 'function' + ) { + throw new Error( + 'knowledge optimization requires a complete OptimizationMethod with a non-empty name and optimize()', + ) + } +} + export function jsonCandidateCodec< TCandidate extends JsonValue, >(): SerializedCandidateCodec { diff --git a/src/retrieval-optimization.ts b/src/retrieval-optimization.ts index 806531d..053bfd7 100644 --- a/src/retrieval-optimization.ts +++ b/src/retrieval-optimization.ts @@ -71,13 +71,10 @@ export interface RunRetrievalImprovementLoopOptions extends RetrievalOptimizatio trainScenarios: readonly RetrievalEvalScenario[] selectionScenarios: readonly RetrievalEvalScenario[] finalScenarios: readonly RetrievalEvalScenario[] - method?: OptimizationMethod + method: OptimizationMethod index?: KnowledgeIndex defaultK?: number retrieve?: RetrievalEvalRetriever - configurations?: readonly RetrievalConfig[] - searchSpace?: RetrievalParameterSearchSpace - boundedSearch?: Omit judges?: readonly JudgeConfig[] metricWeights?: RetrievalMetricWeights } @@ -89,7 +86,6 @@ export interface RunRetrievalImprovementLoopResult trainScenarios: readonly RetrievalEvalScenario[] selectionScenarios: readonly RetrievalEvalScenario[] finalScenarios: readonly RetrievalEvalScenario[] - boundedConfigurations?: readonly RetrievalConfig[] } export function buildBoundedRetrievalConfigs( @@ -135,7 +131,8 @@ export function buildBoundedRetrievalConfigs( /** * Exhaustively checks a small, finite retrieval grid through agent-eval. - * Larger or generative spaces should supply an official OptimizationMethod. + * Pass the returned complete method explicitly to runRetrievalImprovementLoop(). + * Larger or generative spaces should use an official OptimizationMethod. */ export function boundedRetrievalConfigMethod( options: BoundedRetrievalConfigMethodOptions, @@ -146,7 +143,7 @@ export function boundedRetrievalConfigMethod( if (!options.configurations && !options.searchSpace) { throw new Error('bounded retrieval method requires configurations or searchSpace') } - const name = options.name ?? 'bounded-retrieval-config-search' + const name = options.name ?? 'bounded-retrieval-config' return { name, async optimize(input) { @@ -219,20 +216,6 @@ export function boundedRetrievalConfigMethod( export async function runRetrievalImprovementLoop( options: RunRetrievalImprovementLoopOptions, ): Promise { - if (options.method && (options.configurations || options.searchSpace)) { - throw new Error( - 'runRetrievalImprovementLoop accepts method or bounded configurations, not both', - ) - } - const boundedConfigurations = options.method ? undefined : resolveBoundedConfigurations(options) - const method = - options.method ?? - boundedRetrievalConfigMethod({ - ...(options.boundedSearch ?? {}), - ...(options.configurations - ? { configurations: options.configurations } - : { searchSpace: options.searchSpace! }), - }) const dispatch = buildRetrievalEvalDispatch({ index: options.index, defaultK: options.defaultK, @@ -243,13 +226,10 @@ export async function runRetrievalImprovementLoop( trainScenarios, selectionScenarios, finalScenarios, - method: _method, + method, index: _index, defaultK: _defaultK, retrieve: _retrieve, - configurations: _configurations, - searchSpace: _searchSpace, - boundedSearch: _boundedSearch, judges, metricWeights, ...runOptions @@ -274,30 +254,7 @@ export async function runRetrievalImprovementLoop( trainScenarios: [...trainScenarios], selectionScenarios: [...selectionScenarios], finalScenarios: [...finalScenarios], - ...(boundedConfigurations ? { boundedConfigurations } : {}), - } -} - -function resolveBoundedConfigurations( - options: RunRetrievalImprovementLoopOptions, -): RetrievalConfig[] { - if (options.configurations && options.searchSpace) { - throw new Error('runRetrievalImprovementLoop accepts configurations or searchSpace, not both') - } - if (options.configurations) { - return normalizeBoundedConfigurations( - options.configurations, - options.baseline, - options.boundedSearch?.maxConfigurations ?? 128, - ) - } - if (options.searchSpace) { - return buildBoundedRetrievalConfigs(options.searchSpace, { - baseline: options.baseline, - maxConfigurations: options.boundedSearch?.maxConfigurations, - }) } - throw new Error('runRetrievalImprovementLoop requires method, configurations, or searchSpace') } interface BoundedRetrievalMeasurement { diff --git a/tests/kb-improvement/candidate.test.ts b/tests/kb-improvement/candidate.test.ts index 4b2a013..a5504a2 100644 --- a/tests/kb-improvement/candidate.test.ts +++ b/tests/kb-improvement/candidate.test.ts @@ -2,6 +2,7 @@ import { mkdir, readFile, stat, writeFile } from 'node:fs/promises' import { dirname, join } from 'node:path' import { describe, expect, it } from 'vitest' import { + boundedRetrievalConfigMethod, buildEvalKnowledgeBundle, buildKnowledgeIndex, evaluateKnowledgeBaseReadiness, @@ -365,7 +366,11 @@ describe('improveKnowledgeBase', () => { expected: { kind: 'page', pageId: 'refund-policy' }, }, ], - searchSpace: { k: [1, 2] }, + method: boundedRetrievalConfigMethod({ + searchSpace: { k: [1, 2] }, + targetRecall: 1, + configurationConcurrency: 1, + }), retrieve: async ({ k }) => ({ hits: [ { pageId: 'distractor', path: 'knowledge/distractor.md', rank: 1 }, @@ -374,7 +379,6 @@ describe('improveKnowledgeBase', () => { : []), ], }), - boundedSearch: { targetRecall: 1, configurationConcurrency: 1 }, expectUsage: 'off', resamples: 200, }, diff --git a/tests/rag-improvement-loop.test.ts b/tests/rag-improvement-loop.test.ts index dbe3f9a..9102af2 100644 --- a/tests/rag-improvement-loop.test.ts +++ b/tests/rag-improvement-loop.test.ts @@ -7,6 +7,7 @@ import { } from '@tangle-network/agent-eval/campaign' import { afterEach, describe, expect, it } from 'vitest' import { + boundedRetrievalConfigMethod, type RagAnswerEvalArtifact, type RagAnswerEvalScenario, type RetrievalEvalScenario, @@ -127,14 +128,17 @@ describe('RAG knowledge improvement loop', () => { makeScenario('q-selection-c'), ], finalScenarios: [makeScenario('q-final-a'), makeScenario('q-final-b')], - searchSpace: { k: [1, 2] }, + method: boundedRetrievalConfigMethod({ + searchSpace: { k: [1, 2] }, + targetRecall: 1, + configurationConcurrency: 1, + }), retrieve: async ({ k }) => ({ hits: [ { pageId: 'distractor', path: 'knowledge/distractor.md', rank: 1 }, ...(k >= 2 ? [{ pageId: 'gold', path: 'knowledge/gold.md', rank: 2 }] : []), ], }), - boundedSearch: { targetRecall: 1, configurationConcurrency: 1 }, runDir: 'memory://rag-lifecycle-retrieval-test', storage: inMemoryCampaignStorage(), expectUsage: 'off', diff --git a/tests/retrieval-eval.test.ts b/tests/retrieval-eval.test.ts index 5ab3a21..8c7f188 100644 --- a/tests/retrieval-eval.test.ts +++ b/tests/retrieval-eval.test.ts @@ -5,6 +5,7 @@ import { } from '@tangle-network/agent-eval/campaign' import { describe, expect, it } from 'vitest' import { + boundedRetrievalConfigMethod, buildBoundedRetrievalConfigs, buildRetrievalEvalDispatch, type KnowledgeIndex, @@ -256,6 +257,26 @@ describe('retrieval eval', () => { ]) }) + it('requires the caller to supply a complete OptimizationMethod', async () => { + const options = { + baseline: { k: 1 }, + trainScenarios: [retrievalScenario('missing-method-train', 'train query')], + selectionScenarios: [retrievalScenario('missing-method-selection', 'selection query')], + finalScenarios: [ + retrievalScenario('missing-method-final-a', 'final query a'), + retrievalScenario('missing-method-final-b', 'final query b'), + ], + retrieve: retrievalFixture, + runDir: '/runs/retrieval-missing-method-test', + storage: inMemoryCampaignStorage(), + expectUsage: 'off', + } as unknown as Parameters[0] + + await expect(runRetrievalImprovementLoop(options)).rejects.toThrow( + 'knowledge optimization requires a complete OptimizationMethod', + ) + }) + it('rejects renamed duplicate scenarios before starting the method', async () => { let methodCalled = false const method: OptimizationMethod = { @@ -340,11 +361,15 @@ describe('retrieval eval', () => { retrievalScenario('final-a', 'final query a'), retrievalScenario('final-b', 'final query b'), ] + const method = boundedRetrievalConfigMethod({ + configurations: [{ k: 2 }, { k: 3 }], + configurationConcurrency: 1, + targetRecall: 1, + }) const run = () => runRetrievalImprovementLoop({ baseline: { k: 1 }, - configurations: [{ k: 2 }, { k: 3 }], - boundedSearch: { configurationConcurrency: 1, targetRecall: 1 }, + method, trainScenarios, selectionScenarios, finalScenarios, @@ -360,7 +385,7 @@ describe('retrieval eval', () => { const result = await run() expect(result.winnerConfig).toMatchObject({ k: 2 }) - expect(result.boundedConfigurations).toEqual([{ k: 2 }, { k: 3 }]) + expect(result.methodName).toBe('bounded-retrieval-config') expect(retrievedK).not.toContain(3) expect(result.trainScenarios).toHaveLength(1) expect(result.selectionScenarios).toHaveLength(3)