From 68c4de1769cabd465735f83cb31b8e60912c86e9 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 23 Jul 2026 19:31:56 -0600 Subject: [PATCH] feat(optimization): run knowledge searches through complete methods --- AGENTS.md | 4 +- CHANGELOG.md | 19 + README.md | 37 +- docs/eval/rag-eval-roadmap.md | 27 +- package.json | 2 +- pnpm-lock.yaml | 10 +- src/index.ts | 3 + src/kb-improvement.ts | 8 + src/kb-improvement/contracts.ts | 30 +- src/kb-improvement/evaluation.ts | 63 +- src/kb-improvement/optimization.ts | 176 ++++++ src/memory/attempt-log.ts | 4 +- src/memory/experiment/cases.ts | 9 +- src/memory/improvement.ts | 5 +- src/memory/improvement/activation.ts | 126 +++- src/memory/improvement/candidate.ts | 70 +- src/memory/improvement/evaluation.ts | 376 +++++++++++ src/memory/improvement/identity.ts | 53 +- src/memory/improvement/output.ts | 8 +- src/memory/improvement/promotion.ts | 197 +++--- src/memory/improvement/run.ts | 455 ++++--------- src/memory/improvement/types.ts | 97 ++- src/memory/improvement/validation.ts | 132 ++-- src/optimization.ts | 307 +++++++++ src/rag-improvement-loop.ts | 50 +- src/rag-optimization.ts | 102 +++ src/retrieval-eval.ts | 330 ++-------- src/retrieval-optimization.ts | 457 ++++++++++++++ tests/kb-improvement/candidate.test.ts | 40 +- tests/kb-improvement/optimization.test.ts | 268 ++++++++ tests/memory/improvement.test.ts | 736 +++++++--------------- tests/rag-improvement-loop.test.ts | 159 ++++- tests/retrieval-eval.test.ts | 241 +++++-- tests/support/memory.ts | 3 +- 34 files changed, 3047 insertions(+), 1557 deletions(-) create mode 100644 src/kb-improvement/optimization.ts create mode 100644 src/memory/improvement/evaluation.ts create mode 100644 src/optimization.ts create mode 100644 src/rag-optimization.ts create mode 100644 src/retrieval-optimization.ts create mode 100644 tests/kb-improvement/optimization.test.ts diff --git a/AGENTS.md b/AGENTS.md index 0f4be62..db4c707 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -75,7 +75,9 @@ The parser rejects absolute paths, `..`, control characters, and writes outside ## Eval Boundary -Compare candidate knowledge bases on an actual task corpus by running an `@tangle-network/agent-eval` improvement loop (`runImprovementLoop`) over the variants; each run is scored into a `RunRecord`. +Use a complete `OptimizationMethod` from `@tangle-network/agent-eval` with `runRetrievalImprovementLoop()`, `runRagOptimization()`, `optimizeKnowledgeBasePolicy()`, or `runAgentMemoryImprovement()`. +The method owns candidate search and resume compatibility. +This package owns serialized knowledge candidates, real KB or memory adapters, isolated data partitions, and safe activation. Use `knowledgeReleaseReport()` before promotion. It folds the candidate and baseline `RunRecord[]` (plus optional traces and the gate decision) into `agent-eval` release confidence evidence. diff --git a/CHANGELOG.md b/CHANGELOG.md index d28ba5e..eb3c120 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## Unreleased + +### Breaking Changes + +- Retrieval improvement now requires independent train, selection, and final scenarios plus either a complete `OptimizationMethod` or a bounded finite configuration space. +- Memory configuration improvement now requires a baseline configuration, a complete `OptimizationMethod`, and independent train, selection, and final histories. +- Removed the public retrieval and memory proposer-search options; candidate generation and selection now belong to `agent-eval` methods. + +### Added + +- Added a shared serialized-candidate adapter for running complete `agent-eval` optimization methods with canonical candidate identity and untouched final comparison. +- Added full RAG configuration optimization and KB maintenance policy optimization. +- Added bounded retrieval configuration enumeration for small finite spaces. + +### Changed + +- Updated `@tangle-network/agent-eval` to `0.123.8`. +- Kept memory provider evaluations resumable and branch-isolated while moving search ownership to the supplied method. + ## 4.1.0 ### Added diff --git a/README.md b/README.md index 29cb3a0..13f46cb 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,10 @@ Requires Node.js 20.19 or later. | Create a file-backed knowledge base | `initKnowledgeBase`, `addSourceText`, `applyKnowledgeWriteBlocks` | package root | | Search an existing package knowledge base | `createFileSystemSearchProvider` | package root | | Improve a live knowledge base without editing it in place | `improveKnowledgeBase` | package root | -| Tune retrieval against labeled questions | `runRetrievalImprovementLoop` | package root | +| Optimize retrieval or a complete RAG configuration | `runRetrievalImprovementLoop`, `runRagOptimization` | package root | +| Optimize a KB maintenance policy | `optimizeKnowledgeBasePolicy` | package root | | Run retrieval, research, answer checks, and promotion as one process | `runRagKnowledgeImprovementLoop` | package root | -| Compare or integrate agent memory systems | `AgentMemoryAdapter` and provider adapters | `/memory` | +| Compare providers or optimize memory configuration | `AgentMemoryAdapter`, `runAgentMemoryImprovement` | `/memory` | | Read from external authorities | `KnowledgeSource` and source adapters | `/sources` | | Run retrieval, answer, KB, or memory benchmark cases | `runKnowledgeBenchmarkSuite` | `/benchmarks` | | Use live research or coding agents | `runKnowledgeImprovementJob` | `@tangle-network/agent-runtime/knowledge` | @@ -153,12 +154,38 @@ Use the narrowest API that matches the job: | API | What it does | |---|---| -| `runRetrievalImprovementLoop` | Searches retrieval configurations against labeled train and holdout questions, with run and cost limits. | +| `runRetrievalImprovementLoop` | Runs one complete `OptimizationMethod` over serialized retrieval configuration. | +| `boundedRetrievalConfigMethod` | Enumerates a small finite retrieval grid, limited to 128 configurations by default. | +| `runRagOptimization` | Optimizes retrieval and answer behavior as one serialized RAG configuration. | +| `optimizeKnowledgeBasePolicy` | Optimizes a KB maintenance policy, then applies only the selected policy to an isolated candidate. | | `scoreKnowledgeBaseIndex` | Measures KB structure, citations, source freshness, and configured quality thresholds. | | `createRagAnswerQualityHook` | Adapts answer-quality checks such as support, relevance, citations, and abstention. | | `runRagKnowledgeImprovementLoop` | Connects retrieval tuning, gap diagnosis, source acquisition, KB updates, answer checks, and a promotion decision. | | `improveKnowledgeBase` | Adds resumable state, isolated candidates, exact promotion, and conflict detection around that process. | +```ts +const result = await runRetrievalImprovementLoop({ + baseline: { k: 5, reranker: false }, + method, // Any OptimizationMethod that returns this serialized JSON surface. + trainScenarios, + selectionScenarios, + finalScenarios, + retrieve: ({ scenario, config, k }) => search(scenario.query, { ...config, k }), + runDir: 'refund-retrieval-v1', + expectUsage: 'off', // Local search does not make a billable model call. +}) +``` + +The method receives train and selection cases. +`agent-eval` keeps final cases out of the search and measures the exact selected configuration on them afterward. +Reuse the run directory only with the method's compatible resume mode. +Use separate run directories to explore branches in parallel. +Optimizer-specific identity and resume settings stay on the supplied method; this package does not reinterpret them. +The supplied method owns `evaluationVersion`, engine or skill inputs, and resume compatibility. +`agent-eval` component surfaces stay outside this adapter; each knowledge candidate has one canonical serialized identity. +See the [`agent-eval` method guide](https://github.com/tangle-network/agent-eval/blob/main/docs/campaign-proposers.md) for official GEPA and Omni methods. +SkillOpt is skill-only; use it here only when the serialized candidate is itself a skill. + Retrieval and answer generation remain callbacks. This lets the same evaluation code work with local search, vector databases, hybrid search, rerankers, and hosted RAG services. @@ -168,8 +195,10 @@ This lets the same evaluation code work with local search, vector databases, hyb The package is not a memory database. Install the provider you use, create its client, and pass that client to the adapter. -The memory APIs support scoped reads and writes, isolated branches, ordered histories, holdout comparisons, and adapter experiments. +The memory APIs support scoped reads and writes, isolated branches, ordered histories, independent train, selection, and final comparisons, and adapter experiments. Use them to compare a provider against no memory or another provider on the same tasks before changing production behavior. +`runAgentMemoryImprovement` accepts a complete `OptimizationMethod`, evaluates each serialized configuration in an isolated provider branch, and activates only a final-data winner through compare-and-set. +Paid memory improvement defaults to zero-dollar optimization and final limits; set both limits and a per-evaluation maximum before enabling paid work. ## Run benchmarks diff --git a/docs/eval/rag-eval-roadmap.md b/docs/eval/rag-eval-roadmap.md index 81ad45c..247b551 100644 --- a/docs/eval/rag-eval-roadmap.md +++ b/docs/eval/rag-eval-roadmap.md @@ -1,6 +1,6 @@ # RAG Eval Completion Roadmap -Verdict: `runRetrievalImprovementLoop()` is the right first loop, but it is only the retrieval layer. +Verdict: use retrieval-only optimization when the retriever is the only changing component, and full RAG optimization when retrieval and answer behavior must move together. SOTA RAG evaluation requires retrieval quality, context quality, generated-answer quality, abstention behavior, robustness, and operating budgets. ## Research Basis @@ -19,12 +19,15 @@ SOTA RAG evaluation requires retrieval quality, context quality, generated-answe Done: -- `runRetrievalImprovementLoop()` auto-searches retrieval configs through `agent-eval`. +- `runRetrievalImprovementLoop()` runs a complete `agent-eval` optimization method over retrieval configs. +- `runRagOptimization()` does the same for a serialized retrieval and answer configuration. +- `boundedRetrievalConfigMethod()` remains available for finite retrieval grids of at most 128 configurations by default. - `runRagKnowledgeImprovementLoop()` exposes the whole RAG lifecycle as typed phases: retrieval tuning, gap diagnosis, knowledge acquisition, knowledge update, answer-quality eval, and promotion. - Retrieval scenarios can label pages, page paths, sources, source anchors, and source spans. -- The retrieval judge reports recall, MRR, nDCG, precision@k, cost, and held-out promotion. -- The loop is tested with a real `agent-eval` run where `{ k: 2 }` beats `{ k: 1 }`. +- The retrieval judge reports recall, MRR, nDCG, and precision@k; `agent-eval` reports cost separately. +- Selection and final data remain independent, and the optimization method never receives final cases. +- The integration is tested with complete methods for retrieval and full RAG configuration. - The lifecycle loop is tested both with pluggable phase hooks and with a real local KB update through `runKnowledgeResearchLoop()`. - `ragAnswerQualityJudge()` and `createRagAnswerQualityHook()` score context precision/recall/relevance/sufficiency, faithfulness, answer relevance/correctness, citation support, abstention, and unsupported-answer rate. - `normalizeExternalRagScores()` and the row exporters make Ragas, DeepEval, TruLens, RAGChecker, and custom evaluator outputs pluggable instead of hard dependencies. @@ -56,9 +59,9 @@ Required slices: Ship criteria: -- Holdout source-span Recall@5 is at least 0.90. -- Holdout nDCG@5 is at least 0.80. -- Train-to-holdout recall gap is at most 0.08. +- Final source-span Recall@5 is at least 0.90. +- Final nDCG@5 is at least 0.80. +- Train-to-final recall gap is at most 0.08. - Stale or forbidden source hit rate is at most 0.02. - p95 retrieval latency and cost do not regress by more than 10 percent versus baseline. @@ -69,9 +72,9 @@ Score it with deterministic checks first and LLM judges only for semantic qualit Ship criteria: -- Faithfulness or groundedness is at least 0.95 on holdout. -- Answer relevance is at least 0.90 on holdout. -- Answer correctness is at least 0.85 on human-labeled holdout. +- Faithfulness or groundedness is at least 0.95 on final cases. +- Answer relevance is at least 0.90 on final cases. +- Answer correctness is at least 0.85 on human-labeled final cases. - Citation support is at least 0.95 for claims that cite sources. - Unsupported-answer rate on unanswerable questions is at most 0.05. @@ -101,8 +104,8 @@ Ship criteria: ### Phase 4: Production Loop Run the same eval pack on every retrieval or prompt change. -Keep train/dev/holdout isolated. -Never tune on holdout. +Keep train, selection, and final data isolated. +Never tune on final data. Ship criteria: diff --git a/package.json b/package.json index 26b09b3..c8db7cf 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,7 @@ "verify:package": "pnpm run check:skills && node scripts/verify-package.mjs" }, "dependencies": { - "@tangle-network/agent-eval": "^0.122.8", + "@tangle-network/agent-eval": "^0.123.8", "@tangle-network/agent-interface": "^0.31.0", "proper-lockfile": "4.1.2", "zod": "^4.4.3" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 440f089..a437886 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,8 +13,8 @@ importers: .: dependencies: '@tangle-network/agent-eval': - specifier: ^0.122.8 - version: 0.122.8(typescript@5.9.3) + specifier: ^0.123.8 + version: 0.123.8(typescript@5.9.3) '@tangle-network/agent-interface': specifier: ^0.31.0 version: 0.31.0 @@ -968,8 +968,8 @@ packages: '@tangle-network/agent-core@0.4.11': resolution: {integrity: sha512-5B1IjrJ8xDR7w8Hv/MSk2ixul6NEJQ5Ftzo+z6l7ipeFpc0yaf1+Kkml4HDUEmGW/rqdrNpBf9/MpT7i/SY0PA==} - '@tangle-network/agent-eval@0.122.8': - resolution: {integrity: sha512-7v0us+6zpcR+VKqt9olG7C7j4KlTctvwhlHgm2qHU+FBorEJFcJs6JDqWMw+3h7t9ba/07muIb0AfuvSsmJJ8w==} + '@tangle-network/agent-eval@0.123.8': + resolution: {integrity: sha512-uWbtpUUy7cmoawrDcsNh1q9jEAulSOYC/N/qKG3BT+Eezz7bhKQQcLhT0S/m66kcyjXTrre5v2N23KVan8b9TA==} engines: {node: '>=20'} hasBin: true @@ -3232,7 +3232,7 @@ snapshots: '@tangle-network/agent-interface': 0.26.0 zod: 4.4.3 - '@tangle-network/agent-eval@0.122.8(typescript@5.9.3)': + '@tangle-network/agent-eval@0.123.8(typescript@5.9.3)': dependencies: '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) '@ax-llm/ax': 23.0.0(zod@4.4.3) diff --git a/src/index.ts b/src/index.ts index b1523f5..38e6d47 100644 --- a/src/index.ts +++ b/src/index.ts @@ -31,15 +31,18 @@ export { inspectPendingKnowledgeMutation, recoverPendingKnowledgeMutation, } from './mutation-lock' +export * from './optimization' export * from './proposals' export * from './propose-from-finding' export * from './rag-eval' export * from './rag-improvement-loop' +export * from './rag-optimization' export * from './readiness-check' export * from './release' export * from './research-driving-driver' export * from './research-loop' export * from './retrieval-eval' +export * from './retrieval-optimization' export * from './schemas' export * from './search' export * from './sources' diff --git a/src/kb-improvement.ts b/src/kb-improvement.ts index 9765193..a5fe46a 100644 --- a/src/kb-improvement.ts +++ b/src/kb-improvement.ts @@ -11,6 +11,8 @@ export type { KnowledgeImprovementMutationReceipt, KnowledgeImprovementMutationResult, KnowledgeImprovementOptions, + KnowledgeImprovementRagOptimizationOptions, + KnowledgeImprovementRagOptimizationRunInput, KnowledgeImprovementResult, KnowledgeImprovementRetrievalOptions, KnowledgeImprovementRunState, @@ -31,6 +33,12 @@ export { KnowledgeImprovementEvidenceSchema, KnowledgeImprovementRunStateSchema, } from './kb-improvement/contracts' +export type { + KnowledgePolicyDispatch, + OptimizeKnowledgeBasePolicyOptions, + OptimizeKnowledgeBasePolicyResult, +} from './kb-improvement/optimization' +export { optimizeKnowledgeBasePolicy } from './kb-improvement/optimization' export { improveKnowledgeBase } from './kb-improvement/run' export type { KnowledgeImprovementEvent } from './kb-improvement/state' export { diff --git a/src/kb-improvement/contracts.ts b/src/kb-improvement/contracts.ts index 4e8df08..2e122c7 100644 --- a/src/kb-improvement/contracts.ts +++ b/src/kb-improvement/contracts.ts @@ -10,7 +10,11 @@ import type { EvalKnowledgeBundleBuildResult, KnowledgeReadinessSpec, } from '../eval-readiness' -import type { KnowledgeBaseQualityOptions, KnowledgeBaseQualityReport } from '../rag-eval' +import type { + KnowledgeBaseQualityOptions, + KnowledgeBaseQualityReport, + RagAnswerEvalArtifact, +} from '../rag-eval' import type { RagKnowledgeImprovementPhase, RagKnowledgeResearchOptions, @@ -19,8 +23,9 @@ import type { RunRagKnowledgeImprovementLoopOptions, RunRagKnowledgeImprovementLoopResult, } from '../rag-improvement-loop' +import type { RunRagOptimizationOptions } from '../rag-optimization' import type { RunKnowledgeResearchLoopOptions } from '../research-loop' -import type { RunRetrievalImprovementLoopOptions } from '../retrieval-eval' +import type { RunRetrievalImprovementLoopOptions } from '../retrieval-optimization' import type { KnowledgeIndex } from '../types' import type { ValidateKnowledgeOptions, ValidateKnowledgeResult } from '../validate' @@ -455,6 +460,25 @@ export interface KnowledgeImprovementRetrievalOptions runDir?: RunRetrievalImprovementLoopOptions['runDir'] } +export type KnowledgeImprovementRagOptimizationRunInput = Parameters< + RunRagOptimizationOptions['run'] +>[0] & { + runId: string + iteration: number + candidateId: string + root: string + baselineRoot: string + candidateRoot: string + candidateIndex: KnowledgeIndex + baseHash: string +} + +export interface KnowledgeImprovementRagOptimizationOptions + extends Omit { + runDir?: RunRagOptimizationOptions['runDir'] + run(input: KnowledgeImprovementRagOptimizationRunInput): Promise +} + export interface KnowledgeImprovementUpdateInput extends RagKnowledgeUpdateInput { runId: string iteration: number @@ -485,6 +509,7 @@ export interface KnowledgeImprovementOptions { kbQuality?: KnowledgeBaseQualityOptions step?: RunKnowledgeResearchLoopOptions['step'] knowledgeResearch?: Omit + ragOptimization?: KnowledgeImprovementRagOptimizationOptions retrieval?: KnowledgeImprovementRetrievalOptions diagnose?: NonNullable acquireKnowledge?: NonNullable @@ -513,6 +538,7 @@ export const UPDATE_PHASES: readonly RagKnowledgeImprovementPhase[] = [ ] export const EVALUATION_PHASES: readonly RagKnowledgeImprovementPhase[] = [ + 'rag-optimization', 'retrieval-tuning', 'gap-diagnosis', 'answer-quality', diff --git a/src/kb-improvement/evaluation.ts b/src/kb-improvement/evaluation.ts index f4f816b..90e3593 100644 --- a/src/kb-improvement/evaluation.ts +++ b/src/kb-improvement/evaluation.ts @@ -94,6 +94,7 @@ export async function measureCandidate( } return withFrozenCandidateWorkspace(runDir, candidate, candidateRoot, async (snapshot) => { const evaluationLifecycle = await runCandidateEvaluationLifecycle( + runId, runDir, candidate, snapshot.root, @@ -138,6 +139,7 @@ async function runCandidateUpdateLifecycle( } async function runCandidateEvaluationLifecycle( + runId: string, runDir: string, candidate: KnowledgeImprovementCandidateRecord, candidateRoot: string, @@ -146,24 +148,45 @@ async function runCandidateEvaluationLifecycle( ): Promise { if (!shouldRunEvaluationStage(options)) return undefined const candidateIndex = await buildKnowledgeIndex(candidateRoot) - const lifecycle = await runRagKnowledgeImprovementLoop({ - goal: options.goal, - retrieval: options.retrieval - ? { - ...options.retrieval, - index: candidateIndex, - runDir: options.retrieval.runDir ?? join(runDir, 'retrieval', candidate.candidateId), - } - : undefined, - diagnose: options.diagnose, - evaluateAnswers: options.evaluateAnswers, - promote: options.decidePromotion, - enabledPhases: selectedStagePhases(options, EVALUATION_PHASES), - requiredPhases: selectedStageRequiredPhases(options, EVALUATION_PHASES), - signal: options.signal, - now, - }) - return lifecycle + return withBaselineSnapshot(runDir, candidate.baseHash, (baselineRoot) => + runRagKnowledgeImprovementLoop({ + goal: options.goal, + optimization: options.ragOptimization + ? { + ...options.ragOptimization, + runDir: + options.ragOptimization.runDir ?? + join(runDir, 'rag-optimization', candidate.candidateId), + run: (input) => + options.ragOptimization!.run({ + ...input, + runId, + iteration: candidate.iteration, + candidateId: candidate.candidateId, + root: candidateRoot, + baselineRoot, + candidateRoot, + candidateIndex, + baseHash: candidate.baseHash, + }), + } + : undefined, + retrieval: options.retrieval + ? { + ...options.retrieval, + index: candidateIndex, + runDir: options.retrieval.runDir ?? join(runDir, 'retrieval', candidate.candidateId), + } + : undefined, + diagnose: options.diagnose, + evaluateAnswers: options.evaluateAnswers, + promote: options.decidePromotion, + enabledPhases: selectedStagePhases(options, EVALUATION_PHASES), + requiredPhases: selectedStageRequiredPhases(options, EVALUATION_PHASES), + signal: options.signal, + now, + }), + ) } function candidateKnowledgeResearchOptions( @@ -222,7 +245,8 @@ function shouldRunEvaluationStage(options: KnowledgeImprovementOptions): boolean const phases = selectedStagePhases(options, EVALUATION_PHASES) if (phases.length === 0) return false return Boolean( - options.retrieval || + options.ragOptimization || + options.retrieval || options.diagnose || options.evaluateAnswers || options.decidePromotion || @@ -253,6 +277,7 @@ function mergeLifecycleResults( return { goal, phases: lifecycles.flatMap((lifecycle) => lifecycle.phases), + optimization: lastDefined(lifecycles.map((lifecycle) => lifecycle.optimization)), retrieval: lastDefined(lifecycles.map((lifecycle) => lifecycle.retrieval)), findings: lifecycles.flatMap((lifecycle) => lifecycle.findings), acquisition: lastDefined(lifecycles.map((lifecycle) => lifecycle.acquisition)), diff --git a/src/kb-improvement/optimization.ts b/src/kb-improvement/optimization.ts new file mode 100644 index 0000000..81bbc11 --- /dev/null +++ b/src/kb-improvement/optimization.ts @@ -0,0 +1,176 @@ +import type { + DispatchContext, + JsonValue, + OptimizationMethod, + Scenario, +} from '@tangle-network/agent-eval/campaign' +import { stableId } from '../ids' +import { + type RunSerializedKnowledgeOptimizationOptions, + type RunSerializedKnowledgeOptimizationResult, + runSerializedKnowledgeOptimization, +} from '../optimization' +import type { + KnowledgeImprovementOptions, + KnowledgeImprovementResult, + KnowledgeImprovementUpdateInput, +} from './contracts' +import { improveKnowledgeBase } from './run' +import { hashKnowledgeBase } from './workspace' + +type PolicyCandidateOptions = Omit< + KnowledgeImprovementOptions, + 'root' | 'goal' | 'runId' | 'maxCandidates' | 'step' | 'knowledgeResearch' | 'updateKnowledge' +> + +type PolicyOptimizationBaseOptions< + TPolicy extends JsonValue, + TScenario extends Scenario, + TArtifact, +> = Omit< + RunSerializedKnowledgeOptimizationOptions, + 'baseline' | 'method' | 'trainScenarios' | 'selectionScenarios' | 'finalScenarios' +> + +export interface OptimizeKnowledgeBasePolicyOptions< + TPolicy extends JsonValue, + TScenario extends Scenario, + TArtifact, +> extends PolicyOptimizationBaseOptions { + root: string + goal: string + baselinePolicy: TPolicy + method: OptimizationMethod + trainScenarios: readonly TScenario[] + selectionScenarios: readonly TScenario[] + finalScenarios: readonly TScenario[] + /** Stable version for applyPolicy and its external dependencies. */ + policyApplicationRef: string + /** Optional namespace for parallel materialization of the same measured policy. */ + candidateRunLabel?: string + candidate?: PolicyCandidateOptions + applyPolicy( + input: KnowledgeImprovementUpdateInput & { + policy: TPolicy + policySurface: string + policySurfaceHash: string + optimizationMethod: string + }, + ): Promise<{ + applied: boolean + summary: string + metadata?: Record + }> +} + +export interface OptimizeKnowledgeBasePolicyResult { + optimization: RunSerializedKnowledgeOptimizationResult + improvement: KnowledgeImprovementResult +} + +/** + * Optimizes a serialized KB-maintenance policy, then materializes the selected + * policy in one isolated knowledge candidate. Activation remains explicit. + */ +export async function optimizeKnowledgeBasePolicy< + TPolicy extends JsonValue, + TScenario extends Scenario, + TArtifact, +>( + options: OptimizeKnowledgeBasePolicyOptions, +): Promise> { + const { + root, + goal, + baselinePolicy, + method, + trainScenarios, + selectionScenarios, + finalScenarios, + policyApplicationRef, + candidateRunLabel, + candidate, + applyPolicy, + ...optimizationOptions + } = options + if (typeof root !== 'string' || !root.trim()) { + throw new Error('optimizeKnowledgeBasePolicy root must be non-empty') + } + if (typeof goal !== 'string' || !goal.trim()) { + throw new Error('optimizeKnowledgeBasePolicy goal must be non-empty') + } + if (typeof policyApplicationRef !== 'string' || !policyApplicationRef.trim()) { + throw new Error('optimizeKnowledgeBasePolicy policyApplicationRef must be non-empty') + } + if ( + candidateRunLabel !== undefined && + (typeof candidateRunLabel !== 'string' || !candidateRunLabel.trim()) + ) { + throw new Error('optimizeKnowledgeBasePolicy candidateRunLabel must be non-empty') + } + const baseHash = await hashKnowledgeBase(root) + const optimization = await runSerializedKnowledgeOptimization({ + ...optimizationOptions, + baseline: baselinePolicy, + method, + trainScenarios, + selectionScenarios, + finalScenarios, + }) + const winner = optimization.winner + const currentBaseHash = await hashKnowledgeBase(root) + if (currentBaseHash !== baseHash) { + throw new Error( + `knowledge base changed during policy optimization: expected ${baseHash}, got ${currentBaseHash}`, + ) + } + const runId = stableId( + 'kbpolicy', + `${candidateRunLabel ?? 'default'}:${goal}:${optimization.methodName}:${winner.surfaceHash}:${policyApplicationRef}:${baseHash}`, + ) + const improvement = await improveKnowledgeBase({ + ...(candidate ?? {}), + root, + goal, + runId, + maxCandidates: 1, + updateKnowledge: async (input) => { + if (input.baseHash !== baseHash) { + throw new Error( + `knowledge base changed before policy materialization: expected ${baseHash}, got ${input.baseHash}`, + ) + } + const result = await applyPolicy({ + ...input, + policy: structuredClone(winner.value), + policySurface: winner.surface, + policySurfaceHash: winner.surfaceHash, + optimizationMethod: optimization.methodName, + }) + return { + ...result, + metadata: { + ...(result.metadata ?? {}), + optimization: { + method: optimization.methodName, + policySurfaceHash: winner.surfaceHash, + policyApplicationRef, + }, + }, + } + }, + }) + return { optimization, improvement } +} + +export type KnowledgePolicyDispatch< + TPolicy extends JsonValue, + TScenario extends Scenario, + TArtifact, +> = (input: { + candidate: TPolicy + candidateSurface: string + candidateSurfaceHash: string + scenario: TScenario + context: DispatchContext +}) => Promise diff --git a/src/memory/attempt-log.ts b/src/memory/attempt-log.ts index 56f1f02..ea0c105 100644 --- a/src/memory/attempt-log.ts +++ b/src/memory/attempt-log.ts @@ -280,9 +280,7 @@ function isMissingPendingCostCall(error: unknown, callId: string): boolean { function requirePendingCostCallInspection(costLedger: CostLedgerHandle) { if (!costLedger.listPending) { - throw new Error( - 'interrupted memory recovery requires CostLedger.listPending() from @tangle-network/agent-eval 0.122.8 or newer', - ) + throw new Error('interrupted memory recovery requires CostLedger.listPending() support') } return costLedger.listPending() } diff --git a/src/memory/experiment/cases.ts b/src/memory/experiment/cases.ts index 941934e..75169b8 100644 --- a/src/memory/experiment/cases.ts +++ b/src/memory/experiment/cases.ts @@ -1,5 +1,5 @@ /** Converts existing ordered memory benchmark cases into executable histories. */ -import type { JudgeConfig } from '@tangle-network/agent-eval/campaign' +import type { JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign' import type { KnowledgeMemoryBenchmarkCase } from '../../benchmarks/index' import { stableId } from '../../ids' import type { @@ -103,10 +103,9 @@ export function buildAgentMemorySequenceScenarios( ) } -export function agentMemorySequenceJudge(): JudgeConfig< - AgentMemorySequenceArtifact, - AgentMemorySequenceScenario -> { +export function agentMemorySequenceJudge< + TScenario extends Scenario = AgentMemorySequenceScenario, +>(): JudgeConfig { return { name: 'agent-memory-sequence', judgeVersion: 'agent-knowledge:memory-sequence:v2', diff --git a/src/memory/improvement.ts b/src/memory/improvement.ts index 770648a..c72fde4 100644 --- a/src/memory/improvement.ts +++ b/src/memory/improvement.ts @@ -3,10 +3,11 @@ export type { AgentMemoryActivation, AgentMemoryActivationDriver, AgentMemoryDimensionComparison, - AgentMemoryGovernor, + AgentMemoryFinalEvaluation, + AgentMemoryFinalPair, AgentMemoryImprovementRunLease, - AgentMemoryImprovementSeed, AgentMemoryPromotionDecision, + MemoryConfigScenario, RunAgentMemoryImprovementOptions, RunAgentMemoryImprovementResult, } from './improvement/types' diff --git a/src/memory/improvement/activation.ts b/src/memory/improvement/activation.ts index 61124ae..e90411c 100644 --- a/src/memory/improvement/activation.ts +++ b/src/memory/improvement/activation.ts @@ -1,6 +1,18 @@ -import type { CampaignStorage } from '@tangle-network/agent-eval/campaign' +import { + type CampaignStorage, + type JsonValue, + surfaceHash, +} from '@tangle-network/agent-eval/campaign' import { appendDurableJournalEvent } from '../attempt-log' -import type { AgentMemoryActivationEvent, AgentMemoryActivationJournalState } from './types' +import { runBoundedMemoryLifecycle } from '../lifecycle' +import { memoryConfigCodec } from './evaluation' +import type { + AgentMemoryActivationEvent, + AgentMemoryActivationJournalState, + OwnedRunLease, + RunAgentMemoryImprovementOptions, + RunAgentMemoryImprovementResult, +} from './types' export function appendMemoryActivationEvent( storage: CampaignStorage, @@ -53,6 +65,116 @@ export function readMemoryActivationJournal( return { prepared, ...(activated ? { activated } : {}) } } +export async function activateMemoryWinner(input: { + options: RunAgentMemoryImprovementOptions + storage: CampaignStorage + lease: OwnedRunLease + result: RunAgentMemoryImprovementResult + activationEventIdentity: Omit + activationJournalDir: string + activationJournalPath: string + hadPreparedEvent: boolean +}): Promise { + const activationDriver = input.options.activation! + const activationTimeoutMs = input.options.activationTimeoutMs ?? 60_000 + if (!input.hadPreparedEvent) { + await input.lease.assertOwned() + input.storage.ensureDir(input.activationJournalDir) + appendMemoryActivationEvent(input.storage, input.activationJournalPath, { + ...input.activationEventIdentity, + status: 'prepared', + recordedAt: (input.options.now ?? (() => new Date()))().toISOString(), + }) + } + + await input.lease.assertOwned() + const currentConfig = await runBoundedMemoryLifecycle({ + operation: `${activationDriver.ref}: read current memory configuration`, + timeoutMs: activationTimeoutMs, + run: () => activationDriver.readCurrent(), + }) + await input.lease.assertOwned() + const codec = memoryConfigCodec(input.options) + const currentHash = surfaceHash(codec.serialize(currentConfig)) + if ( + currentHash !== input.result.baselineSurfaceHash && + currentHash !== input.result.winnerSurfaceHash + ) { + throw new Error( + `memory activation target '${activationDriver.ref}' changed concurrently; expected '${input.result.baselineSurfaceHash}' or '${input.result.winnerSurfaceHash}', found '${currentHash}'`, + ) + } + + let outcome: NonNullable + if (currentHash === input.result.winnerSurfaceHash) { + outcome = input.hadPreparedEvent ? 'recovered' : 'already-current' + input.result.activation.status = 'recovered' + } else { + let compareError: unknown + try { + await runBoundedMemoryLifecycle({ + operation: `${activationDriver.ref}: activate memory configuration`, + timeoutMs: activationTimeoutMs, + run: () => + activationDriver.compareAndSet({ + activationId: input.result.activation.id, + expectedConfig: input.result.baselineConfig, + expectedSurfaceHash: input.result.baselineSurfaceHash, + config: input.result.winnerConfig, + surfaceHash: input.result.winnerSurfaceHash, + decision: input.result.decision, + optimization: input.result.optimization, + finalEvaluation: input.result.finalEvaluation, + }), + }) + } catch (error) { + compareError = error + } + await input.lease.assertOwned() + + let observedConfig: TConfig + try { + observedConfig = await runBoundedMemoryLifecycle({ + operation: `${activationDriver.ref}: confirm memory configuration`, + timeoutMs: activationTimeoutMs, + run: () => activationDriver.readCurrent(), + }) + } catch (error) { + if (compareError) { + throw new AggregateError( + [compareError, error], + `memory activation '${input.result.activation.id}' failed and its live state could not be confirmed`, + ) + } + throw error + } + await input.lease.assertOwned() + const observedHash = surfaceHash(codec.serialize(observedConfig)) + if (observedHash !== input.result.winnerSurfaceHash) { + const mismatch = new Error( + `memory activation '${input.result.activation.id}' did not install the measured winner; found '${observedHash}'`, + ) + if (compareError) { + throw new AggregateError( + [compareError, mismatch], + `memory activation '${input.result.activation.id}' failed without applying the measured winner`, + ) + } + throw mismatch + } + outcome = compareError ? 'recovered' : 'applied' + input.result.activation.status = compareError ? 'recovered' : 'activated' + } + + await input.lease.assertOwned() + appendMemoryActivationEvent(input.storage, input.activationJournalPath, { + ...input.activationEventIdentity, + status: 'activated', + outcome, + recordedAt: (input.options.now ?? (() => new Date()))().toISOString(), + }) +} + function parseMemoryActivationEvent( value: unknown, path: string, diff --git a/src/memory/improvement/candidate.ts b/src/memory/improvement/candidate.ts index 8db16b8..be07971 100644 --- a/src/memory/improvement/candidate.ts +++ b/src/memory/improvement/candidate.ts @@ -1,84 +1,33 @@ -import type { - CampaignStorage, - CostLedgerHandle, - createRunCostLedger, - Governor, - SurfaceProposer, -} from '@tangle-network/agent-eval/campaign' +import type { CampaignStorage, JsonValue } from '@tangle-network/agent-eval/campaign' import type { AgentMemoryExperimentCandidate, RunAgentMemoryExperimentOptions } from '../experiment' -import type { AgentMemoryGovernor, OwnedRunLease, RunAgentMemoryImprovementOptions } from './types' +import type { OwnedRunLease, RunAgentMemoryImprovementOptions } from './types' -export function withCostContext( - proposer: SurfaceProposer, - costLedger: CostLedgerHandle, - lease: OwnedRunLease, - label: string, -): SurfaceProposer { - return { - kind: proposer.kind, - async propose(context) { - await lease.assertOwned() - const proposal = await proposer.propose({ - ...context, - costLedger, - costPhase: `memory.proposal.${context.track?.id ?? label}`, - }) - await lease.assertOwned() - return proposal - }, - ...(proposer.decide ? { decide: (input) => proposer.decide!(input) } : {}), - } -} - -export function withGovernorCostContext( - governor: AgentMemoryGovernor, - costLedger: CostLedgerHandle, - lease: OwnedRunLease, -): Governor { - return { - async decide(context) { - await lease.assertOwned() - const decision = await governor.decide({ - ...context, - costLedger, - costPhase: 'memory.governor', - }) - await lease.assertOwned() - return decision - }, - } -} - -export async function buildCandidate( +export async function buildCandidate( options: RunAgentMemoryImprovementOptions, config: TConfig, hash: string, - role: string, ): Promise { + const id = `memory-config-${hash}` const built = await options.createCandidate({ config, - candidateId: `${role}-${hash}`, + candidateId: id, surfaceHash: hash, }) return { ...built, - id: `${role}-${hash}`, + id, ref: built.ref, } } -export function experimentOptions( +export function experimentOptions( options: RunAgentMemoryImprovementOptions, - costLedger: ReturnType, storage: CampaignStorage, lease: OwnedRunLease, ): Pick< RunAgentMemoryExperimentOptions, | 'storage' - | 'seed' - | 'reps' | 'resumable' - | 'maxConcurrency' | 'dispatchTimeoutMs' | 'cleanupTimeoutMs' | 'maxRecoveryAttempts' @@ -87,16 +36,12 @@ export function experimentOptions( | 'executeStepRef' | 'onBranchSnapshot' | 'cleanupBranches' - | 'costLedger' | 'acquireRunLease' | 'now' > { return { storage, - seed: options.seed, - reps: options.reps, resumable: options.resumable, - maxConcurrency: options.sequenceConcurrency, dispatchTimeoutMs: options.dispatchTimeoutMs, cleanupTimeoutMs: options.cleanupTimeoutMs, maxRecoveryAttempts: options.maxRecoveryAttempts, @@ -105,7 +50,6 @@ export function experimentOptions( executeStepRef: options.executeStepRef, onBranchSnapshot: options.onBranchSnapshot, cleanupBranches: options.cleanupBranches ?? true, - costLedger, acquireRunLease: async () => ({ assertOwned: () => lease.assertOwned(), release() {}, diff --git a/src/memory/improvement/evaluation.ts b/src/memory/improvement/evaluation.ts new file mode 100644 index 0000000..a3ae0a3 --- /dev/null +++ b/src/memory/improvement/evaluation.ts @@ -0,0 +1,376 @@ +import { dirname, join } from 'node:path' +import { canonicalJson } from '@tangle-network/agent-eval' +import { + type CampaignCostMeter, + type CampaignStorage, + type JsonValue, + surfaceHash, +} from '@tangle-network/agent-eval/campaign' +import { stableId } from '../../ids' +import type { SerializedCandidateCodec } from '../../optimization' +import { + type AgentMemorySequence, + type AgentMemorySequenceArtifact, + runAgentMemoryExperiment, +} from '../experiment' +import { buildCandidate, experimentOptions } from './candidate' +import { memorySequenceFingerprint, parseMemoryConfig, serializeMemoryConfig } from './identity' +import type { + AgentMemoryFinalEvaluation, + AgentMemoryFinalPair, + MemoryConfigScenario, + OwnedRunLease, + RunAgentMemoryImprovementOptions, +} from './types' + +interface StoredMemoryArtifact { + schema: 1 + surfaceHash: string + sequenceFingerprint: string + sequenceId: string + rep: number + seed: number + artifact: AgentMemorySequenceArtifact +} + +export function memoryConfigCodec( + options: RunAgentMemoryImprovementOptions, +): SerializedCandidateCodec { + const serializeRaw = options.serializeConfig ?? ((config: TConfig) => canonicalJson(config)) + const parseRaw = options.parseConfig ?? ((surface: string) => JSON.parse(surface) as TConfig) + return { + serialize: (config) => serializeMemoryConfig(serializeRaw, config), + parse: (surface) => parseMemoryConfig(parseRaw, surface), + } +} + +export function memoryConfigScenarios( + sequences: readonly AgentMemorySequence[], +): MemoryConfigScenario[] { + return sequences.map(memoryConfigScenario) +} + +export async function evaluateMemoryCandidate(input: { + options: RunAgentMemoryImprovementOptions + storage: CampaignStorage + runDir: string + lease: OwnedRunLease + config: TConfig + surfaceHash: string + scenario: MemoryConfigScenario + rep: number + seed: number + final: boolean + cost: CampaignCostMeter + signal: AbortSignal +}): Promise { + await input.lease.assertOwned() + const artifactPath = memoryArtifactPath( + input.runDir, + input.surfaceHash, + input.scenario, + input.rep, + input.seed, + ) + const stored = readStoredMemoryArtifact(input.storage, artifactPath, { + surfaceHash: input.surfaceHash, + scenario: input.scenario, + rep: input.rep, + seed: input.seed, + }) + if (stored) { + if (input.final) { + writeStoredMemoryArtifact( + input.storage, + memoryFinalArtifactPath(input.runDir, input.surfaceHash, input.scenario, input.rep), + storedMemoryArtifactRecord(input, stored), + ) + } + return stored + } + + const candidate = await buildCandidate(input.options, input.config, input.surfaceHash) + const evaluationCostLimit = input.options.maximumEvaluationCostUsd ?? 0 + const evaluationId = stableId( + 'memory_eval', + canonicalJson({ + surfaceHash: input.surfaceHash, + sequence: memorySequenceFingerprint(input.scenario.sequence), + rep: input.rep, + seed: input.seed, + }), + ) + const paid = await input.cost.runPaidCall({ + actor: 'agent-knowledge:memory-config-evaluation', + model: candidate.ref, + signal: input.signal, + maximumCharge: { + externallyEnforcedMaximumUsd: evaluationCostLimit, + }, + execute: async () => { + const experiment = await runAgentMemoryExperiment({ + ...experimentOptions(input.options, input.storage, input.lease), + experimentId: `${input.options.experimentId}:${evaluationId}`, + experimentRunId: evaluationId, + sequences: [input.scenario.sequence], + candidates: [candidate], + runDir: join(input.runDir, 'evaluations', evaluationId), + seed: input.seed, + reps: 1, + maxConcurrency: 1, + costCeiling: evaluationCostLimit, + costPhase: `memory.config.${input.surfaceHash}`, + }) + const cell = experiment.campaign.cells[0] + if (!cell || cell.error || cell.artifact.candidateId !== candidate.id) { + throw new Error( + `${input.surfaceHash}/${input.scenario.sequenceId}: memory config evaluation did not complete`, + ) + } + const cost = experiment.campaign.aggregates.cost + return { + artifact: cell.artifact, + costUsd: cell.cached ? 0 : experiment.totalCostUsd, + cost, + } + }, + receipt: (value) => ({ + model: candidate.ref, + inputTokens: value.cost.inputTokens, + outputTokens: value.cost.outputTokens, + ...(value.cost.reasoningTokens !== undefined + ? { reasoningTokens: value.cost.reasoningTokens } + : {}), + cachedTokens: value.cost.cachedTokens, + ...(value.cost.cacheWriteTokens !== undefined + ? { cacheWriteTokens: value.cost.cacheWriteTokens } + : {}), + actualCostUsd: value.costUsd, + costUnknown: !value.cost.accountingComplete, + usageUnknown: !value.cost.usageComplete, + }), + }) + if (!paid.succeeded) throw paid.error + await input.lease.assertOwned() + const record = storedMemoryArtifactRecord(input, paid.value.artifact) + writeStoredMemoryArtifact(input.storage, artifactPath, record) + if (input.final) { + writeStoredMemoryArtifact( + input.storage, + memoryFinalArtifactPath(input.runDir, input.surfaceHash, input.scenario, input.rep), + record, + ) + } + return paid.value.artifact +} + +export function loadFinalEvaluation(input: { + options: RunAgentMemoryImprovementOptions + storage: CampaignStorage + runDir: string + baselineSurfaceHash: string + winnerSurfaceHash: string +}): AgentMemoryFinalEvaluation { + const pairs: AgentMemoryFinalPair[] = [] + const reps = input.options.reps ?? 1 + for (const sequence of input.options.finalSequences) { + const scenario = memoryConfigScenario(sequence) + for (let rep = 0; rep < reps; rep += 1) { + const baseline = readStoredMemoryArtifact( + input.storage, + memoryFinalArtifactPath(input.runDir, input.baselineSurfaceHash, scenario, rep), + { surfaceHash: input.baselineSurfaceHash, scenario, rep }, + ) + const winner = readStoredMemoryArtifact( + input.storage, + memoryFinalArtifactPath(input.runDir, input.winnerSurfaceHash, scenario, rep), + { surfaceHash: input.winnerSurfaceHash, scenario, rep }, + ) + if (!baseline || !winner) { + throw new Error( + `memory final artifact is missing for sequence '${sequence.id}' repetition ${rep}`, + ) + } + pairs.push({ sequenceId: sequence.id, rep, baseline, winner }) + } + } + return { + manifestHash: surfaceHash( + canonicalJson({ + baselineSurfaceHash: input.baselineSurfaceHash, + winnerSurfaceHash: input.winnerSurfaceHash, + pairs, + }), + ), + pairs, + } +} + +export function memoryArtifactPath( + runDir: string, + candidateSurfaceHash: string, + scenario: MemoryConfigScenario, + rep: number, + seed: number, +): string { + return join( + runDir, + 'memory-config-artifacts', + candidateSurfaceHash, + stableId('sequence', scenario.sequenceId), + `rep-${rep}-${stableId('seed', String(seed))}.json`, + ) +} + +function memoryFinalArtifactPath( + runDir: string, + candidateSurfaceHash: string, + scenario: MemoryConfigScenario, + rep: number, +): string { + return join( + runDir, + 'memory-final-artifacts', + candidateSurfaceHash, + stableId('sequence', scenario.sequenceId), + `rep-${rep}.json`, + ) +} + +function memoryConfigScenario(sequence: AgentMemorySequence): MemoryConfigScenario { + return { + id: sequence.id, + kind: 'agent-memory-config-search', + sequenceId: sequence.id, + sequence, + } +} + +function readStoredMemoryArtifact( + storage: CampaignStorage, + path: string, + expected: { + surfaceHash: string + scenario: MemoryConfigScenario + rep: number + seed?: number + }, +): AgentMemorySequenceArtifact | undefined { + const stored = storage.read(path) + if (stored === undefined) { + if (storage.exists(path)) throw new Error(`cannot read memory config artifact '${path}'`) + return undefined + } + let record: unknown + try { + record = JSON.parse(stored) + } catch (error) { + throw new Error(`invalid memory config artifact '${path}'`, { cause: error }) + } + if (!record || typeof record !== 'object' || Array.isArray(record)) { + throw new Error(`invalid memory config artifact '${path}'`) + } + const value = record as Partial + if ( + value.schema !== 1 || + value.surfaceHash !== expected.surfaceHash || + value.sequenceFingerprint !== memorySequenceFingerprint(expected.scenario.sequence) || + value.sequenceId !== expected.scenario.sequenceId || + value.rep !== expected.rep || + !Number.isSafeInteger(value.seed) || + (expected.seed !== undefined && value.seed !== expected.seed) + ) { + throw new Error(`memory config artifact '${path}' does not match its evaluation cell`) + } + return parseMemoryArtifact( + value.artifact, + `memory-config-${expected.surfaceHash}`, + expected.scenario.sequenceId, + path, + ) +} + +function writeStoredMemoryArtifact( + storage: CampaignStorage, + path: string, + record: StoredMemoryArtifact, +): void { + const serialized = `${JSON.stringify(record, null, 2)}\n` + const existing = storage.read(path) + if (existing !== undefined) { + if (canonicalJson(JSON.parse(existing)) !== canonicalJson(record)) { + throw new Error(`memory config artifact '${path}' conflicts with durable content`) + } + return + } + if (storage.exists(path)) throw new Error(`cannot read memory config artifact '${path}'`) + storage.ensureDir(dirname(path)) + storage.write(path, serialized) +} + +function storedMemoryArtifactRecord( + input: { + surfaceHash: string + scenario: MemoryConfigScenario + rep: number + seed: number + }, + artifact: AgentMemorySequenceArtifact, +): StoredMemoryArtifact { + return { + schema: 1, + surfaceHash: input.surfaceHash, + sequenceFingerprint: memorySequenceFingerprint(input.scenario.sequence), + sequenceId: input.scenario.sequenceId, + rep: input.rep, + seed: input.seed, + artifact, + } +} + +function parseMemoryArtifact( + value: unknown, + candidateId: string, + sequenceId: string, + path: string, +): AgentMemorySequenceArtifact { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`memory config artifact '${path}' has no artifact`) + } + const artifact = value as Partial + if ( + artifact.candidateId !== candidateId || + artifact.sequenceId !== sequenceId || + typeof artifact.score !== 'number' || + !Number.isFinite(artifact.score) || + artifact.score < 0 || + artifact.score > 1 || + typeof artifact.passed !== 'boolean' || + !isFiniteNumberRecord(artifact.dimensions) || + !isNonnegativeIntegerRecord(artifact.dimensionSampleCounts) || + !Array.isArray(artifact.probes) || + typeof artifact.branchDigest !== 'string' || + !artifact.branchDigest || + typeof artifact.journalEntries !== 'number' || + !Number.isSafeInteger(artifact.journalEntries) || + artifact.journalEntries < 0 || + typeof artifact.durationMs !== 'number' || + !Number.isFinite(artifact.durationMs) || + artifact.durationMs < 0 + ) { + throw new Error(`memory config artifact '${path}' is malformed`) + } + return artifact as AgentMemorySequenceArtifact +} + +function isFiniteNumberRecord(value: unknown): value is Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false + return Object.values(value).every((entry) => typeof entry === 'number' && Number.isFinite(entry)) +} + +function isNonnegativeIntegerRecord(value: unknown): value is Record { + return ( + isFiniteNumberRecord(value) && + Object.values(value).every((entry) => Number.isSafeInteger(entry) && entry >= 0) + ) +} diff --git a/src/memory/improvement/identity.ts b/src/memory/improvement/identity.ts index 1167a2b..1968f58 100644 --- a/src/memory/improvement/identity.ts +++ b/src/memory/improvement/identity.ts @@ -2,7 +2,7 @@ import { join } from 'node:path' import { canonicalJson } from '@tangle-network/agent-eval' import { type CampaignStorage, - type MutableSurface, + type JsonValue, surfaceHash, } from '@tangle-network/agent-eval/campaign' import type { AgentMemorySequence } from '../experiment' @@ -12,7 +12,7 @@ import { type RunAgentMemoryImprovementOptions, } from './types' -export function assertMemoryImprovementIdentity( +export function assertMemoryImprovementIdentity( options: RunAgentMemoryImprovementOptions, storage: CampaignStorage, runDir: string, @@ -20,41 +20,31 @@ export function assertMemoryImprovementIdentity( ): void { const path = join(runDir, 'memory-improvement-manifest.json') const identity = { - schema: 6, + schema: 7, implementationRef: MEMORY_IMPROVEMENT_IMPLEMENTATION_REF, experimentId: options.experimentId, improvementRef: options.improvementRef, + method: options.method.name, activationRef: options.activation?.ref ?? null, - proposerKind: options.proposer.kind, - proposerKinds: Object.fromEntries( - Object.entries(options.proposers ?? {}) - .sort(([a], [b]) => a.localeCompare(b)) - .map(([name, proposer]) => [name, proposer.kind]), - ), - populationSize: options.populationSize ?? 4, - budget: { maxSteps: options.budget.maxSteps }, + baselineConfig: serialize(options.baselineConfig), executeStepRef: options.executeStepRef ?? null, cleanupBranches: options.cleanupBranches ?? true, promotionPolicy: normalizedPromotionPolicy(options), - seed: options.seed ?? null, + seed: options.seed ?? 42, reps: options.reps ?? 1, - seeds: options.seeds.map((entry) => ({ - config: serialize(entry.config), - track: entry.track, - vision: entry.vision ?? null, - proposer: entry.proposer, - })), + maxOptimizationCostUsd: options.maxOptimizationCostUsd ?? null, + maxFinalCostUsd: options.maxFinalCostUsd ?? null, + maximumEvaluationCostUsd: options.maximumEvaluationCostUsd ?? null, + allowIncompleteCostAccounting: options.allowIncompleteCostAccounting ?? false, trainSequences: options.trainSequences, - holdoutSequences: options.holdoutSequences, + selectionSequences: options.selectionSequences, + finalSequences: options.finalSequences, } const identityHash = surfaceHash(canonicalJson(identity)) const stored = storage.read(path) if (stored === undefined) { if (storage.exists(path)) throw new Error(`cannot read memory improvement manifest '${path}'`) - if ( - storage.exists(join(runDir, 'lineage.jsonl')) || - storage.exists(join(runDir, 'memory-improvement-result.json')) - ) { + if (storage.exists(join(runDir, 'memory-improvement-result.json'))) { throw new Error(`memory improvement run '${runDir}' has state without an identity manifest`) } storage.write(path, `${JSON.stringify({ identityHash, identity }, null, 2)}\n`) @@ -73,19 +63,12 @@ export function assertMemoryImprovementIdentity( canonicalJson((manifest as Record).identity) !== canonicalJson(identity) ) { throw new Error( - `memory improvement run '${runDir}' does not match its persisted inputs or implementationRef`, + `memory improvement run '${runDir}' does not match its persisted inputs or improvementRef`, ) } } -export function requireStringSurface(surface: MutableSurface): string { - if (typeof surface !== 'string' || surface.trim().length === 0) { - throw new Error('memory config proposer must return a JSON string surface') - } - return surface -} - -export function serializeMemoryConfig( +export function serializeMemoryConfig( serialize: (config: TConfig) => string, config: TConfig, ): string { @@ -101,7 +84,7 @@ export function serializeMemoryConfig( return surface } -export function parseMemoryConfig( +export function parseMemoryConfig( parse: (surface: string) => TConfig, surface: string, ): TConfig { @@ -114,7 +97,7 @@ export function parseMemoryConfig( } } -export function assertMemoryConfigRoundTrip( +export function assertMemoryConfigRoundTrip( config: TConfig, serialize: (config: TConfig) => string, parse: (surface: string) => TConfig, @@ -122,7 +105,7 @@ export function assertMemoryConfigRoundTrip( const first = serialize(config) const second = serialize(parse(first)) if (first !== second) { - throw new Error('memory config serializer and parser must round-trip seed configs exactly') + throw new Error('memory config serializer and parser must round-trip exactly') } } diff --git a/src/memory/improvement/output.ts b/src/memory/improvement/output.ts index 396261c..cef4403 100644 --- a/src/memory/improvement/output.ts +++ b/src/memory/improvement/output.ts @@ -1,7 +1,7 @@ -import type { CampaignStorage } from '@tangle-network/agent-eval/campaign' +import type { CampaignStorage, JsonValue } from '@tangle-network/agent-eval/campaign' import type { RunAgentMemoryImprovementResult } from './types' -export function writeMemoryImprovementResult( +export function writeMemoryImprovementResult( storage: CampaignStorage, experimentId: string, result: RunAgentMemoryImprovementResult, @@ -16,9 +16,11 @@ export function writeMemoryImprovementResult( baselineSurface: result.baselineSurface, winnerSurface: result.winnerSurface, decision: result.decision, + method: result.optimization.methodName, + comparison: result.optimization.comparison, + finalEvaluation: result.finalEvaluation, activation: result.activation, totalCostUsd: result.totalCostUsd, - lineage: result.lineage.toGraph(), }, null, 2, diff --git a/src/memory/improvement/promotion.ts b/src/memory/improvement/promotion.ts index b8f4925..a710a27 100644 --- a/src/memory/improvement/promotion.ts +++ b/src/memory/improvement/promotion.ts @@ -1,39 +1,51 @@ -import { heldoutSignificance, type PairedHoldout } from '@tangle-network/agent-eval/campaign' -import type { - AgentMemorySequence, - AgentMemorySequenceArtifact, - AgentMemorySequenceProbe, - RunAgentMemoryExperimentResult, -} from '../experiment' +import type { JsonValue, PairedHoldout } from '@tangle-network/agent-eval/campaign' +import { heldoutSignificance } from '@tangle-network/agent-eval/campaign' +import type { RunSerializedKnowledgeOptimizationResult } from '../../optimization' +import type { AgentMemorySequence, AgentMemorySequenceProbe } from '../experiment' import { + type AgentMemoryFinalEvaluation, type AgentMemoryPromotionDecision, DEFAULT_CRITICAL_DIMENSIONS, type RunAgentMemoryImprovementOptions, } from './types' -export function decidePromotion(input: { +export function decidePromotion(input: { options: RunAgentMemoryImprovementOptions - result: RunAgentMemoryExperimentResult - baselineId: string - winnerId: string + optimization: RunSerializedKnowledgeOptimizationResult + finalEvaluation: AgentMemoryFinalEvaluation + unchanged: boolean }): AgentMemoryPromotionDecision { - const { options, result, baselineId, winnerId } = input - const baselineRow = result.rows.find((row) => row.candidateId === baselineId) - const winnerRow = result.rows.find((row) => row.candidateId === winnerId) - if (!baselineRow || !winnerRow) throw new Error('holdout result is missing a comparison arm') - const paired = pairedArtifacts(result, baselineId, winnerId, (artifact) => artifact.score) - const significance = heldoutSignificance(paired, options.significance) + const { options, finalEvaluation } = input + const baselineScores = finalEvaluation.pairs.map((pair) => pair.baseline.score) + const winnerScores = finalEvaluation.pairs.map((pair) => pair.winner.score) + const baselineScore = mean(baselineScores) + const winnerScore = mean(winnerScores) + if (input.unchanged) { + return { + status: 'no-change', + reasons: ['optimization selected the baseline configuration'], + baselineScore, + winnerScore, + lift: 0, + criticalDimensions: [], + } + } + + const significance = heldoutSignificance( + { + before: baselineScores, + after: winnerScores, + cellIds: finalEvaluation.pairs.map((pair) => `${pair.sequenceId}:${pair.rep}`), + }, + options.significance, + ) const tolerance = options.criticalDimensionTolerance ?? 0.05 const criticalDimensions = (options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS).map( (dimension) => { const expectedN = - applicableSequenceCount(options.holdoutSequences, dimension) * (options.reps ?? 1) - const dimensionPairs = pairedArtifacts(result, baselineId, winnerId, (artifact) => - (artifact.dimensionSampleCounts?.[dimension] ?? 0) > 0 - ? artifact.dimensions[dimension] - : undefined, - ) - const comparison = heldoutSignificance(dimensionPairs, { + applicableSequenceCount(options.finalSequences, dimension) * (options.reps ?? 1) + const pairs = pairedDimension(finalEvaluation, dimension) + const comparison = heldoutSignificance(pairs, { ...options.significance, deltaThreshold: 0, }) @@ -52,25 +64,40 @@ export function decidePromotion(input: { }, ) const reasons: string[] = [] - if (baselineRow.cellsFailed > 0 || winnerRow.cellsFailed > 0) { - reasons.push('at least one holdout cell failed') + const optimizationCost = input.optimization.comparison.optimizationCost + const finalCost = input.optimization.comparison.testCost + if ( + !input.optimization.comparison.totalCost.accountingComplete && + !options.allowIncompleteCostAccounting + ) { + reasons.push('optimization or final cost accounting is incomplete') + } + if (optimizationCost.totalCostUsd > (options.maxOptimizationCostUsd ?? 0)) { + reasons.push( + `optimization cost ${optimizationCost.totalCostUsd} exceeds the configured limit ${options.maxOptimizationCostUsd ?? 0}`, + ) + } + if (finalCost.totalCostUsd > (options.maxFinalCostUsd ?? 0)) { + reasons.push( + `final comparison cost ${finalCost.totalCostUsd} exceeds the configured limit ${options.maxFinalCostUsd ?? 0}`, + ) } if (!significance.significant) { reasons.push( significance.fewRuns - ? `only ${significance.n} paired holdout cells; more are required` - : 'holdout lift is not confidently above the promotion threshold', + ? `only ${significance.n} paired final cells; more are required` + : 'final lift is not confidently above the promotion threshold', ) } - if (winnerRow.scoreMean < (options.minHoldoutScore ?? 0)) { - reasons.push(`winner holdout score ${winnerRow.scoreMean} is below the required minimum`) + if (winnerScore < (options.minFinalScore ?? 0)) { + reasons.push(`winner final score ${winnerScore} is below the required minimum`) } for (const dimension of criticalDimensions) { if (!dimension.measured) { reasons.push( dimension.expectedN === 0 - ? `critical dimension ${dimension.dimension} has no applicable holdout histories` - : `critical dimension ${dimension.dimension} was measured on ${dimension.n}/${dimension.expectedN} applicable paired holdout cells`, + ? `critical dimension ${dimension.dimension} has no applicable final histories` + : `critical dimension ${dimension.dimension} was measured on ${dimension.n}/${dimension.expectedN} applicable paired final cells`, ) } else if (dimension.regressed) { reasons.push(`${dimension.dimension} may regress beyond ${tolerance}`) @@ -79,14 +106,48 @@ export function decidePromotion(input: { return { status: reasons.length === 0 ? 'promote' : 'hold', reasons, - baselineScore: baselineRow.scoreMean, - winnerScore: winnerRow.scoreMean, - lift: winnerRow.scoreMean - baselineRow.scoreMean, + baselineScore, + winnerScore, + lift: winnerScore - baselineScore, significance, criticalDimensions, } } +export function normalizedPromotionPolicy( + options: RunAgentMemoryImprovementOptions, +): Record { + return { + significance: { + deltaThreshold: options.significance?.deltaThreshold ?? 0, + minProductiveRuns: options.significance?.minProductiveRuns ?? 3, + confidence: options.significance?.confidence ?? 0.95, + resamples: options.significance?.resamples ?? 2000, + seed: options.significance?.seed ?? 1337, + statistic: options.significance?.statistic ?? 'mean', + }, + criticalDimensions: [...(options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS)], + criticalDimensionTolerance: options.criticalDimensionTolerance ?? 0.05, + minFinalScore: options.minFinalScore ?? 0, + maxOptimizationCostUsd: options.maxOptimizationCostUsd ?? 0, + maxFinalCostUsd: options.maxFinalCostUsd ?? 0, + allowIncompleteCostAccounting: options.allowIncompleteCostAccounting ?? false, + } +} + +function pairedDimension(result: AgentMemoryFinalEvaluation, dimension: string): PairedHoldout { + const applicable = result.pairs.filter( + (pair) => + (pair.baseline.dimensionSampleCounts[dimension] ?? 0) > 0 && + (pair.winner.dimensionSampleCounts[dimension] ?? 0) > 0, + ) + return { + before: applicable.map((pair) => pair.baseline.dimensions[dimension]!), + after: applicable.map((pair) => pair.winner.dimensions[dimension]!), + cellIds: applicable.map((pair) => `${pair.sequenceId}:${pair.rep}`), + } +} + function applicableSequenceCount( sequences: readonly AgentMemorySequence[], dimension: string, @@ -120,70 +181,6 @@ function probeAppliesToDimension(probe: AgentMemorySequenceProbe, dimension: str } } -export function normalizedPromotionPolicy( - options: RunAgentMemoryImprovementOptions, -): Record { - return { - significance: { - deltaThreshold: options.significance?.deltaThreshold ?? 0, - minProductiveRuns: options.significance?.minProductiveRuns ?? 3, - confidence: options.significance?.confidence ?? 0.95, - resamples: options.significance?.resamples ?? 2000, - seed: options.significance?.seed ?? 1337, - statistic: options.significance?.statistic ?? 'mean', - }, - criticalDimensions: [...(options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS)], - criticalDimensionTolerance: options.criticalDimensionTolerance ?? 0.05, - minHoldoutScore: options.minHoldoutScore ?? 0, - } -} - -function pairedArtifacts( - result: RunAgentMemoryExperimentResult, - baselineId: string, - winnerId: string, - select: (artifact: AgentMemorySequenceArtifact) => number | undefined, -): PairedHoldout { - const baseline = artifactValues(result, baselineId, select) - const winner = artifactValues(result, winnerId, select) - const keys = [...baseline.keys()].filter((key) => winner.has(key)).sort() - return { - before: keys.map((key) => baseline.get(key)!), - after: keys.map((key) => winner.get(key)!), - cellIds: keys, - } -} - -function artifactValues( - result: RunAgentMemoryExperimentResult, - candidateId: string, - select: (artifact: AgentMemorySequenceArtifact) => number | undefined, -): Map { - const values = new Map() - for (const cell of result.campaign.cells) { - if (cell.error || cell.artifact.candidateId !== candidateId) continue - const value = select(cell.artifact) - if (value === undefined || !Number.isFinite(value)) continue - values.set(`${cell.artifact.sequenceId}:${cell.rep}`, value) - } - return values -} - -export function sequenceScores( - result: RunAgentMemoryExperimentResult, - sequences: readonly AgentMemorySequence[], - candidateId: string, -): number[] { - const bySequence = new Map() - for (const cell of result.campaign.cells) { - if (cell.error || cell.artifact.candidateId !== candidateId) continue - const bucket = bySequence.get(cell.artifact.sequenceId) ?? [] - bucket.push(cell.artifact.score) - bySequence.set(cell.artifact.sequenceId, bucket) - } - return sequences.map((sequence) => mean(bySequence.get(sequence.id) ?? [])) -} - function mean(values: readonly number[]): number { - return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length + return values.reduce((sum, value) => sum + value, 0) / values.length } diff --git a/src/memory/improvement/run.ts b/src/memory/improvement/run.ts index 9286370..651f4fb 100644 --- a/src/memory/improvement/run.ts +++ b/src/memory/improvement/run.ts @@ -2,40 +2,31 @@ import { join } from 'node:path' import { canonicalJson } from '@tangle-network/agent-eval' import { type CampaignStorage, - campaignLineageStore, - createRunCostLedger, fsCampaignStorage, - type MutableSurface, - memLineageStore, + type JsonValue, resolveRunDir, - runLineageLoop, surfaceHash, } from '@tangle-network/agent-eval/campaign' -import { assertNoInterruptedPaidCalls, reconcileInterruptedRunPaidCalls } from '../attempt-log' -import { type RunAgentMemoryExperimentResult, runAgentMemoryExperiment } from '../experiment' -import { runBoundedMemoryLifecycle } from '../lifecycle' +import { runSerializedKnowledgeOptimization } from '../../optimization' +import { type AgentMemorySequenceArtifact, agentMemorySequenceJudge } from '../experiment' import { acquireAgentMemoryRunLease } from '../run-control' -import { appendMemoryActivationEvent, readMemoryActivationJournal } from './activation' +import { activateMemoryWinner, readMemoryActivationJournal } from './activation' import { - buildCandidate, - experimentOptions, - withCostContext, - withGovernorCostContext, -} from './candidate' + evaluateMemoryCandidate, + loadFinalEvaluation, + memoryArtifactPath, + memoryConfigCodec, + memoryConfigScenarios, +} from './evaluation' import { assertMemoryConfigRoundTrip, assertMemoryImprovementIdentity, memorySequenceFingerprint, - parseMemoryConfig, - requireStringSurface, - serializeMemoryConfig, } from './identity' import { writeMemoryImprovementResult } from './output' -import { decidePromotion, normalizedPromotionPolicy, sequenceScores } from './promotion' +import { decidePromotion, normalizedPromotionPolicy } from './promotion' import type { AgentMemoryActivation, - AgentMemoryActivationEvent, - AgentMemoryPromotionDecision, MemoryConfigScenario, OwnedRunLease, RunAgentMemoryImprovementOptions, @@ -43,44 +34,10 @@ import type { } from './types' import { assertMemoryImprovementOptions } from './validation' -/** Searches branchable memory configurations and activates only a fresh holdout win. */ -export async function runAgentMemoryImprovement( +/** Optimizes memory configuration with an external method and activates only a fresh final win. */ +export async function runAgentMemoryImprovement( options: RunAgentMemoryImprovementOptions, ): Promise> { - if ('onPromote' in options) { - throw new Error( - 'memory improvement onPromote was removed; use activation.readCurrent and activation.compareAndSet', - ) - } - if (options.seeds.length === 0) throw new Error('memory improvement requires seed configs') - if (options.trainSequences.length === 0) { - throw new Error('memory improvement requires training sequences') - } - if (options.holdoutSequences.length === 0) { - throw new Error('memory improvement requires holdout sequences') - } - const trainIds = new Set(options.trainSequences.map((sequence) => sequence.id)) - const overlap = options.holdoutSequences - .map((sequence) => sequence.id) - .filter((id) => trainIds.has(id)) - if (overlap.length > 0) { - throw new Error(`memory improvement train/holdout overlap: ${overlap.join(', ')}`) - } - const trainFingerprints = new Map( - options.trainSequences.map((sequence) => [memorySequenceFingerprint(sequence), sequence.id]), - ) - const duplicateHistories = options.holdoutSequences.flatMap((sequence) => { - const trainId = trainFingerprints.get(memorySequenceFingerprint(sequence)) - return trainId ? [`${trainId}/${sequence.id}`] : [] - }) - if (duplicateHistories.length > 0) { - throw new Error( - `memory improvement train/holdout histories duplicate content: ${duplicateHistories.join(', ')}`, - ) - } - if (typeof options.improvementRef !== 'string' || !options.improvementRef.trim()) { - throw new Error('memory improvement improvementRef must be a non-empty string') - } assertMemoryImprovementOptions(options) const storage = options.storage ?? fsCampaignStorage() const runDir = resolveRunDir(options.runDir, options.repo) @@ -120,190 +77,132 @@ export async function runAgentMemoryImprovement( return result } -async function runAgentMemoryImprovementOwned( +async function runAgentMemoryImprovementOwned( options: RunAgentMemoryImprovementOptions, storage: CampaignStorage, runDir: string, lease: OwnedRunLease, ): Promise> { await lease.assertOwned() - const serializeRaw = options.serializeConfig ?? ((config: TConfig) => canonicalJson(config)) - const parseRaw = options.parseConfig ?? ((surface: string) => JSON.parse(surface) as TConfig) - const serialize = (config: TConfig): string => serializeMemoryConfig(serializeRaw, config) - const parse = (surface: string): TConfig => parseMemoryConfig(parseRaw, surface) - for (const seed of options.seeds) assertMemoryConfigRoundTrip(seed.config, serialize, parse) + const codec = memoryConfigCodec(options) + assertMemoryConfigRoundTrip(options.baselineConfig, codec.serialize, codec.parse) if (options.activation && !storage.append) { throw new Error('memory activation requires CampaignStorage.append') } - if (options.resumable !== false && !options.lineageStore && !storage.append) { - throw new Error('resumable memory improvement requires CampaignStorage.append') - } - assertMemoryImprovementIdentity(options, storage, runDir, serialize) - const costLedger = createRunCostLedger({ - storage, - runDir, - costCeilingUsd: options.maxTotalCostUsd ?? 0, - }) - reconcileInterruptedRunPaidCalls(costLedger, 'memory improvement run') - assertNoInterruptedPaidCalls(costLedger, 'memory improvement recovery') - const trainScenarios: MemoryConfigScenario[] = options.trainSequences.map((sequence) => ({ - id: sequence.id, - kind: 'agent-memory-config-search', - sequenceId: sequence.id, - })) - const evaluations = new Map>() + assertMemoryImprovementIdentity(options, storage, runDir, codec.serialize) - const evaluateSurface = async ( - surface: MutableSurface, - ): Promise<{ score: number; scoreVector: number[] }> => { - await lease.assertOwned() - const text = requireStringSurface(surface) - const config = parse(text) - const canonicalSurface = serialize(config) - const hash = surfaceHash(canonicalSurface) - let pending = evaluations.get(hash) - if (!pending) { - pending = (async () => { - const candidate = await buildCandidate(options, config, hash, `search-${hash}`) - await lease.assertOwned() - const experiment = await runAgentMemoryExperiment({ - ...experimentOptions(options, costLedger, storage, lease), - experimentId: `${options.experimentId}:search:${hash}`, - sequences: options.trainSequences, - candidates: [candidate], - runDir: join(runDir, 'search', hash), - costPhase: `memory.search.${hash}`, + const trainScenarios = memoryConfigScenarios(options.trainSequences) + const selectionScenarios = memoryConfigScenarios(options.selectionSequences) + const finalScenarios = memoryConfigScenarios(options.finalSequences) + const finalScenarioIds = new Set(finalScenarios.map((scenario) => scenario.id)) + const pending = new Map>() + const optimizationRunOptions = { + ...(options.optimizationRunOptions ?? {}), + storage, + ...(options.reps !== undefined ? { reps: options.reps } : {}), + ...(options.resumable !== undefined ? { resumable: options.resumable } : {}), + ...(options.sequenceConcurrency !== undefined + ? { maxConcurrency: options.sequenceConcurrency } + : {}), + ...(options.dispatchTimeoutMs !== undefined + ? { dispatchTimeoutMs: options.dispatchTimeoutMs } + : {}), + ...(options.maxOptimizationCostUsd !== undefined + ? { costCeiling: options.maxOptimizationCostUsd } + : { costCeiling: 0 }), + expectUsage: 'off' as const, + } + const optimization = await runSerializedKnowledgeOptimization({ + baseline: options.baselineConfig, + method: options.method, + trainScenarios, + selectionScenarios, + finalScenarios, + codec, + scenarioFingerprint: (scenario) => memorySequenceFingerprint(scenario.sequence), + dispatchCandidate: ({ candidate, candidateSurfaceHash, scenario, context }) => { + const key = memoryArtifactPath( + runDir, + candidateSurfaceHash, + scenario, + context.rep, + context.seed, + ) + let operation = pending.get(key) + if (!operation) { + operation = evaluateMemoryCandidate({ + options, + storage, + runDir, + lease, + config: candidate, + surfaceHash: candidateSurfaceHash, + scenario, + rep: context.rep, + seed: context.seed, + final: finalScenarioIds.has(scenario.id), + cost: context.cost, + signal: context.signal, }) - await lease.assertOwned() - return experiment - })() - evaluations.set(hash, pending) - void pending.catch(() => evaluations.delete(hash)) - } - const result = await pending - await lease.assertOwned() - const row = result.rows[0] - if (!row || row.cellsFailed > 0) { - throw new Error(`${hash}: memory candidate did not complete every training cell`) - } - return { - score: row.scoreMean, - scoreVector: sequenceScores(result, options.trainSequences, row.candidateId), - } - } - - const lineageStore = - options.lineageStore ?? - (options.resumable === false - ? memLineageStore() - : campaignLineageStore(storage, join(runDir, 'lineage.jsonl'))) - const lineageOptions = { - seeds: options.seeds.map((seed) => ({ - surface: serialize(seed.config), - track: seed.track, - proposer: seed.proposer, - ...(seed.vision !== undefined ? { vision: seed.vision } : {}), - })), - scenarios: trainScenarios, - proposer: withCostContext(options.proposer, costLedger, lease, 'default'), - proposers: options.proposers - ? Object.fromEntries( - Object.entries(options.proposers).map(([name, proposer]) => [ - name, - withCostContext(proposer, costLedger, lease, name), - ]), - ) - : undefined, - scoreSurface: evaluateSurface, - governor: options.governor - ? withGovernorCostContext(options.governor, costLedger, lease) - : undefined, - budget: { - ...options.budget, - maxNodes: options.seeds.length + options.budget.maxSteps, + pending.set(key, operation) + void operation.catch(() => pending.delete(key)) + } + return operation }, - store: lineageStore, - populationSize: options.populationSize, - candidateConcurrency: options.candidateConcurrency, - } - const search = await runLineageLoop(lineageOptions) + judges: [agentMemorySequenceJudge()], + runDir, + repo: options.repo, + storage, + seed: options.seed, + reps: options.reps, + resumable: options.resumable, + costCeiling: options.maxFinalCostUsd ?? 0, + maxConcurrency: options.sequenceConcurrency, + dispatchTimeoutMs: options.dispatchTimeoutMs, + expectUsage: 'off', + optimizationRunOptions, + now: options.now, + }) await lease.assertOwned() - const best = search.best - if (!best) throw new Error('memory improvement produced no measured config') - - const baselineSurface = serialize(options.seeds[0]!.config) - const baselineHash = surfaceHash(baselineSurface) - const winnerConfig = parse(requireStringSurface(best.surface)) - const winnerSurface = serialize(winnerConfig) - const winnerHash = surfaceHash(winnerSurface) - const baselineConfig = parse(baselineSurface) - - let holdout: RunAgentMemoryExperimentResult | undefined - let decision: AgentMemoryPromotionDecision - if (winnerHash === baselineHash) { - const baselineMeasurement = await evaluateSurface(baselineSurface) - decision = { - status: 'no-change', - reasons: ['search did not find a config better than the baseline'], - baselineScore: baselineMeasurement.score, - winnerScore: baselineMeasurement.score, - lift: 0, - criticalDimensions: [], - } - } else { - await lease.assertOwned() - const [baselineCandidate, winnerCandidate] = await Promise.all([ - buildCandidate(options, baselineConfig, baselineHash, 'baseline'), - buildCandidate(options, winnerConfig, winnerHash, 'winner'), - ]) - await lease.assertOwned() - holdout = await runAgentMemoryExperiment({ - ...experimentOptions(options, costLedger, storage, lease), - experimentId: `${options.experimentId}:holdout`, - sequences: options.holdoutSequences, - candidates: [baselineCandidate, winnerCandidate], - runDir: join(runDir, 'holdout'), - costPhase: 'memory.holdout', - }) - decision = decidePromotion({ - options, - result: holdout, - baselineId: baselineCandidate.id, - winnerId: winnerCandidate.id, - }) - } + const finalEvaluation = loadFinalEvaluation({ + options, + storage, + runDir, + baselineSurfaceHash: optimization.baseline.surfaceHash, + winnerSurfaceHash: optimization.winner.surfaceHash, + }) + const unchanged = optimization.baseline.surfaceHash === optimization.winner.surfaceHash + const decision = decidePromotion({ options, optimization, finalEvaluation, unchanged }) const resultJsonPath = join(runDir, 'memory-improvement-result.json') const activationRef = options.activation?.ref ?? 'not-configured' const activationId = `memory-activation-${surfaceHash( canonicalJson({ experimentId: options.experimentId, improvementRef: options.improvementRef, + method: optimization.methodName, activationRef, - baselineSurfaceHash: baselineHash, - winnerSurfaceHash: winnerHash, - holdoutManifestHash: holdout?.campaign.manifestHash ?? null, + baselineSurfaceHash: optimization.baseline.surfaceHash, + winnerSurfaceHash: optimization.winner.surfaceHash, + finalEvaluationHash: finalEvaluation.manifestHash, promotionPolicy: normalizedPromotionPolicy(options), }), )}` const activationJournalDir = join(runDir, 'activations') const activationJournalPath = join(activationJournalDir, `${activationId}.jsonl`) - const activationEligible = decision.status === 'promote' && holdout !== undefined - const activationEventIdentity = holdout - ? { - schema: 1 as const, - activationId, - experimentId: options.experimentId, - activationRef, - baselineSurfaceHash: baselineHash, - winnerSurfaceHash: winnerHash, - holdoutManifestHash: holdout.campaign.manifestHash, - } - : undefined - const activationJournal = - activationEligible && activationEventIdentity - ? readMemoryActivationJournal(storage, activationJournalPath, activationEventIdentity) - : { prepared: false } + const activationEligible = decision.status === 'promote' + const activationEventIdentity = { + schema: 2 as const, + activationId, + experimentId: options.experimentId, + activationRef, + baselineSurfaceHash: optimization.baseline.surfaceHash, + winnerSurfaceHash: optimization.winner.surfaceHash, + finalEvaluationHash: finalEvaluation.manifestHash, + } + const activationJournal = activationEligible + ? readMemoryActivationJournal(storage, activationJournalPath, activationEventIdentity) + : { prepared: false } const activation: AgentMemoryActivation = { id: activationId, status: !activationEligible @@ -316,122 +215,32 @@ async function runAgentMemoryImprovementOwned( journalPath: activationJournalPath, } const result = { - lineage: search.lineage, - baselineConfig, - winnerConfig, - baselineSurface, - winnerSurface, - baselineSurfaceHash: baselineHash, - winnerSurfaceHash: winnerHash, + optimization, + baselineConfig: optimization.baseline.value, + winnerConfig: optimization.winner.value, + baselineSurface: optimization.baseline.surface, + winnerSurface: optimization.winner.surface, + baselineSurfaceHash: optimization.baseline.surfaceHash, + winnerSurfaceHash: optimization.winner.surfaceHash, decision, + finalEvaluation, activation, - ...(holdout ? { holdout } : {}), - totalCostUsd: costLedger.summary().totalCostUsd, + totalCostUsd: optimization.comparison.totalCost.totalCostUsd, resultJsonPath, } satisfies RunAgentMemoryImprovementResult await lease.assertOwned() writeMemoryImprovementResult(storage, options.experimentId, result) - if ( - activationEligible && - !activationJournal.activated && - activationEventIdentity && - holdout && - options.activation - ) { - const activationDriver = options.activation - const activationTimeoutMs = options.activationTimeoutMs ?? 60_000 - const hadPreparedEvent = activationJournal.prepared - if (!hadPreparedEvent) { - await lease.assertOwned() - storage.ensureDir(activationJournalDir) - appendMemoryActivationEvent(storage, activationJournalPath, { - ...activationEventIdentity, - status: 'prepared', - recordedAt: (options.now ?? (() => new Date()))().toISOString(), - }) - } - await lease.assertOwned() - const currentConfig = await runBoundedMemoryLifecycle({ - operation: `${activationDriver.ref}: read current memory configuration`, - timeoutMs: activationTimeoutMs, - run: () => activationDriver.readCurrent(), - }) - await lease.assertOwned() - const currentHash = surfaceHash(serialize(currentConfig)) - if (currentHash !== baselineHash && currentHash !== winnerHash) { - throw new Error( - `memory activation target '${activationDriver.ref}' changed concurrently; expected '${baselineHash}' or '${winnerHash}', found '${currentHash}'`, - ) - } - - let outcome: NonNullable - if (currentHash === winnerHash) { - outcome = hadPreparedEvent ? 'recovered' : 'already-current' - activation.status = 'recovered' - } else { - let compareError: unknown - try { - await runBoundedMemoryLifecycle({ - operation: `${activationDriver.ref}: activate memory configuration`, - timeoutMs: activationTimeoutMs, - run: () => - activationDriver.compareAndSet({ - activationId, - expectedConfig: baselineConfig, - expectedSurfaceHash: baselineHash, - config: winnerConfig, - surfaceHash: winnerHash, - decision, - lineage: search.lineage, - holdout, - }), - }) - } catch (error) { - compareError = error - } - await lease.assertOwned() - - let observedConfig: TConfig - try { - observedConfig = await runBoundedMemoryLifecycle({ - operation: `${activationDriver.ref}: confirm memory configuration`, - timeoutMs: activationTimeoutMs, - run: () => activationDriver.readCurrent(), - }) - } catch (error) { - if (compareError) { - throw new AggregateError( - [compareError, error], - `memory activation '${activationId}' failed and its live state could not be confirmed`, - ) - } - throw error - } - await lease.assertOwned() - const observedHash = surfaceHash(serialize(observedConfig)) - if (observedHash !== winnerHash) { - const mismatch = new Error( - `memory activation '${activationId}' did not install the measured winner; found '${observedHash}'`, - ) - if (compareError) { - throw new AggregateError( - [compareError, mismatch], - `memory activation '${activationId}' failed without applying the measured winner`, - ) - } - throw mismatch - } - outcome = compareError ? 'recovered' : 'applied' - activation.status = compareError ? 'recovered' : 'activated' - } - - await lease.assertOwned() - appendMemoryActivationEvent(storage, activationJournalPath, { - ...activationEventIdentity, - status: 'activated', - outcome, - recordedAt: (options.now ?? (() => new Date()))().toISOString(), + if (activationEligible && !activationJournal.activated && options.activation) { + await activateMemoryWinner({ + options, + storage, + lease, + result, + activationEventIdentity, + activationJournalDir, + activationJournalPath, + hadPreparedEvent: activationJournal.prepared, }) writeMemoryImprovementResult(storage, options.experimentId, result) } diff --git a/src/memory/improvement/types.ts b/src/memory/improvement/types.ts index 0aa38ed..41eaecd 100644 --- a/src/memory/improvement/types.ts +++ b/src/memory/improvement/types.ts @@ -1,20 +1,18 @@ import type { CampaignStorage, - CostLedgerHandle, - GovernorContext, - GovernorOp, HeldoutSignificance, HeldoutSignificanceOptions, - Lineage, - LineageStore, + JsonValue, + OptimizationMethod, + OptimizationMethodRunOptions, Scenario, - SurfaceProposer, } from '@tangle-network/agent-eval/campaign' +import type { RunSerializedKnowledgeOptimizationResult } from '../../optimization' import type { AgentMemoryExperimentCandidate, AgentMemorySequence, + AgentMemorySequenceArtifact, RunAgentMemoryExperimentOptions, - RunAgentMemoryExperimentResult, } from '../experiment' import type { AgentMemoryAcquireRunLease, @@ -23,13 +21,6 @@ import type { OwnedAgentMemoryRunLease, } from '../run-control' -export interface AgentMemoryImprovementSeed { - config: TConfig - track: string - vision?: string - proposer: string -} - export interface AgentMemoryDimensionComparison { dimension: string n: number @@ -52,6 +43,18 @@ export interface AgentMemoryPromotionDecision { criticalDimensions: readonly AgentMemoryDimensionComparison[] } +export interface AgentMemoryFinalPair { + sequenceId: string + rep: number + baseline: AgentMemorySequenceArtifact + winner: AgentMemorySequenceArtifact +} + +export interface AgentMemoryFinalEvaluation { + manifestHash: string + pairs: readonly AgentMemoryFinalPair[] +} + export interface AgentMemoryActivation { id: string status: @@ -64,7 +67,7 @@ export interface AgentMemoryActivation { journalPath: string } -export interface AgentMemoryActivationDriver { +export interface AgentMemoryActivationDriver { /** Change whenever activation behavior or the external target changes. */ ref: string /** Return the exact currently active configuration. */ @@ -77,20 +80,20 @@ export interface AgentMemoryActivationDriver { config: TConfig surfaceHash: string decision: AgentMemoryPromotionDecision - lineage: Lineage - holdout: RunAgentMemoryExperimentResult + optimization: RunSerializedKnowledgeOptimizationResult + finalEvaluation: AgentMemoryFinalEvaluation }): Promise } export interface AgentMemoryActivationEvent { - schema: 1 + schema: 2 status: 'prepared' | 'activated' activationId: string experimentId: string activationRef: string baselineSurfaceHash: string winnerSurfaceHash: string - holdoutManifestHash: string + finalEvaluationHash: string recordedAt: string outcome?: 'applied' | 'recovered' | 'already-current' } @@ -102,21 +105,13 @@ export interface AgentMemoryActivationJournalState { export type AgentMemoryImprovementRunLease = AgentMemoryRunLease -export interface AgentMemoryGovernor { - decide( - context: GovernorContext & { - costLedger: CostLedgerHandle - costPhase: string - }, - ): GovernorOp | Promise -} - -export interface RunAgentMemoryImprovementOptions { +export interface RunAgentMemoryImprovementOptions { experimentId: string + baselineConfig: TConfig + method: OptimizationMethod trainSequences: readonly AgentMemorySequence[] - holdoutSequences: readonly AgentMemorySequence[] - /** First entry is the current baseline; remaining entries seed independent search tracks. */ - seeds: readonly AgentMemoryImprovementSeed[] + selectionSequences: readonly AgentMemorySequence[] + finalSequences: readonly AgentMemorySequence[] createCandidate(input: { config: TConfig candidateId: string @@ -124,32 +119,35 @@ export interface RunAgentMemoryImprovementOptions { }): | Omit | Promise> - proposer: SurfaceProposer - /** Optional proposer implementations keyed by seed and branch proposer labels. */ - proposers?: Readonly> - /** Stable version or commit for the candidate factory, proposer, and governor. */ + /** Stable version or commit for method config, candidate construction, and execution behavior. */ improvementRef: string - governor?: AgentMemoryGovernor - budget: { maxSteps: number } - populationSize?: number - candidateConcurrency?: number - sequenceConcurrency?: number runDir: string repo?: string storage?: CampaignStorage - lineageStore?: LineageStore /** Required with custom storage when all controllers are confined to one process. */ controllerMode?: AgentMemoryControllerMode - /** Required for distributed controllers using custom storage. Worker concurrency is independent. */ + /** Required for distributed controllers using custom storage. */ acquireRunLease?: AgentMemoryAcquireRunLease seed?: number reps?: number resumable?: boolean + sequenceConcurrency?: number dispatchTimeoutMs?: number cleanupTimeoutMs?: number maxRecoveryAttempts?: number maxRecoveryRetriesPerAttempt?: number - maxTotalCostUsd?: number + /** Method search spend limit. */ + maxOptimizationCostUsd?: number + /** Final comparison spend limit. */ + maxFinalCostUsd?: number + /** Enforced maximum for one config and one sequence. Required with either spend limit. */ + maximumEvaluationCostUsd?: number + /** Allow activation when a method cannot fully account for cost. Default false. */ + allowIncompleteCostAccounting?: boolean + optimizationRunOptions?: OptimizationMethodRunOptions< + MemoryConfigScenario, + AgentMemorySequenceArtifact + > executeStep?: RunAgentMemoryExperimentOptions['executeStep'] executeStepRef?: string onBranchSnapshot?: RunAgentMemoryExperimentOptions['onBranchSnapshot'] @@ -159,14 +157,14 @@ export interface RunAgentMemoryImprovementOptions { significance?: HeldoutSignificanceOptions criticalDimensions?: readonly string[] criticalDimensionTolerance?: number - minHoldoutScore?: number + minFinalScore?: number activation?: AgentMemoryActivationDriver activationTimeoutMs?: number now?: () => Date } -export interface RunAgentMemoryImprovementResult { - lineage: Lineage +export interface RunAgentMemoryImprovementResult { + optimization: RunSerializedKnowledgeOptimizationResult baselineConfig: TConfig winnerConfig: TConfig baselineSurface: string @@ -174,8 +172,8 @@ export interface RunAgentMemoryImprovementResult { baselineSurfaceHash: string winnerSurfaceHash: string decision: AgentMemoryPromotionDecision + finalEvaluation: AgentMemoryFinalEvaluation activation: AgentMemoryActivation - holdout?: RunAgentMemoryExperimentResult totalCostUsd: number resultJsonPath: string } @@ -183,6 +181,7 @@ export interface RunAgentMemoryImprovementResult { export interface MemoryConfigScenario extends Scenario { kind: 'agent-memory-config-search' sequenceId: string + sequence: AgentMemorySequence } export const DEFAULT_CRITICAL_DIMENSIONS = [ @@ -191,6 +190,6 @@ export const DEFAULT_CRITICAL_DIMENSIONS = [ 'memory_event_recall', ] as const -export const MEMORY_IMPROVEMENT_IMPLEMENTATION_REF = 'agent-knowledge:memory-improvement:v2' +export const MEMORY_IMPROVEMENT_IMPLEMENTATION_REF = 'agent-knowledge:memory-improvement:v3' export type OwnedRunLease = OwnedAgentMemoryRunLease diff --git a/src/memory/improvement/validation.ts b/src/memory/improvement/validation.ts index 67bba42..45d77ae 100644 --- a/src/memory/improvement/validation.ts +++ b/src/memory/improvement/validation.ts @@ -1,26 +1,30 @@ +import type { JsonValue } from '@tangle-network/agent-eval/campaign' import type { AgentMemorySequence } from '../experiment' import { memorySequenceFingerprint } from './identity' import { DEFAULT_CRITICAL_DIMENSIONS, type RunAgentMemoryImprovementOptions } from './types' -export function assertMemoryImprovementOptions( +export function assertMemoryImprovementOptions( options: RunAgentMemoryImprovementOptions, ): void { for (const [name, value] of [ ['experimentId', options.experimentId], ['runDir', options.runDir], + ['improvementRef', options.improvementRef], ] as const) { if (typeof value !== 'string' || !value.trim()) { throw new Error(`memory improvement ${name} must be a non-empty string`) } } - if (typeof options.proposer?.kind !== 'string' || !options.proposer.kind.trim()) { - throw new Error('memory improvement proposer.kind must be a non-empty string') - } - if (typeof options.proposer?.propose !== 'function') { - throw new Error('memory improvement proposer.propose must be a function') + if ( + !options.method || + typeof options.method.name !== 'string' || + !options.method.name.trim() || + typeof options.method.optimize !== 'function' + ) { + throw new Error('memory improvement method must be a complete OptimizationMethod') } - if (options.governor !== undefined && typeof options.governor.decide !== 'function') { - throw new Error('memory improvement governor.decide must be a function') + if (typeof options.createCandidate !== 'function') { + throw new Error('memory improvement createCandidate must be a function') } if (options.activation !== undefined) { if (typeof options.activation.ref !== 'string' || !options.activation.ref.trim()) { @@ -33,29 +37,13 @@ export function assertMemoryImprovementOptions( throw new Error('memory improvement activation.compareAndSet must be a function') } } - for (const [name, proposer] of Object.entries(options.proposers ?? {})) { - if (!name.trim()) throw new Error('memory improvement proposer labels must be non-empty') - if ( - !proposer || - typeof proposer.kind !== 'string' || - !proposer.kind.trim() || - typeof proposer.propose !== 'function' - ) { - throw new Error(`memory improvement proposer '${name}' is invalid`) - } - } if (options.serializeConfig !== undefined && typeof options.serializeConfig !== 'function') { throw new Error('memory improvement serializeConfig must be a function') } if (options.parseConfig !== undefined && typeof options.parseConfig !== 'function') { throw new Error('memory improvement parseConfig must be a function') } - if (!Number.isSafeInteger(options.budget.maxSteps) || options.budget.maxSteps < 0) { - throw new Error('memory improvement budget.maxSteps must be a non-negative safe integer') - } for (const [name, value] of [ - ['populationSize', options.populationSize], - ['candidateConcurrency', options.candidateConcurrency], ['sequenceConcurrency', options.sequenceConcurrency], ['reps', options.reps], ['maxRecoveryAttempts', options.maxRecoveryAttempts], @@ -65,11 +53,33 @@ export function assertMemoryImprovementOptions( throw new Error(`memory improvement ${name} must be a positive safe integer`) } } + for (const [name, value] of [ + ['maxOptimizationCostUsd', options.maxOptimizationCostUsd], + ['maxFinalCostUsd', options.maxFinalCostUsd], + ] as const) { + if (value !== undefined && (!Number.isFinite(value) || value < 0)) { + throw new Error(`memory improvement ${name} must be a non-negative finite number`) + } + } if ( - options.maxTotalCostUsd !== undefined && - (!Number.isFinite(options.maxTotalCostUsd) || options.maxTotalCostUsd < 0) + options.maximumEvaluationCostUsd !== undefined && + (!Number.isFinite(options.maximumEvaluationCostUsd) || options.maximumEvaluationCostUsd <= 0) ) { - throw new Error('memory improvement maxTotalCostUsd must be a non-negative finite number') + throw new Error('memory improvement maximumEvaluationCostUsd must be a positive finite number') + } + if ( + ((options.maxOptimizationCostUsd ?? 0) > 0 || (options.maxFinalCostUsd ?? 0) > 0) && + options.maximumEvaluationCostUsd === undefined + ) { + throw new Error( + 'memory improvement maximumEvaluationCostUsd is required when a spend limit is configured', + ) + } + if ( + options.allowIncompleteCostAccounting !== undefined && + typeof options.allowIncompleteCostAccounting !== 'boolean' + ) { + throw new Error('memory improvement allowIncompleteCostAccounting must be boolean') } if ( options.activationTimeoutMs !== undefined && @@ -81,22 +91,9 @@ export function assertMemoryImprovementOptions( if (tolerance !== undefined && (!Number.isFinite(tolerance) || tolerance < 0 || tolerance > 1)) { throw new Error('memory improvement criticalDimensionTolerance must be between 0 and 1') } - const minimum = options.minHoldoutScore + const minimum = options.minFinalScore if (minimum !== undefined && (!Number.isFinite(minimum) || minimum < 0 || minimum > 1)) { - throw new Error('memory improvement minHoldoutScore must be between 0 and 1') - } - for (const seed of options.seeds) { - if ( - typeof seed.track !== 'string' || - !seed.track.trim() || - typeof seed.proposer !== 'string' || - !seed.proposer.trim() - ) { - throw new Error('memory improvement seeds require non-empty track and proposer values') - } - if (seed.vision !== undefined && (typeof seed.vision !== 'string' || !seed.vision.trim())) { - throw new Error('memory improvement seed vision must be a non-empty string when provided') - } + throw new Error('memory improvement minFinalScore must be between 0 and 1') } const dimensions = options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS if (dimensions.some((dimension) => typeof dimension !== 'string' || !dimension.trim())) { @@ -105,23 +102,46 @@ export function assertMemoryImprovementOptions( if (new Set(dimensions).size !== dimensions.length) { throw new Error('memory improvement criticalDimensions must be unique') } - assertDistinctSequenceContent(options.trainSequences, 'train') - assertDistinctSequenceContent(options.holdoutSequences, 'holdout') + assertIndependentSequences( + options.trainSequences, + options.selectionSequences, + options.finalSequences, + ) } -function assertDistinctSequenceContent( - sequences: readonly AgentMemorySequence[], - split: string, +function assertIndependentSequences( + train: readonly AgentMemorySequence[], + selection: readonly AgentMemorySequence[], + final: readonly AgentMemorySequence[], ): void { - const idsByFingerprint = new Map() - for (const sequence of sequences) { - const fingerprint = memorySequenceFingerprint(sequence) - const prior = idsByFingerprint.get(fingerprint) - if (prior) { - throw new Error( - `memory improvement ${split} histories duplicate content: ${prior}/${sequence.id}`, - ) + if (train.length === 0) throw new Error('memory improvement requires training sequences') + if (selection.length === 0) throw new Error('memory improvement requires selection sequences') + if (final.length < 2) { + throw new Error('memory improvement requires at least 2 final sequences') + } + const ids = new Map() + const content = new Map() + for (const [split, sequences] of [ + ['train', train], + ['selection', selection], + ['final', final], + ] as const) { + for (const sequence of sequences) { + const priorId = ids.get(sequence.id) + if (priorId) { + throw new Error( + `memory improvement ${priorId}/${split} sequences share id '${sequence.id}'`, + ) + } + ids.set(sequence.id, split) + const fingerprint = memorySequenceFingerprint(sequence) + const priorContent = content.get(fingerprint) + if (priorContent) { + throw new Error( + `memory improvement ${priorContent.split}/${split} histories duplicate content at '${priorContent.sequenceId}'/'${sequence.id}'`, + ) + } + content.set(fingerprint, { split, sequenceId: sequence.id }) } - idsByFingerprint.set(fingerprint, sequence.id) } } diff --git a/src/optimization.ts b/src/optimization.ts new file mode 100644 index 0000000..74890e3 --- /dev/null +++ b/src/optimization.ts @@ -0,0 +1,307 @@ +import { canonicalJson } from '@tangle-network/agent-eval' +import { + type CompareOptimizationMethodsOptions, + compareOptimizationMethods, + type DispatchContext, + type JsonValue, + type MutableSurface, + type OptimizationMethod, + type OptimizationMethodComparison, + type Scenario, + surfaceHash, +} from '@tangle-network/agent-eval/campaign' + +export interface SerializedCandidateCodec { + serialize(candidate: TCandidate): string + parse(surface: string): TCandidate +} + +export interface SerializedCandidate { + value: TCandidate + surface: string + surfaceHash: string +} + +type ComparisonOptions = Omit< + CompareOptimizationMethodsOptions, + | 'methods' + | 'baselineSurface' + | 'trainScenarios' + | 'selectionScenarios' + | 'testScenarios' + | 'dispatchWithSurface' +> + +export interface RunSerializedKnowledgeOptimizationOptions< + TCandidate extends JsonValue, + TScenario extends Scenario, + TArtifact, +> extends ComparisonOptions { + baseline: TCandidate + method: OptimizationMethod + trainScenarios: readonly TScenario[] + selectionScenarios: readonly TScenario[] + finalScenarios: readonly TScenario[] + dispatchCandidate(input: { + candidate: TCandidate + candidateSurface: string + candidateSurfaceHash: string + scenario: TScenario + context: DispatchContext + }): Promise + codec?: SerializedCandidateCodec + /** Detects duplicated cases whose IDs differ within or across data partitions. */ + scenarioFingerprint?: (scenario: TScenario) => string +} + +export interface RunSerializedKnowledgeOptimizationResult { + methodName: string + baseline: SerializedCandidate + winner: SerializedCandidate + comparison: OptimizationMethodComparison +} + +/** + * Runs one complete agent-eval method over a serialized knowledge candidate. + * The method receives train and selection data; agent-eval owns final scoring. + */ +export async function runSerializedKnowledgeOptimization< + TCandidate extends JsonValue, + TScenario extends Scenario, + TArtifact, +>( + options: RunSerializedKnowledgeOptimizationOptions, +): Promise> { + const codec = options.codec ?? jsonCandidateCodec() + const baseline = normalizeCandidate(options.baseline, codec, 'baseline') + assertPartitionContent( + options.trainScenarios, + options.selectionScenarios, + options.finalScenarios, + options.scenarioFingerprint ?? scenarioContentFingerprint, + ) + + const method = canonicalCandidateMethod(options.method, codec) + const { + baseline: _baseline, + method: _method, + trainScenarios, + selectionScenarios, + finalScenarios, + dispatchCandidate, + codec: _codec, + scenarioFingerprint: _scenarioFingerprint, + ...comparisonOptions + } = options + const optimizationRunOptions = { + ...(comparisonOptions.dispatchRef !== undefined + ? { dispatchRef: comparisonOptions.dispatchRef } + : {}), + ...(comparisonOptions.reps !== undefined ? { reps: comparisonOptions.reps } : {}), + ...(comparisonOptions.resumable !== undefined + ? { resumable: comparisonOptions.resumable } + : {}), + ...(comparisonOptions.labeledStore !== undefined + ? { labeledStore: comparisonOptions.labeledStore } + : {}), + ...(comparisonOptions.captureSource !== undefined + ? { captureSource: comparisonOptions.captureSource } + : {}), + ...(comparisonOptions.captureSourceVersionHash !== undefined + ? { captureSourceVersionHash: comparisonOptions.captureSourceVersionHash } + : {}), + ...(comparisonOptions.maxConcurrency !== undefined + ? { maxConcurrency: comparisonOptions.maxConcurrency } + : {}), + ...(comparisonOptions.dispatchTimeoutMs !== undefined + ? { dispatchTimeoutMs: comparisonOptions.dispatchTimeoutMs } + : {}), + ...(comparisonOptions.repo !== undefined ? { repo: comparisonOptions.repo } : {}), + ...(comparisonOptions.tracing !== undefined ? { tracing: comparisonOptions.tracing } : {}), + ...(comparisonOptions.expectUsage !== undefined + ? { expectUsage: comparisonOptions.expectUsage } + : {}), + ...(comparisonOptions.now !== undefined ? { now: comparisonOptions.now } : {}), + ...(comparisonOptions.buildTraceWriter !== undefined + ? { buildTraceWriter: comparisonOptions.buildTraceWriter } + : {}), + ...(comparisonOptions.storage !== undefined ? { storage: comparisonOptions.storage } : {}), + ...(comparisonOptions.cellPlacement !== undefined + ? { cellPlacement: comparisonOptions.cellPlacement } + : {}), + ...(comparisonOptions.optimizationRunOptions ?? {}), + } + const comparison = await compareOptimizationMethods({ + ...comparisonOptions, + optimizationRunOptions, + methods: [method], + baselineSurface: baseline.surface, + trainScenarios: trainScenarios.map((scenario) => structuredClone(scenario)), + selectionScenarios: selectionScenarios.map((scenario) => structuredClone(scenario)), + testScenarios: finalScenarios.map((scenario) => structuredClone(scenario)), + dispatchWithSurface: async (surface, scenario, context) => { + const candidate = normalizeSurface(surface, codec, 'candidate') + return dispatchCandidate({ + candidate: structuredClone(candidate.value), + candidateSurface: candidate.surface, + candidateSurfaceHash: candidate.surfaceHash, + scenario, + context, + }) + }, + }) + const winner = normalizeSurface(comparison.best.winnerSurface, codec, 'winner') + return { + methodName: options.method.name, + baseline, + winner, + comparison, + } +} + +export function jsonCandidateCodec< + TCandidate extends JsonValue, +>(): SerializedCandidateCodec { + return { + serialize: (candidate) => canonicalJson(candidate), + parse(surface) { + let parsed: unknown + try { + parsed = JSON.parse(surface) + } catch (error) { + throw new Error('serialized knowledge candidate is not valid JSON', { cause: error }) + } + return parsed as TCandidate + }, + } +} + +export function jsonObjectCandidateCodec< + TCandidate extends Record, +>(): SerializedCandidateCodec { + const codec = jsonCandidateCodec() + return { + serialize: codec.serialize, + parse(surface) { + const candidate = codec.parse(surface) + if (!candidate || typeof candidate !== 'object' || Array.isArray(candidate)) { + throw new Error('serialized knowledge candidate must be a JSON object') + } + return candidate + }, + } +} + +export function scenarioContentFingerprint(scenario: Scenario): string { + const { + id: _id, + tags: _tags, + split: _split, + splitTag: _splitTag, + ...content + } = scenario as Scenario & { split?: unknown; splitTag?: unknown } + return surfaceHash(canonicalJson(content as JsonValue)) +} + +function canonicalCandidateMethod< + TCandidate extends JsonValue, + TScenario extends Scenario, + TArtifact, +>( + method: OptimizationMethod, + codec: SerializedCandidateCodec, +): OptimizationMethod { + return { + name: method.name, + async optimize(input) { + const result = await method.optimize(input) + const winner = normalizeSurface(result.winnerSurface, codec, `${method.name} winner`) + return { + ...result, + winnerSurface: winner.surface, + } + }, + } +} + +function normalizeCandidate( + candidate: TCandidate, + codec: SerializedCandidateCodec, + label: string, +): SerializedCandidate { + let surface: string + try { + surface = codec.serialize(candidate) + } catch (error) { + throw new Error(`${label} serializer failed`, { cause: error }) + } + return normalizeSurface(surface, codec, label) +} + +function normalizeSurface( + surface: MutableSurface, + codec: SerializedCandidateCodec, + label: string, +): SerializedCandidate { + if (typeof surface !== 'string' || surface.trim().length === 0) { + throw new Error(`${label} must be a non-empty serialized string`) + } + let value: TCandidate + try { + value = codec.parse(surface) + } catch (error) { + const detail = error instanceof Error ? `: ${error.message}` : '' + throw new Error(`${label} parser failed${detail}`, { cause: error }) + } + let canonicalSurface: string + try { + canonicalSurface = codec.serialize(value) + } catch (error) { + throw new Error(`${label} serializer failed after parsing`, { cause: error }) + } + if (typeof canonicalSurface !== 'string' || canonicalSurface.trim().length === 0) { + throw new Error(`${label} serializer must return a non-empty string`) + } + const roundTrip = codec.serialize(codec.parse(canonicalSurface)) + if (roundTrip !== canonicalSurface) { + throw new Error(`${label} codec must round-trip to one canonical surface`) + } + return { + value: structuredClone(value), + surface: canonicalSurface, + surfaceHash: surfaceHash(canonicalSurface), + } +} + +function assertPartitionContent( + train: readonly TScenario[], + selection: readonly TScenario[], + final: readonly TScenario[], + fingerprint: ((scenario: TScenario) => string) | undefined, +): void { + if (!fingerprint) return + const owner = new Map() + for (const [name, scenarios] of [ + ['train', train], + ['selection', selection], + ['final', final], + ] as const) { + for (const scenario of scenarios) { + const identity = fingerprint(scenario) + if (typeof identity !== 'string' || identity.trim().length === 0) { + throw new Error(`scenarioFingerprint returned no identity for '${scenario.id}'`) + } + const prior = owner.get(identity) + if (prior) { + const scope = + prior.partition === name + ? `${name} partition duplicates` + : `${prior.partition}/${name} partitions duplicate` + throw new Error( + `serialized knowledge optimization ${scope} scenario content at '${prior.scenarioId}'/'${scenario.id}'`, + ) + } + owner.set(identity, { partition: name, scenarioId: scenario.id }) + } + } +} diff --git a/src/rag-improvement-loop.ts b/src/rag-improvement-loop.ts index b6a5915..bcedede 100644 --- a/src/rag-improvement-loop.ts +++ b/src/rag-improvement-loop.ts @@ -1,4 +1,9 @@ import type { JsonValue } from '@tangle-network/agent-eval/campaign' +import { + type RunRagOptimizationOptions, + type RunRagOptimizationResult, + runRagOptimization, +} from './rag-optimization' import { type KnowledgeResearchLoopDecision, type KnowledgeResearchLoopResult, @@ -9,9 +14,10 @@ import { type RunRetrievalImprovementLoopOptions, type RunRetrievalImprovementLoopResult, runRetrievalImprovementLoop, -} from './retrieval-eval' +} from './retrieval-optimization' export type RagKnowledgeImprovementPhase = + | 'rag-optimization' | 'retrieval-tuning' | 'gap-diagnosis' | 'knowledge-acquisition' @@ -56,6 +62,7 @@ export interface RagKnowledgeImprovementPhaseResult { export interface RagPhaseInputBase { goal: string phases: readonly RagKnowledgeImprovementPhaseResult[] + optimization?: RunRagOptimizationResult signal?: AbortSignal } @@ -117,6 +124,7 @@ export interface RagKnowledgeResearchOptions export interface RunRagKnowledgeImprovementLoopOptions { goal: string + optimization?: RunRagOptimizationOptions retrieval?: RunRetrievalImprovementLoopOptions diagnose?: (input: RagDiagnosisInput) => MaybePromise acquireKnowledge?: ( @@ -135,6 +143,7 @@ export interface RunRagKnowledgeImprovementLoopOptions { export interface RunRagKnowledgeImprovementLoopResult { goal: string phases: readonly RagKnowledgeImprovementPhaseResult[] + optimization?: RunRagOptimizationResult retrieval?: RunRetrievalImprovementLoopResult findings: readonly RagGapFinding[] acquisition?: KnowledgeResearchLoopDecision @@ -151,6 +160,7 @@ export async function runRagKnowledgeImprovementLoop( assertConfiguredRequiredPhases(options) const now = options.now ?? (() => new Date()) const phases: RagKnowledgeImprovementPhaseResult[] = [] + let optimization: RunRagOptimizationResult | undefined let retrieval: RunRetrievalImprovementLoopResult | undefined let findings: RagGapFinding[] = [] let acquisition: KnowledgeResearchLoopDecision | undefined @@ -158,6 +168,28 @@ export async function runRagKnowledgeImprovementLoop( let answerQuality: RagAnswerQualityResult | undefined let promotion: RagPromotionResult | undefined + if ( + phaseEnabled(options, 'rag-optimization') && + (options.optimization || + options.enabledPhases?.includes('rag-optimization') || + options.requiredPhases?.includes('rag-optimization')) + ) { + if (options.optimization) { + optimization = await runPhase( + phases, + now, + 'rag-optimization', + async () => { + assertNotAborted(options.signal) + return runRagOptimization(options.optimization!) + }, + summarizeRagOptimization, + ) + } else { + skipPhase(phases, now, 'rag-optimization', 'no full RAG optimization options provided') + } + } + if (phaseEnabled(options, 'retrieval-tuning')) { if (options.retrieval) { retrieval = await runPhase( @@ -187,6 +219,7 @@ export async function runRagKnowledgeImprovementLoop( return options.diagnose!({ goal: options.goal, phases, + optimization, signal: options.signal, retrieval, }) @@ -210,6 +243,7 @@ export async function runRagKnowledgeImprovementLoop( return options.acquireKnowledge!({ goal: options.goal, phases, + optimization, signal: options.signal, retrieval, findings, @@ -233,6 +267,7 @@ export async function runRagKnowledgeImprovementLoop( return options.updateKnowledge!({ goal: options.goal, phases, + optimization, signal: options.signal, retrieval, findings, @@ -268,6 +303,7 @@ export async function runRagKnowledgeImprovementLoop( return options.evaluateAnswers!({ goal: options.goal, phases, + optimization, signal: options.signal, retrieval, findings, @@ -294,6 +330,7 @@ export async function runRagKnowledgeImprovementLoop( return options.promote!({ goal: options.goal, phases, + optimization, signal: options.signal, retrieval, findings, @@ -312,6 +349,7 @@ export async function runRagKnowledgeImprovementLoop( return { goal: options.goal, phases, + optimization, retrieval, findings, acquisition, @@ -321,6 +359,10 @@ export async function runRagKnowledgeImprovementLoop( } } +function summarizeRagOptimization(result: RunRagOptimizationResult): string { + return `${result.methodName}; winner=${result.winner.surfaceHash}; final_lift=${result.comparison.best.lift.toFixed(3)}` +} + async function runKnowledgeResearchUpdate( options: RunRagKnowledgeImprovementLoopOptions, acquisition: KnowledgeResearchLoopDecision | undefined, @@ -400,7 +442,7 @@ function skipPhase( } function summarizeRetrievalResult(result: RunRetrievalImprovementLoopResult): string { - return `${result.candidates.length} candidate(s); winner=${JSON.stringify(result.winnerConfig)}` + return `${result.methodName}; winner=${result.winner.surfaceHash}; final_lift=${result.comparison.best.lift.toFixed(3)}` } function summarizeAcquisitionDecision(decision: KnowledgeResearchLoopDecision): string { @@ -445,6 +487,8 @@ function phaseConfigured( phase: RagKnowledgeImprovementPhase, ): boolean { switch (phase) { + case 'rag-optimization': + return Boolean(options.optimization) case 'retrieval-tuning': return Boolean(options.retrieval) case 'gap-diagnosis': @@ -462,6 +506,8 @@ function phaseConfigured( function requiredPhaseMessage(phase: RagKnowledgeImprovementPhase): string { switch (phase) { + case 'rag-optimization': + return 'required phase rag-optimization requires optimization options' case 'retrieval-tuning': return 'required phase retrieval-tuning requires retrieval options' case 'gap-diagnosis': diff --git a/src/rag-optimization.ts b/src/rag-optimization.ts new file mode 100644 index 0000000..0ae64de --- /dev/null +++ b/src/rag-optimization.ts @@ -0,0 +1,102 @@ +import type { + DispatchContext, + JsonValue, + JudgeConfig, + OptimizationMethod, +} from '@tangle-network/agent-eval/campaign' +import { + jsonObjectCandidateCodec, + type RunSerializedKnowledgeOptimizationOptions, + type RunSerializedKnowledgeOptimizationResult, + runSerializedKnowledgeOptimization, + scenarioContentFingerprint, +} from './optimization' +import type { RagAnswerEvalArtifact, RagAnswerEvalScenario } from './rag-eval' +import { ragAnswerQualityJudge } from './rag-eval' + +export type RagOptimizationConfig = Record + +type RagOptimizationBaseOptions = Omit< + RunSerializedKnowledgeOptimizationOptions< + RagOptimizationConfig, + RagAnswerEvalScenario, + RagAnswerEvalArtifact + >, + | 'baseline' + | 'method' + | 'trainScenarios' + | 'selectionScenarios' + | 'finalScenarios' + | 'dispatchCandidate' + | 'judges' + | 'codec' + | 'scenarioFingerprint' +> + +export interface RunRagOptimizationOptions extends RagOptimizationBaseOptions { + baseline: RagOptimizationConfig + method: OptimizationMethod + trainScenarios: readonly RagAnswerEvalScenario[] + selectionScenarios: readonly RagAnswerEvalScenario[] + finalScenarios: readonly RagAnswerEvalScenario[] + run(input: { + config: RagOptimizationConfig + configSurface: string + configSurfaceHash: string + scenario: RagAnswerEvalScenario + context: DispatchContext + }): Promise + judges?: readonly JudgeConfig[] +} + +export interface RunRagOptimizationResult + extends RunSerializedKnowledgeOptimizationResult { + baselineConfig: RagOptimizationConfig + winnerConfig: RagOptimizationConfig + trainScenarios: readonly RagAnswerEvalScenario[] + selectionScenarios: readonly RagAnswerEvalScenario[] + finalScenarios: readonly RagAnswerEvalScenario[] +} + +/** Optimizes retrieval and answer behavior together as one serialized RAG configuration. */ +export async function runRagOptimization( + options: RunRagOptimizationOptions, +): Promise { + const { + baseline, + method, + trainScenarios, + selectionScenarios, + finalScenarios, + run, + judges, + ...runOptions + } = options + const result = await runSerializedKnowledgeOptimization({ + ...runOptions, + baseline, + method, + trainScenarios, + selectionScenarios, + finalScenarios, + codec: jsonObjectCandidateCodec(), + judges: [...(judges ?? [ragAnswerQualityJudge()])], + scenarioFingerprint: scenarioContentFingerprint, + dispatchCandidate: ({ candidate, candidateSurface, candidateSurfaceHash, scenario, context }) => + run({ + config: candidate, + configSurface: candidateSurface, + configSurfaceHash: candidateSurfaceHash, + scenario, + context, + }), + }) + return { + ...result, + baselineConfig: result.baseline.value, + winnerConfig: result.winner.value, + trainScenarios: [...trainScenarios], + selectionScenarios: [...selectionScenarios], + finalScenarios: [...finalScenarios], + } +} diff --git a/src/retrieval-eval.ts b/src/retrieval-eval.ts index 14da2c9..abe0c0d 100644 --- a/src/retrieval-eval.ts +++ b/src/retrieval-eval.ts @@ -1,24 +1,15 @@ -import { - type DispatchContext, - type Gate, - type GenerationRecord, - heldOutGate, - type JsonValue, - type JudgeConfig, - type MutableSurface, - type ParameterCandidate, - parameterSweepProposer, - type RunImprovementLoopOptions, - type RunImprovementLoopResult, - runImprovementLoop, - type Scenario, - type SurfaceProposer, +import { canonicalJson } from '@tangle-network/agent-eval' +import type { + DispatchContext, + JsonValue, + JudgeConfig, + MutableSurface, + Scenario, } from '@tangle-network/agent-eval/campaign' import { searchKnowledge } from './search' import type { KnowledgeIndex, KnowledgeSearchResult } from './types' export type RetrievalConfig = Record -export type RetrievalParameterSearchSpace = Record export type RetrievalGoldTarget = | { kind: 'page'; pageId: string } @@ -115,64 +106,16 @@ export interface RetrievalRecallJudgeOptions { weights?: RetrievalMetricWeights } -export interface BuildRetrievalParameterCandidatesOptions { - baseline?: RetrievalConfig +export interface PartitionRetrievalScenariosOptions { + selectionFraction?: number + finalFraction?: number + seed?: number } -export interface RetrievalParameterSweepProposerOptions { - candidates?: readonly ParameterCandidate[] - searchSpace?: RetrievalParameterSearchSpace - baseline?: RetrievalConfig -} - -type RetrievalLoopBaseOptions = RunImprovementLoopOptions< - RetrievalEvalScenario, - RetrievalEvalArtifact -> - -export interface RunRetrievalImprovementLoopOptions { - baseline: RetrievalConfig - scenarios: readonly RetrievalEvalScenario[] - holdoutScenarios?: readonly RetrievalEvalScenario[] - index?: KnowledgeIndex - defaultK?: number - retrieve?: RetrievalEvalRetriever - candidates?: readonly ParameterCandidate[] - searchSpace?: RetrievalParameterSearchSpace - judges?: readonly JudgeConfig[] - gate?: Gate - metricWeights?: RetrievalMetricWeights - targetRecall?: number - holdoutFraction?: number - splitSeed?: number - deltaThreshold?: number - runDir?: RetrievalLoopBaseOptions['runDir'] - seed?: RetrievalLoopBaseOptions['seed'] - reps?: RetrievalLoopBaseOptions['reps'] - resumable?: RetrievalLoopBaseOptions['resumable'] - costCeiling?: RetrievalLoopBaseOptions['costCeiling'] - maxConcurrency?: RetrievalLoopBaseOptions['maxConcurrency'] - dispatchTimeoutMs?: RetrievalLoopBaseOptions['dispatchTimeoutMs'] - expectUsage?: RetrievalLoopBaseOptions['expectUsage'] - tracing?: RetrievalLoopBaseOptions['tracing'] - storage?: RetrievalLoopBaseOptions['storage'] - populationSize?: RetrievalLoopBaseOptions['populationSize'] - maxGenerations?: RetrievalLoopBaseOptions['maxGenerations'] - promoteTopK?: RetrievalLoopBaseOptions['promoteTopK'] - maxImprovementShots?: RetrievalLoopBaseOptions['maxImprovementShots'] - report?: RetrievalLoopBaseOptions['report'] - findings?: RetrievalLoopBaseOptions['findings'] - now?: RetrievalLoopBaseOptions['now'] -} - -export interface RunRetrievalImprovementLoopResult - extends RunImprovementLoopResult { - baselineConfig: RetrievalConfig - winnerConfig: RetrievalConfig - trainScenarios: readonly RetrievalEvalScenario[] - holdoutScenarios: readonly RetrievalEvalScenario[] - candidates: readonly ParameterCandidate[] - targetRecall?: number +export interface RetrievalScenarioPartitions { + trainScenarios: RetrievalEvalScenario[] + selectionScenarios: RetrievalEvalScenario[] + finalScenarios: RetrievalEvalScenario[] } export function retrievalConfigSurface(config: RetrievalConfig): string { @@ -322,201 +265,40 @@ export function scoreRetrievalArtifact( } } -export function buildRetrievalParameterCandidates( - searchSpace: RetrievalParameterSearchSpace, - options: BuildRetrievalParameterCandidatesOptions = {}, -): ParameterCandidate[] { - const candidates: ParameterCandidate[] = [] - for (const [path, values] of Object.entries(searchSpace).sort(([a], [b]) => a.localeCompare(b))) { - for (const value of values) { - if ( - options.baseline && - canonicalJson(getConfigPath(options.baseline, path)) === canonicalJson(value) - ) { - continue - } - candidates.push({ - label: `${path}=${formatCandidateValue(value)}`, - rationale: `Set retrieval config ${path} to ${formatCandidateValue(value)}`, - changes: [{ path, value }], - }) - } - } - return candidates -} - -export function retrievalParameterSweepProposer( - options: RetrievalParameterSweepProposerOptions, -): SurfaceProposer { - const candidates = - options.candidates ?? - (options.searchSpace - ? buildRetrievalParameterCandidates(options.searchSpace, { baseline: options.baseline }) - : []) - - if (candidates.length === 0) { - throw new Error('retrievalParameterSweepProposer requires at least one candidate') - } - - return parameterSweepProposer({ candidates }) -} - -export async function runRetrievalImprovementLoop( - options: RunRetrievalImprovementLoopOptions, -): Promise { - const split = splitRetrievalScenarios(options) - const candidates = resolveRetrievalCandidates(options) - const populationSize = options.populationSize ?? Math.max(1, Math.min(4, candidates.length)) - const maxGenerations = - options.maxGenerations ?? Math.max(1, Math.ceil(candidates.length / populationSize)) - const proposer = withTargetRecallStop( - retrievalParameterSweepProposer({ candidates }), - options.targetRecall, - ) - const gate = - options.gate ?? - heldOutGate({ - scenarios: split.holdoutScenarios, - deltaThreshold: options.deltaThreshold ?? 0.02, - }) - const result = await runImprovementLoop({ - baselineSurface: retrievalConfigSurface(options.baseline), - scenarios: split.trainScenarios, - holdoutScenarios: split.holdoutScenarios, - dispatchWithSurface: buildRetrievalEvalDispatch({ - index: options.index, - defaultK: options.defaultK, - retrieve: options.retrieve, - }), - judges: [...(options.judges ?? [retrievalRecallJudge({ weights: options.metricWeights })])], - proposer, - gate, - autoOnPromote: 'none', - runDir: options.runDir ?? '.agent-knowledge/retrieval-improvement', - seed: options.seed, - reps: options.reps, - resumable: options.resumable, - costCeiling: options.costCeiling, - maxConcurrency: options.maxConcurrency, - dispatchTimeoutMs: options.dispatchTimeoutMs, - expectUsage: options.expectUsage ?? 'off', - tracing: options.tracing, - storage: options.storage, - populationSize, - maxGenerations, - promoteTopK: options.promoteTopK, - maxImprovementShots: options.maxImprovementShots, - report: options.report, - findings: options.findings, - now: options.now, - }) - - return { - ...result, - baselineConfig: options.baseline, - winnerConfig: retrievalConfigFromSurface(result.winnerSurface), - trainScenarios: split.trainScenarios, - holdoutScenarios: split.holdoutScenarios, - candidates, - targetRecall: options.targetRecall, - } -} - -function splitRetrievalScenarios(options: RunRetrievalImprovementLoopOptions): { - trainScenarios: RetrievalEvalScenario[] - holdoutScenarios: RetrievalEvalScenario[] -} { - const scenarios = [...options.scenarios] - if (scenarios.length === 0) { - throw new Error('runRetrievalImprovementLoop requires at least one training scenario') - } - if (options.holdoutScenarios) { - const holdoutScenarios = [...options.holdoutScenarios] - if (holdoutScenarios.length === 0) { - throw new Error('runRetrievalImprovementLoop holdoutScenarios must not be empty') - } - return { trainScenarios: scenarios, holdoutScenarios } - } - - if (scenarios.length < 2) { - throw new Error( - 'runRetrievalImprovementLoop requires at least 2 scenarios when holdoutScenarios are not provided', - ) - } - const holdoutFraction = options.holdoutFraction ?? 0.3 - if (!Number.isFinite(holdoutFraction) || holdoutFraction <= 0 || holdoutFraction >= 1) { +export function partitionRetrievalScenarios( + scenarios: readonly RetrievalEvalScenario[], + options: PartitionRetrievalScenariosOptions = {}, +): RetrievalScenarioPartitions { + if (scenarios.length < 4) { throw new Error( - `runRetrievalImprovementLoop holdoutFraction must be > 0 and < 1, got ${String(holdoutFraction)}`, + 'partitionRetrievalScenarios requires at least 4 scenarios for non-empty train and selection partitions plus 2 final scenarios', ) } - const shuffled = seededShuffle(scenarios, options.splitSeed ?? options.seed ?? 42) - const holdoutCount = Math.min( - shuffled.length - 1, - Math.max(1, Math.round(shuffled.length * holdoutFraction)), - ) - const splitIndex = shuffled.length - holdoutCount - return { - trainScenarios: shuffled.slice(0, splitIndex), - holdoutScenarios: shuffled.slice(splitIndex), - } -} - -function resolveRetrievalCandidates( - options: RunRetrievalImprovementLoopOptions, -): ParameterCandidate[] { - const candidates = options.candidates - ? [...options.candidates] - : options.searchSpace - ? buildRetrievalParameterCandidates(options.searchSpace, { baseline: options.baseline }) - : [] - - if (candidates.length === 0) { - throw new Error('runRetrievalImprovementLoop requires candidates or searchSpace') - } - return candidates -} - -function withTargetRecallStop( - proposer: SurfaceProposer, - targetRecall: number | undefined, -): SurfaceProposer { - if (targetRecall === undefined) { - return proposer + const selectionFraction = options.selectionFraction ?? 0.2 + const finalFraction = options.finalFraction ?? 0.2 + assertPartitionFraction(selectionFraction, 'selectionFraction') + assertPartitionFraction(finalFraction, 'finalFraction') + if (selectionFraction + finalFraction >= 1) { + throw new Error('selectionFraction + finalFraction must be less than 1') } - if (!Number.isFinite(targetRecall) || targetRecall < 0 || targetRecall > 1) { - throw new Error(`targetRecall must be between 0 and 1, got ${String(targetRecall)}`) + const shuffled = seededShuffle(scenarios, options.seed ?? 42) + const finalCount = Math.max(2, Math.round(shuffled.length * finalFraction)) + const selectionCount = Math.max(1, Math.round(shuffled.length * selectionFraction)) + if (finalCount + selectionCount >= shuffled.length) { + throw new Error('retrieval scenario fractions leave no training scenarios') } + const trainEnd = shuffled.length - selectionCount - finalCount return { - kind: `${proposer.kind}:target-recall`, - propose: (context) => proposer.propose(context), - decide(args) { - const baseDecision = proposer.decide?.(args) - if (baseDecision?.stop) { - return baseDecision - } - const bestRecall = bestObservedRecall(args.history) - if (bestRecall >= targetRecall) { - return { - stop: true, - reason: `target recall ${targetRecall} reached with train recall ${bestRecall}`, - } - } - return { stop: false } - }, + trainScenarios: shuffled.slice(0, trainEnd), + selectionScenarios: shuffled.slice(trainEnd, trainEnd + selectionCount), + finalScenarios: shuffled.slice(trainEnd + selectionCount), } } -function bestObservedRecall(history: GenerationRecord[]): number { - let best = Number.NEGATIVE_INFINITY - for (const generation of history) { - for (const candidate of generation.candidates) { - const recall = candidate.dimensions.recall - if (recall !== undefined && Number.isFinite(recall) && recall > best) { - best = recall - } - } +function assertPartitionFraction(value: number, name: string): void { + if (!Number.isFinite(value) || value <= 0 || value >= 1) { + throw new Error(`${name} must be greater than 0 and less than 1`) } - return best } function retrievalK( @@ -663,40 +445,6 @@ function normalizeWeight(value: number | undefined): number { return value } -function getConfigPath(config: RetrievalConfig, path: string): JsonValue | undefined { - let current: JsonValue | undefined = config - for (const part of path.split('.')) { - if (!isJsonObject(current)) { - return undefined - } - current = current[part] - } - return current -} - -function formatCandidateValue(value: JsonValue): string { - if (typeof value === 'string') { - return value - } - return canonicalJson(value) -} - -function canonicalJson(value: JsonValue | undefined): string { - if (value === undefined) { - return 'undefined' - } - if (Array.isArray(value)) { - return `[${value.map(canonicalJson).join(',')}]` - } - if (isJsonObject(value)) { - const entries = Object.entries(value) - .sort(([a], [b]) => a.localeCompare(b)) - .map(([key, child]) => `${JSON.stringify(key)}:${canonicalJson(child)}`) - return `{${entries.join(',')}}` - } - return JSON.stringify(value) -} - function isJsonObject(value: unknown): value is Record { return Boolean(value) && typeof value === 'object' && !Array.isArray(value) } diff --git a/src/retrieval-optimization.ts b/src/retrieval-optimization.ts new file mode 100644 index 0000000..806531d --- /dev/null +++ b/src/retrieval-optimization.ts @@ -0,0 +1,457 @@ +import { canonicalJson } from '@tangle-network/agent-eval' +import { + type CampaignResult, + campaignMeanComposite, + costFromLedgerSummary, + createRunCostLedger, + fsCampaignStorage, + type JsonValue, + type JudgeConfig, + type OptimizationMethod, + type OptimizationMethodRunOptions, + runCampaign, + surfaceHash, +} from '@tangle-network/agent-eval/campaign' +import { + jsonObjectCandidateCodec, + type RunSerializedKnowledgeOptimizationOptions, + type RunSerializedKnowledgeOptimizationResult, + runSerializedKnowledgeOptimization, +} from './optimization' +import { + buildRetrievalEvalDispatch, + type RetrievalConfig, + type RetrievalEvalArtifact, + type RetrievalEvalRetriever, + type RetrievalEvalScenario, + type RetrievalMetricWeights, + retrievalConfigFromSurface, + retrievalConfigSurface, + retrievalRecallJudge, +} from './retrieval-eval' +import type { KnowledgeIndex } from './types' + +export type RetrievalParameterSearchSpace = Record + +export interface BuildBoundedRetrievalConfigsOptions { + baseline: RetrievalConfig + maxConfigurations?: number +} + +export interface BoundedRetrievalConfigMethodOptions { + name?: string + configurations?: readonly RetrievalConfig[] + searchSpace?: RetrievalParameterSearchSpace + maxConfigurations?: number + /** Configuration campaigns run in parallel. Default 4. */ + configurationConcurrency?: number + targetRecall?: number + runOptions?: OptimizationMethodRunOptions +} + +type RetrievalOptimizationBaseOptions = Omit< + RunSerializedKnowledgeOptimizationOptions< + RetrievalConfig, + RetrievalEvalScenario, + RetrievalEvalArtifact + >, + | 'baseline' + | 'method' + | 'trainScenarios' + | 'selectionScenarios' + | 'finalScenarios' + | 'dispatchCandidate' + | 'judges' + | 'codec' + | 'scenarioFingerprint' +> + +export interface RunRetrievalImprovementLoopOptions extends RetrievalOptimizationBaseOptions { + baseline: RetrievalConfig + trainScenarios: readonly RetrievalEvalScenario[] + selectionScenarios: readonly RetrievalEvalScenario[] + finalScenarios: readonly RetrievalEvalScenario[] + method?: OptimizationMethod + index?: KnowledgeIndex + defaultK?: number + retrieve?: RetrievalEvalRetriever + configurations?: readonly RetrievalConfig[] + searchSpace?: RetrievalParameterSearchSpace + boundedSearch?: Omit + judges?: readonly JudgeConfig[] + metricWeights?: RetrievalMetricWeights +} + +export interface RunRetrievalImprovementLoopResult + extends RunSerializedKnowledgeOptimizationResult { + baselineConfig: RetrievalConfig + winnerConfig: RetrievalConfig + trainScenarios: readonly RetrievalEvalScenario[] + selectionScenarios: readonly RetrievalEvalScenario[] + finalScenarios: readonly RetrievalEvalScenario[] + boundedConfigurations?: readonly RetrievalConfig[] +} + +export function buildBoundedRetrievalConfigs( + searchSpace: RetrievalParameterSearchSpace, + options: BuildBoundedRetrievalConfigsOptions, +): RetrievalConfig[] { + const maxConfigurations = options.maxConfigurations ?? 128 + if (!Number.isSafeInteger(maxConfigurations) || maxConfigurations <= 0) { + throw new Error('maxConfigurations must be a positive safe integer') + } + const entries = Object.entries(searchSpace).sort(([left], [right]) => left.localeCompare(right)) + if (entries.length === 0) { + throw new Error('bounded retrieval search requires at least one parameter') + } + let total = 1 + for (const [path, values] of entries) { + assertSafeConfigPath(path) + if (values.length === 0) { + throw new Error(`bounded retrieval search parameter '${path}' has no values`) + } + total *= values.length + if (!Number.isSafeInteger(total) || total > maxConfigurations) { + throw new Error( + `bounded retrieval search expands to more than ${maxConfigurations} configurations; use an OptimizationMethod for larger spaces`, + ) + } + } + + let configurations: RetrievalConfig[] = [structuredClone(options.baseline)] + for (const [path, values] of entries) { + configurations = configurations.flatMap((config) => + values.map((value) => setConfigPath(config, path, value)), + ) + } + const baselineSurface = retrievalConfigSurface(options.baseline) + const unique = new Map() + for (const configuration of configurations) { + const surface = retrievalConfigSurface(configuration) + if (surface !== baselineSurface) unique.set(surface, configuration) + } + return [...unique.values()] +} + +/** + * Exhaustively checks a small, finite retrieval grid through agent-eval. + * Larger or generative spaces should supply an official OptimizationMethod. + */ +export function boundedRetrievalConfigMethod( + options: BoundedRetrievalConfigMethodOptions, +): OptimizationMethod { + if (options.configurations && options.searchSpace) { + throw new Error('bounded retrieval method accepts configurations or searchSpace, not both') + } + if (!options.configurations && !options.searchSpace) { + throw new Error('bounded retrieval method requires configurations or searchSpace') + } + const name = options.name ?? 'bounded-retrieval-config-search' + return { + name, + async optimize(input) { + const baseline = retrievalConfigFromSurface(input.baselineSurface) + const maxConfigurations = options.maxConfigurations ?? 128 + const configurations = options.configurations + ? normalizeBoundedConfigurations(options.configurations, baseline, maxConfigurations) + : buildBoundedRetrievalConfigs(options.searchSpace!, { + baseline, + maxConfigurations, + }) + if (configurations.length === 0) { + return { + winnerSurface: input.baselineSurface, + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + durationMs: 0, + } + } + assertTargetRecall(options.targetRecall) + const concurrency = options.configurationConcurrency ?? 4 + assertConfigurationConcurrency(concurrency) + const startedAt = Date.now() + const runOptions = { + ...input.runOptions, + ...(options.runOptions ?? {}), + } + const storage = runOptions.storage ?? fsCampaignStorage() + const costLedger = createRunCostLedger({ + storage, + runDir: `${input.runDir}/bounded-cost`, + costCeilingUsd: runOptions.costCeiling, + }) + const surfaces = [ + input.baselineSurface, + ...configurations.map(retrievalConfigSurface), + ] as string[] + let winner: BoundedRetrievalMeasurement | undefined + for (let offset = 0; offset < surfaces.length; offset += concurrency) { + const batch = await Promise.all( + surfaces + .slice(offset, offset + concurrency) + .map((surface) => + measureBoundedRetrievalSurface(surface, input, runOptions, storage, costLedger), + ), + ) + for (const measurement of batch) { + if (!measurement.complete) continue + if (!winner || measurement.composite > winner.composite) winner = measurement + } + if ( + options.targetRecall !== undefined && + winner?.recall !== undefined && + winner.recall >= options.targetRecall + ) { + break + } + } + if (!winner) { + throw new Error('bounded retrieval search produced no complete selection measurement') + } + return { + winnerSurface: winner.surface, + cost: costFromLedgerSummary(costLedger.summary()), + durationMs: Date.now() - startedAt, + } + }, + } +} + +export async function runRetrievalImprovementLoop( + options: RunRetrievalImprovementLoopOptions, +): Promise { + if (options.method && (options.configurations || options.searchSpace)) { + throw new Error( + 'runRetrievalImprovementLoop accepts method or bounded configurations, not both', + ) + } + const boundedConfigurations = options.method ? undefined : resolveBoundedConfigurations(options) + const method = + options.method ?? + boundedRetrievalConfigMethod({ + ...(options.boundedSearch ?? {}), + ...(options.configurations + ? { configurations: options.configurations } + : { searchSpace: options.searchSpace! }), + }) + const dispatch = buildRetrievalEvalDispatch({ + index: options.index, + defaultK: options.defaultK, + retrieve: options.retrieve, + }) + const { + baseline, + trainScenarios, + selectionScenarios, + finalScenarios, + method: _method, + index: _index, + defaultK: _defaultK, + retrieve: _retrieve, + configurations: _configurations, + searchSpace: _searchSpace, + boundedSearch: _boundedSearch, + judges, + metricWeights, + ...runOptions + } = options + const result = await runSerializedKnowledgeOptimization({ + ...runOptions, + baseline, + method, + trainScenarios, + selectionScenarios, + finalScenarios, + codec: jsonObjectCandidateCodec(), + judges: [...(judges ?? [retrievalRecallJudge({ weights: metricWeights })])], + scenarioFingerprint: retrievalScenarioFingerprint, + dispatchCandidate: ({ candidateSurface, scenario, context }) => + dispatch(candidateSurface, scenario, context), + }) + return { + ...result, + baselineConfig: result.baseline.value, + winnerConfig: result.winner.value, + trainScenarios: [...trainScenarios], + selectionScenarios: [...selectionScenarios], + finalScenarios: [...finalScenarios], + ...(boundedConfigurations ? { boundedConfigurations } : {}), + } +} + +function resolveBoundedConfigurations( + options: RunRetrievalImprovementLoopOptions, +): RetrievalConfig[] { + if (options.configurations && options.searchSpace) { + throw new Error('runRetrievalImprovementLoop accepts configurations or searchSpace, not both') + } + if (options.configurations) { + return normalizeBoundedConfigurations( + options.configurations, + options.baseline, + options.boundedSearch?.maxConfigurations ?? 128, + ) + } + if (options.searchSpace) { + return buildBoundedRetrievalConfigs(options.searchSpace, { + baseline: options.baseline, + maxConfigurations: options.boundedSearch?.maxConfigurations, + }) + } + throw new Error('runRetrievalImprovementLoop requires method, configurations, or searchSpace') +} + +interface BoundedRetrievalMeasurement { + surface: string + composite: number + recall?: number + complete: boolean +} + +async function measureBoundedRetrievalSurface( + surface: string, + input: Parameters< + OptimizationMethod['optimize'] + >[0], + runOptions: OptimizationMethodRunOptions, + storage: ReturnType, + costLedger: ReturnType, +): Promise { + const hash = surfaceHash(surface) + const campaign = await runCampaign({ + ...runOptions, + scenarios: [...input.selectionScenarios], + dispatch: (scenario, context) => input.dispatchWithSurface(surface, scenario, context), + dispatchRef: `${runOptions.dispatchRef ?? 'bounded-retrieval'}:${hash}`, + judges: [...input.judges], + runDir: `${input.runDir}/bounded-candidates/${hash}`, + seed: input.seed, + storage, + costLedger, + costPhase: `${runOptions.costPhase ?? 'bounded-retrieval'}.${hash}`, + }) + const recall = campaignDimensionMean(campaign, 'recall') + return { + surface, + composite: campaignMeanComposite(campaign), + ...(recall !== undefined ? { recall } : {}), + complete: campaignIsComplete(campaign), + } +} + +function assertTargetRecall(targetRecall: number | undefined): void { + if ( + targetRecall !== undefined && + (!Number.isFinite(targetRecall) || targetRecall < 0 || targetRecall > 1) + ) { + throw new Error(`targetRecall must be between 0 and 1, got ${String(targetRecall)}`) + } +} + +function assertConfigurationConcurrency(value: number): void { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error('bounded retrieval configurationConcurrency must be a positive safe integer') + } +} + +function campaignIsComplete( + campaign: CampaignResult, +): boolean { + const expectedCells = campaign.scenarios.length * campaign.reps + return ( + campaign.cells.length === expectedCells && + campaign.cells.every( + (cell) => + !cell.error && + Object.values(cell.judgeScores).some( + (score) => score.composite !== undefined && Number.isFinite(score.composite), + ), + ) + ) +} + +function campaignDimensionMean( + campaign: CampaignResult, + dimension: string, +): number | undefined { + const values: number[] = [] + for (const cell of campaign.cells) { + if (cell.error) continue + for (const score of Object.values(cell.judgeScores)) { + const value = score.dimensions[dimension] + if (value !== undefined && Number.isFinite(value)) values.push(value) + } + } + if (values.length === 0) return undefined + return values.reduce((sum, value) => sum + value, 0) / values.length +} + +function normalizeBoundedConfigurations( + configurations: readonly RetrievalConfig[], + baseline: RetrievalConfig, + maxConfigurations: number, +): RetrievalConfig[] { + if (!Number.isSafeInteger(maxConfigurations) || maxConfigurations <= 0) { + throw new Error('maxConfigurations must be a positive safe integer') + } + if (configurations.length > maxConfigurations) { + throw new Error( + `bounded retrieval search received ${configurations.length} configurations, exceeding maxConfigurations=${maxConfigurations}`, + ) + } + const baselineSurface = retrievalConfigSurface(baseline) + const unique = new Map() + for (const configuration of configurations) { + const surface = retrievalConfigSurface(configuration) + if (surface !== baselineSurface) unique.set(surface, structuredClone(configuration)) + } + return [...unique.values()] +} + +function setConfigPath(config: RetrievalConfig, path: string, value: JsonValue): RetrievalConfig { + assertSafeConfigPath(path) + const result = structuredClone(config) + const parts = path.split('.') + let current: Record = result + for (const part of parts.slice(0, -1)) { + const child = current[part] + if (child !== undefined && !isJsonObject(child)) { + throw new Error(`retrieval config path '${path}' crosses non-object '${part}'`) + } + const next = child ? structuredClone(child) : {} + current[part] = next + current = next + } + current[parts.at(-1)!] = structuredClone(value) + return result +} + +function assertSafeConfigPath(path: string): void { + const parts = path.split('.') + if ( + parts.length === 0 || + parts.some( + (part) => + !part || + part === '__proto__' || + part === 'prototype' || + part === 'constructor' || + !/^[A-Za-z0-9_-]+$/.test(part), + ) + ) { + throw new Error(`unsafe retrieval config path '${path}'`) + } +} + +function retrievalScenarioFingerprint(scenario: RetrievalEvalScenario): string { + return surfaceHash( + canonicalJson({ + query: scenario.query, + expected: scenario.expected, + k: scenario.k ?? null, + } as JsonValue), + ) +} + +function isJsonObject(value: unknown): value is Record { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) +} diff --git a/tests/kb-improvement/candidate.test.ts b/tests/kb-improvement/candidate.test.ts index cc0d407..4b2a013 100644 --- a/tests/kb-improvement/candidate.test.ts +++ b/tests/kb-improvement/candidate.test.ts @@ -323,7 +323,7 @@ describe('improveKnowledgeBase', () => { }), retrieval: { baseline: { k: 1 }, - scenarios: [ + trainScenarios: [ { id: 'q-train', kind: 'retrieval-eval', @@ -331,11 +331,37 @@ describe('improveKnowledgeBase', () => { expected: { kind: 'page', pageId: 'refund-policy' }, }, ], - holdoutScenarios: [ + selectionScenarios: [ { - id: 'q-holdout', + id: 'q-selection-a', kind: 'retrieval-eval', - query: 'billing refund', + query: 'selection a billing refund', + expected: { kind: 'page', pageId: 'refund-policy' }, + }, + { + id: 'q-selection-b', + kind: 'retrieval-eval', + query: 'selection b billing refund', + expected: { kind: 'page', pageId: 'refund-policy' }, + }, + { + id: 'q-selection-c', + kind: 'retrieval-eval', + query: 'selection c billing refund', + expected: { kind: 'page', pageId: 'refund-policy' }, + }, + ], + finalScenarios: [ + { + id: 'q-final-a', + kind: 'retrieval-eval', + query: 'final a billing refund', + expected: { kind: 'page', pageId: 'refund-policy' }, + }, + { + id: 'q-final-b', + kind: 'retrieval-eval', + query: 'final b billing refund', expected: { kind: 'page', pageId: 'refund-policy' }, }, ], @@ -348,11 +374,9 @@ describe('improveKnowledgeBase', () => { : []), ], }), - targetRecall: 1, - deltaThreshold: 0.01, - populationSize: 1, - maxGenerations: 1, + boundedSearch: { targetRecall: 1, configurationConcurrency: 1 }, expectUsage: 'off', + resamples: 200, }, }) diff --git a/tests/kb-improvement/optimization.test.ts b/tests/kb-improvement/optimization.test.ts new file mode 100644 index 0000000..eba3b6b --- /dev/null +++ b/tests/kb-improvement/optimization.test.ts @@ -0,0 +1,268 @@ +import { mkdir, readFile, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { + inMemoryCampaignStorage, + type OptimizationMethod, + type Scenario, +} from '@tangle-network/agent-eval/campaign' +import { describe, expect, it } from 'vitest' +import { + hashKnowledgeBase, + improveKnowledgeBase, + optimizeKnowledgeBasePolicy, + type RagAnswerEvalArtifact, + type RagAnswerEvalScenario, + scenarioContentFingerprint, +} from '../../src/index' +import { mutableCandidateRoot, passingMetric, withKb } from '../support/kb-improvement' + +interface PolicyScenario extends Scenario { + kind: 'kb-policy-eval' + prompt: string +} + +interface PolicyArtifact { + score: number +} + +type Policy = { evidence: 'none' | 'required'; maxSources: number } + +describe('optimizeKnowledgeBasePolicy', () => { + it('runs full RAG evaluation against the isolated candidate KB', async () => { + await withKb(async (root) => { + const method: OptimizationMethod = { + name: 'fixture-candidate-rag-method', + async optimize(input) { + expect('testScenarios' in input).toBe(false) + return { + winnerSurface: '{"mode":"grounded"}', + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + } + }, + } + const scenario = (id: string): RagAnswerEvalScenario => ({ + id, + kind: 'rag-answer-eval', + query: `${id} candidate policy`, + }) + const seenCandidateRoots = new Set() + + const result = await improveKnowledgeBase({ + root, + goal: 'Evaluate RAG against candidate knowledge', + runId: 'candidate-rag-optimization', + async updateKnowledge({ candidateRoot }) { + const path = join(candidateRoot, 'knowledge', 'candidate-policy.md') + await mkdir(dirname(path), { recursive: true }) + await writeFile( + path, + [ + '---', + 'id: candidate-policy', + 'title: Candidate Policy', + '---', + '# Candidate Policy', + 'Candidate-only evidence.', + ].join('\n'), + ) + return { applied: true, summary: 'wrote candidate knowledge' } + }, + ragOptimization: { + baseline: { mode: 'unsupported' }, + method, + trainScenarios: [scenario('candidate-rag-train')], + selectionScenarios: [scenario('candidate-rag-selection')], + finalScenarios: [scenario('candidate-rag-final-a'), scenario('candidate-rag-final-b')], + async run({ + config, + scenario: item, + baseHash, + baselineRoot, + candidateRoot, + candidateIndex, + }) { + seenCandidateRoots.add(candidateRoot) + expect(candidateRoot).not.toBe(root) + expect(baselineRoot).not.toBe(root) + expect(await hashKnowledgeBase(baselineRoot)).toBe(baseHash) + expect(candidateIndex.pages.map((page) => page.id)).toContain('candidate-policy') + const score = config.mode === 'grounded' ? 1 : 0 + return { + query: item.query, + answer: score ? 'Candidate-only evidence.' : 'Unsupported answer.', + contexts: [], + metadata: { score }, + } + }, + judges: [ + { + name: 'candidate-rag-quality', + dimensions: [{ key: 'quality', description: 'candidate RAG quality' }], + score: ({ artifact }) => { + const score = Number(artifact.metadata?.score ?? 0) + return { composite: score, dimensions: { quality: score } } + }, + }, + ], + storage: inMemoryCampaignStorage(), + expectUsage: 'off', + resamples: 200, + }, + requiredPhases: ['rag-optimization'], + evaluate: passingMetric, + }) + + expect(seenCandidateRoots.size).toBe(1) + expect(result.lifecycle?.optimization?.winner.value).toEqual({ mode: 'grounded' }) + expect(result.lifecycle?.optimization?.comparison.testScenarioIds).toEqual([ + 'candidate-rag-final-a', + 'candidate-rag-final-b', + ]) + }) + }) + + it('runs a complete method and applies only the exact winner to an isolated candidate', async () => { + await withKb(async (root) => { + const methodInputs: string[][] = [] + const method: OptimizationMethod = { + name: 'fixture-kb-policy-method', + async optimize(input) { + methodInputs.push([ + ...input.trainScenarios.map((scenario) => scenario.id), + ...input.selectionScenarios.map((scenario) => scenario.id), + ]) + expect('testScenarios' in input).toBe(false) + return { + winnerSurface: '{"evidence":"required","maxSources":4}', + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + } + }, + } + const scenario = (id: string): PolicyScenario => ({ + id, + kind: 'kb-policy-eval', + prompt: `${id} source-backed update`, + }) + + const result = await optimizeKnowledgeBasePolicy({ + root, + goal: 'Select a source-backed KB maintenance policy', + baselinePolicy: { evidence: 'none', maxSources: 1 }, + method, + trainScenarios: [scenario('policy-train')], + selectionScenarios: [scenario('policy-selection')], + finalScenarios: [scenario('policy-final-a'), scenario('policy-final-b')], + policyApplicationRef: 'write-maintenance-policy:v1', + dispatchCandidate: async ({ candidate }) => ({ + score: candidate.evidence === 'required' && candidate.maxSources >= 2 ? 1 : 0, + }), + judges: [ + { + name: 'policy-quality', + dimensions: [{ key: 'quality', description: 'policy satisfies evidence rules' }], + score: ({ artifact }) => ({ + composite: artifact.score, + dimensions: { quality: artifact.score }, + }), + }, + ], + scenarioFingerprint: scenarioContentFingerprint, + runDir: 'memory://kb-policy-optimization-test', + storage: inMemoryCampaignStorage(), + expectUsage: 'off', + resamples: 200, + candidate: { evaluate: passingMetric }, + async applyPolicy({ candidateRoot, policy, policySurfaceHash, optimizationMethod }) { + expect(policy).toEqual({ evidence: 'required', maxSources: 4 }) + expect(optimizationMethod).toBe('fixture-kb-policy-method') + const path = join(candidateRoot, 'knowledge', 'maintenance-policy.md') + await mkdir(dirname(path), { recursive: true }) + await writeFile( + path, + `# Maintenance Policy\n\n${policySurfaceHash}: require source evidence.\n`, + ) + return { applied: true, summary: 'wrote selected maintenance policy' } + }, + }) + + expect(methodInputs).toEqual([['policy-train', 'policy-selection']]) + expect(result.optimization.winner.value).toEqual({ + evidence: 'required', + maxSources: 4, + }) + expect(result.optimization.comparison.testScenarioIds).toEqual([ + 'policy-final-a', + 'policy-final-b', + ]) + expect(result.improvement.state.status).toBe('candidate-ready') + expect(result.improvement.promoted).toBe(false) + expect(result.improvement.lifecycle?.knowledgeUpdate?.metadata?.optimization).toEqual({ + method: 'fixture-kb-policy-method', + policySurfaceHash: result.optimization.winner.surfaceHash, + policyApplicationRef: 'write-maintenance-policy:v1', + }) + await expect( + readFile(join(root, 'knowledge', 'maintenance-policy.md'), 'utf8'), + ).rejects.toMatchObject({ code: 'ENOENT' }) + + const candidateRoot = mutableCandidateRoot(root, result.improvement) + await expect( + readFile(join(candidateRoot, 'knowledge', 'maintenance-policy.md'), 'utf8'), + ).resolves.toContain(result.optimization.winner.surfaceHash) + }) + }) + + it('does not materialize a policy winner after the live knowledge base changes', async () => { + await withKb(async (root) => { + let applyCalls = 0 + const method: OptimizationMethod = { + name: 'concurrent-kb-change', + async optimize() { + await writeFile(join(root, 'knowledge', 'concurrent-change.md'), '# Concurrent change\n') + return { + winnerSurface: '{"evidence":"required","maxSources":2}', + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + } + }, + } + const scenario = (id: string): PolicyScenario => ({ + id, + kind: 'kb-policy-eval', + prompt: `${id} policy`, + }) + + await expect( + optimizeKnowledgeBasePolicy({ + root, + goal: 'Reject a policy measured against changing knowledge', + baselinePolicy: { evidence: 'none', maxSources: 1 }, + method, + trainScenarios: [scenario('changing-train')], + selectionScenarios: [scenario('changing-selection')], + finalScenarios: [scenario('changing-final-a'), scenario('changing-final-b')], + policyApplicationRef: 'changing-policy:v1', + dispatchCandidate: async () => ({ score: 1 }), + judges: [ + { + name: 'changing-policy-quality', + dimensions: [{ key: 'quality', description: 'policy quality' }], + score: ({ artifact }) => ({ + composite: artifact.score, + dimensions: { quality: artifact.score }, + }), + }, + ], + runDir: 'memory://changing-kb-policy-test', + storage: inMemoryCampaignStorage(), + expectUsage: 'off', + resamples: 200, + async applyPolicy() { + applyCalls += 1 + return { applied: true, summary: 'must not run' } + }, + }), + ).rejects.toThrow('knowledge base changed during policy optimization') + expect(applyCalls).toBe(0) + }) + }) +}) diff --git a/tests/memory/improvement.test.ts b/tests/memory/improvement.test.ts index af1d8cd..a8c1329 100644 --- a/tests/memory/improvement.test.ts +++ b/tests/memory/improvement.test.ts @@ -1,195 +1,120 @@ +import { canonicalJson } from '@tangle-network/agent-eval' import { - createRunCostLedger, + campaignMeanComposite, inMemoryCampaignStorage, - type SurfaceProposer, + type JsonValue, + type OptimizationMethod, + runCampaign, + surfaceHash, } from '@tangle-network/agent-eval/campaign' import { describe, expect, it } from 'vitest' -import { runAgentMemoryImprovement as runAgentMemoryImprovementRaw } from '../../src/memory/index' +import { stableId } from '../../src/ids' +import { + type AgentMemorySequence, + type AgentMemorySequenceArtifact, + type MemoryConfigScenario, + type RunAgentMemoryImprovementOptions, + runAgentMemoryImprovement as runAgentMemoryImprovementRaw, +} from '../../src/memory/index' import { createScopedTestAdapter, runAgentMemoryImprovement } from '../support/memory' -describe('agent memory improvement', () => { - it('fails fast when a JavaScript caller uses the removed onPromote option', async () => { - await expect( - runAgentMemoryImprovementRaw({ - onPromote() {}, - } as unknown as RunAgentMemoryImprovementOptions), - ).rejects.toThrow( - 'onPromote was removed; use activation.readCurrent and activation.compareAndSet', - ) - }) +type Config = { visibility: 'private' | 'team' } - it('requires an explicit controller policy for custom improvement storage', async () => { - await expect( - runAgentMemoryImprovementRaw({ - experimentId: 'custom-improvement-storage', - trainSequences: [improvementSequence('train', 'train')], - holdoutSequences: [improvementSequence('holdout', 'holdout')], - seeds: [ - { - config: { mode: 'baseline' }, - track: 'baseline', - proposer: 'default', - }, - ], - createCandidate: () => ({ - ref: 'memory:v1', - createAdapter: () => createScopedTestAdapter('memory'), - }), - proposer: { kind: 'noop', propose: async () => [] }, - improvementRef: 'custom-improvement-storage:v1', - budget: { maxSteps: 1 }, - runDir: '/runs/custom-improvement-storage', - storage: inMemoryCampaignStorage(), - }), - ).rejects.toThrow("requires acquireRunLease or controllerMode='process-local'") - }) - - it('searches isolated configs and activates only a fresh holdout win', async () => { - type Config = { visibility: 'private' | 'team' | 'shared' } +describe('agent memory improvement', () => { + it('runs a complete method, keeps final data private, resumes, and activates once', async () => { const storage = inMemoryCampaignStorage() - const promoted: Config[] = [] + const methodInputs: string[][] = [] + let candidateConstructions = 0 let activeConfig: Config = { visibility: 'private' } const activationIds: string[] = [] - const contenderIds = new Set() - const activeContenderCalls = new Map() - let maxConcurrentConfigs = 0 - let reportContendersActive: (() => void) | undefined - const contendersActive = new Promise((resolve) => { - reportContendersActive = resolve - }) - let releaseContenders: (() => void) | undefined - const continueContenders = new Promise((resolve) => { - releaseContenders = resolve - }) - let proposalCalls = 0 - const proposer: SurfaceProposer = { - kind: 'team-sharing-proposer', - async propose() { - proposalCalls += 1 - return [ - { - surface: JSON.stringify({ visibility: 'team' }), - label: 'share within the team', - rationale: "the second agent needs the first agent's accepted fact", - }, - { - surface: JSON.stringify({ visibility: 'shared' }), - label: 'share globally', - rationale: 'compare a broader sharing policy under the same histories', - }, - ] - }, - } - + const method = selectingMethod( + [{ visibility: 'private' }, { visibility: 'team' }], + methodInputs, + ) const options: RunAgentMemoryImprovementOptions = { - experimentId: 'improve-team-memory', - trainSequences: [ - improvementSequence('train-a', 'train'), - improvementSequence('train-b', 'train'), + experimentId: 'complete-method-memory', + baselineConfig: { visibility: 'private' }, + method, + trainSequences: [improvementSequence('train-a', 'train')], + selectionSequences: [improvementSequence('selection-a', 'validation')], + finalSequences: [ + improvementSequence('final-a', 'test'), + improvementSequence('final-b', 'test'), ], - holdoutSequences: [ - improvementSequence('holdout-a', 'holdout'), - improvementSequence('holdout-b', 'holdout'), - ], - seeds: [ - { - config: { visibility: 'private' }, - track: 'baseline', - proposer: 'seed', - }, - ], - proposer, - improvementRef: 'team-memory-policy/v1', - budget: { maxSteps: 1 }, - populationSize: 2, - candidateConcurrency: 2, - sequenceConcurrency: 4, - runDir: '/runs/improve-team-memory', + improvementRef: 'team-memory-policy/v2', + runDir: '/runs/complete-method-memory', storage, + controllerMode: 'process-local', + sequenceConcurrency: 4, significance: { minProductiveRuns: 2, resamples: 200, seed: 7 }, createCandidate: ({ config, candidateId }) => { - if (config.visibility !== 'private') contenderIds.add(candidateId) + candidateConstructions += 1 return { - ref: `visibility:${config.visibility}:v1`, - label: config.visibility, + ref: `visibility:${config.visibility}:v2`, policy: { read: [config.visibility], write: config.visibility }, createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), } }, - executeStepRef: 'parallel-config-proof/v1', - executeStep: async ({ candidateId, step }) => { - if (step.id !== 'research' || !contenderIds.has(candidateId)) return - activeContenderCalls.set(candidateId, (activeContenderCalls.get(candidateId) ?? 0) + 1) - maxConcurrentConfigs = Math.max(maxConcurrentConfigs, activeContenderCalls.size) - if (activeContenderCalls.size === 2) reportContendersActive?.() - try { - await continueContenders - } finally { - const remaining = (activeContenderCalls.get(candidateId) ?? 1) - 1 - if (remaining === 0) activeContenderCalls.delete(candidateId) - else activeContenderCalls.set(candidateId, remaining) - } - }, activation: { - ref: 'memory-policy/live:v1', + ref: 'memory-policy/live:v2', async readCurrent() { return structuredClone(activeConfig) }, - async compareAndSet({ activationId, expectedConfig, config }) { + async compareAndSet({ activationId, expectedConfig, config, optimization }) { expect(activeConfig).toEqual(expectedConfig) + expect(optimization.winner.surfaceHash).toBe(surfaceHash(canonicalJson(config))) activationIds.push(activationId) activeConfig = structuredClone(config) - promoted.push(structuredClone(config)) }, }, } - const firstRun = runAgentMemoryImprovement(options) - await contendersActive - await expect(runAgentMemoryImprovement(options)).rejects.toThrow('active controller') - releaseContenders?.() - const result = await firstRun + const result = await runAgentMemoryImprovement(options) + + expect(methodInputs).toEqual([['train-a', 'selection-a']]) + expect(result.winnerConfig).toEqual({ visibility: 'team' }) + expect(result.winnerSurface).toBe('{"visibility":"team"}') + expect(result.finalEvaluation.pairs).toHaveLength(2) + expect(result.finalEvaluation.pairs.map((pair) => pair.sequenceId)).toEqual([ + 'final-a', + 'final-b', + ]) expect(result.decision).toMatchObject({ status: 'promote', - reasons: [], baselineScore: 0.25, winnerScore: 1, lift: 0.75, }) - expect(result.decision.significance).toMatchObject({ n: 2, significant: true }) - expect(result.winnerConfig).toEqual({ visibility: 'team' }) - expect(result.holdout?.campaign.cells).toHaveLength(4) - expect(maxConcurrentConfigs).toBe(2) - expect(promoted).toEqual([{ visibility: 'team' }]) - expect(result.activation).toMatchObject({ status: 'activated' }) - expect(storage.read(result.resultJsonPath)).toContain('"status": "promote"') + expect(result.activation.status).toBe('activated') + expect(activeConfig).toEqual({ visibility: 'team' }) + expect(activationIds).toEqual([result.activation.id]) expect( - JSON.parse(storage.read('/runs/improve-team-memory/memory-improvement-manifest.json') ?? '{}') - .identity?.schema, - ).toBe(6) + JSON.parse( + storage.read('/runs/complete-method-memory/memory-improvement-manifest.json') ?? '{}', + ).identity?.schema, + ).toBe(7) + expect( + storage.read( + `/runs/complete-method-memory/memory-config-artifacts/${result.winnerSurfaceHash}/${stableId('sequence', 'final-a')}/rep-0-${stableId('seed', '42')}.json`, + ), + ).toContain('"sequenceId": "final-a"') + expect( + storage.read( + `/runs/complete-method-memory/memory-final-artifacts/${result.winnerSurfaceHash}/${stableId('sequence', 'final-a')}/rep-0.json`, + ), + ).toContain('"seed": 42') + const constructionsAfterFirstRun = candidateConstructions const resumed = await runAgentMemoryImprovement(options) - expect(proposalCalls).toBe(1) - expect(promoted).toEqual([{ visibility: 'team' }]) + + expect(candidateConstructions).toBe(constructionsAfterFirstRun) expect(activationIds).toEqual([result.activation.id]) - expect(resumed.activation).toEqual({ - ...result.activation, - status: 'already-activated', - }) - expect(activeConfig).toEqual({ visibility: 'team' }) - await expect( - runAgentMemoryImprovement({ ...options, budget: { maxSteps: 2 } }), - ).rejects.toThrow('does not match its persisted inputs or implementationRef') - await expect(runAgentMemoryImprovement({ ...options, minHoldoutScore: 0.99 })).rejects.toThrow( - 'does not match its persisted inputs or implementationRef', - ) - await expect( - runAgentMemoryImprovement({ ...options, improvementRef: 'team-memory-policy/v2' }), - ).rejects.toThrow('does not match its persisted inputs or implementationRef') + expect(resumed.activation.status).toBe('already-activated') + expect(resumed.winnerSurfaceHash).toBe(result.winnerSurfaceHash) + expect(resumed.finalEvaluation.manifestHash).toBe(result.finalEvaluation.manifestHash) }) - it('recovers when the live config changes before the activation event is persisted', async () => { - type Config = { visibility: 'private' | 'team' } + it('recovers an applied activation whose final journal write was interrupted', async () => { const storage = inMemoryCampaignStorage() const append = storage.append!.bind(storage) let rejectActivatedEvent = true @@ -206,47 +131,13 @@ describe('agent memory improvement', () => { } let activeConfig: Config = { visibility: 'private' } let compareAndSetCalls = 0 - const options: RunAgentMemoryImprovementOptions = { + const options = baseOptions({ experimentId: 'recover-memory-activation', - trainSequences: [ - improvementSequence('activation-train-a', 'train'), - improvementSequence('activation-train-b', 'train'), - ], - holdoutSequences: [ - improvementSequence('activation-holdout-a', 'holdout'), - improvementSequence('activation-holdout-b', 'holdout'), - ], - seeds: [ - { - config: { visibility: 'private' }, - track: 'baseline', - proposer: 'seed', - }, - ], - proposer: { - kind: 'team-sharing-proposer', - async propose() { - return [ - { - surface: JSON.stringify({ visibility: 'team' }), - label: 'share with team', - rationale: 'the second agent needs the accepted fact', - }, - ] - }, - }, - improvementRef: 'recover-memory-activation:v1', - budget: { maxSteps: 1 }, runDir: '/runs/recover-memory-activation', storage, - significance: { minProductiveRuns: 2, resamples: 200, seed: 11 }, - createCandidate: ({ config, candidateId }) => ({ - ref: `visibility:${config.visibility}:v1`, - policy: { read: [config.visibility], write: config.visibility }, - createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), - }), + method: selectingMethod([{ visibility: 'private' }, { visibility: 'team' }]), activation: { - ref: 'memory-policy/live:v1', + ref: 'memory-policy/live:v2', async readCurrent() { return structuredClone(activeConfig) }, @@ -256,7 +147,7 @@ describe('agent memory improvement', () => { activeConfig = structuredClone(config) }, }, - } + }) await expect(runAgentMemoryImprovement(options)).rejects.toThrow( 'activation journal unavailable', @@ -267,342 +158,183 @@ describe('agent memory improvement', () => { const recovered = await runAgentMemoryImprovement(options) expect(recovered.activation.status).toBe('recovered') expect(compareAndSetCalls).toBe(1) + const resumed = await runAgentMemoryImprovement(options) expect(resumed.activation.status).toBe('already-activated') expect(compareAndSetCalls).toBe(1) }) - it('routes independent tracks to their named proposers with track context', async () => { - type Config = { visibility: 'private' | 'team' } - const trackContexts: Array<{ - id?: string - operation?: string - vision?: string - generation: number - costPhase?: string - hasCostLedger: boolean - }> = [] - let governorCostPhase: string | undefined - let governorHasCostLedger = false - const trackProposer: SurfaceProposer = { - kind: 'team-memory-researcher', - async propose(context) { - trackContexts.push({ - id: context.track?.id, - operation: context.track?.operation, - vision: context.track?.vision, - generation: context.generation, - costPhase: context.costPhase, - hasCostLedger: context.costLedger !== undefined, - }) - return [JSON.stringify({ visibility: 'team' })] - }, - } - - await runAgentMemoryImprovement({ - experimentId: 'named-track-proposers', - trainSequences: [improvementSequence('track-train', 'train')], - holdoutSequences: [improvementSequence('track-holdout', 'holdout')], - seeds: [ - { config: { visibility: 'private' }, track: 'baseline', proposer: 'baseline' }, - { - config: { visibility: 'private' }, - track: 'sharing-research', - proposer: 'team-memory-researcher', - vision: 'test whether team memory transfers accepted facts', - }, - ], - proposer: { - kind: 'unexpected-fallback', - async propose() { - throw new Error('named track should not use the fallback proposer') - }, - }, - proposers: { 'team-memory-researcher': trackProposer }, - governor: { - decide(context) { - governorCostPhase = context.costPhase - governorHasCostLedger = context.costLedger !== undefined - return { op: 'extend', track: 'sharing-research' } - }, - }, - improvementRef: 'named-track-proposers/v1', - budget: { maxSteps: 1 }, - populationSize: 1, - runDir: '/runs/named-track-proposers', - storage: inMemoryCampaignStorage(), - createCandidate: ({ config, candidateId }) => ({ - ref: `visibility:${config.visibility}:v1`, - policy: { read: [config.visibility], write: config.visibility }, - createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), - }), + it('rejects a resumed final artifact whose embedded candidate identity changed', async () => { + const storage = inMemoryCampaignStorage() + const options = baseOptions({ + experimentId: 'corrupt-memory-artifact', + runDir: '/runs/corrupt-memory-artifact', + storage, }) + const first = await runAgentMemoryImprovement(options) + const artifactPath = + `/runs/corrupt-memory-artifact/memory-final-artifacts/${first.winnerSurfaceHash}` + + `/${stableId('sequence', 'final-a')}/rep-0.json` + const record = JSON.parse(storage.read(artifactPath)!) + record.artifact.candidateId = 'memory-config-from-another-surface' + storage.write(artifactPath, `${JSON.stringify(record, null, 2)}\n`) - expect(trackContexts).toEqual([ - { - id: 'sharing-research', - operation: 'extend', - vision: 'test whether team memory transfers accepted facts', - generation: 1, - costPhase: 'memory.proposal.sharing-research', - hasCostLedger: true, - }, - ]) - expect(governorCostPhase).toBe('memory.governor') - expect(governorHasCostLedger).toBe(true) + await expect(runAgentMemoryImprovement(options)).rejects.toThrow( + `memory config artifact '${artifactPath}' is malformed`, + ) }) - it('holds a winner when holdout histories do not test a critical dimension', async () => { - type Config = { visibility: 'private' | 'team' } - const result = await runAgentMemoryImprovement({ - experimentId: 'missing-critical-dimension', - trainSequences: [improvementSequence('critical-train', 'train')], - holdoutSequences: [ - improvementSequence('critical-holdout-a', 'holdout', false), - improvementSequence('critical-holdout-b', 'holdout', false), - improvementSequence('critical-holdout-c', 'holdout', false), + it('rejects copied data across train, selection, and final partitions', async () => { + const train = improvementSequence('train-original', 'train') + const options = baseOptions({ + experimentId: 'copied-memory-data', + runDir: '/runs/copied-memory-data', + trainSequences: [train], + finalSequences: [ + { ...train, id: 'renamed-final', split: 'test' }, + improvementSequence('final-b', 'test'), ], - seeds: [{ config: { visibility: 'private' }, track: 'baseline', proposer: 'sharing' }], - proposer: { - kind: 'sharing', - async propose() { - return [JSON.stringify({ visibility: 'team' })] - }, - }, - improvementRef: 'missing-critical-dimension/v1', - budget: { maxSteps: 1 }, - populationSize: 1, - runDir: '/runs/missing-critical-dimension', - storage: inMemoryCampaignStorage(), - significance: { minProductiveRuns: 1, resamples: 100, seed: 9 }, - criticalDimensions: ['memory_stale_safe'], - createCandidate: ({ config, candidateId }) => ({ - ref: `visibility:${config.visibility}:v1`, - policy: { read: [config.visibility], write: config.visibility }, - createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), - }), }) - expect(result.decision.status).toBe('hold') - expect(result.decision.criticalDimensions).toEqual([ - expect.objectContaining({ - dimension: 'memory_stale_safe', - n: 0, - expectedN: 0, - measured: false, - }), - ]) - expect(result.decision.reasons).toContain( - 'critical dimension memory_stale_safe has no applicable holdout histories', + await expect(runAgentMemoryImprovement(options)).rejects.toThrow( + "train/final histories duplicate content at 'train-original'/'renamed-final'", ) }) - it('stops before a proposer call would exceed the run-wide cost limit', async () => { - let proposerExecuted = false - - await expect( - runAgentMemoryImprovement({ - experimentId: 'proposer-cost-limit', - trainSequences: [improvementSequence('cost-train', 'train')], - holdoutSequences: [improvementSequence('cost-holdout', 'holdout')], - seeds: [ - { - config: { visibility: 'private' as const }, - track: 'baseline', - proposer: 'costed', - }, - ], - proposer: { - kind: 'costed', - async propose(context) { - if (!context.costLedger) throw new Error('missing run cost ledger') - const paid = await context.costLedger.runPaidCall({ - actor: 'memory-config-proposer', - channel: 'agent', - phase: context.costPhase, - model: 'fixture-model', - maximumCharge: { externallyEnforcedMaximumUsd: 0.06 }, - execute: async () => { - proposerExecuted = true - return JSON.stringify({ visibility: 'team' }) - }, - receipt: () => ({ - model: 'fixture-model', - inputTokens: 0, - outputTokens: 0, - usageUnknown: true, - actualCostUsd: 0.06, - }), - }) - if (!paid.succeeded) throw paid.error - return [paid.value] - }, - }, - improvementRef: 'proposer-cost-limit/v1', - budget: { maxSteps: 1 }, - maxTotalCostUsd: 0.05, - runDir: '/runs/proposer-cost-limit', - storage: inMemoryCampaignStorage(), - createCandidate: ({ config, candidateId }) => ({ - ref: `visibility:${config.visibility}:v1`, - policy: { read: [config.visibility], write: config.visibility }, - createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), - }), - }), - ).rejects.toThrow('would exceed ceiling 0.05') - expect(proposerExecuted).toBe(false) - }) - - it('charges an interrupted proposer reservation before resuming the search', async () => { - type Config = { visibility: 'private' | 'team' } - const storage = inMemoryCampaignStorage() - const runDir = '/runs/interrupted-proposer-recovery' - const append = storage.append!.bind(storage) - let failFirstProposerReceipt = true - storage.append = (path, value, expectedBytes) => { - if ( - failFirstProposerReceipt && - path.endsWith('/cost-ledger.jsonl') && - value.includes('"status":"settled"') && - value.includes('memory-config-proposer') - ) { - failFirstProposerReceipt = false - throw new Error('simulated process exit before proposer receipt') - } - return append(path, value, expectedBytes) - } - let proposerCalls = 0 - const options: RunAgentMemoryImprovementOptions = { - experimentId: 'interrupted-proposer-recovery', - trainSequences: [improvementSequence('proposer-train', 'train')], - holdoutSequences: [improvementSequence('proposer-holdout', 'holdout')], - seeds: [ - { - config: { visibility: 'private' }, - track: 'baseline', - proposer: 'costed', - }, - ], - proposer: { - kind: 'costed', - async propose(context) { - proposerCalls += 1 - if (!context.costLedger) throw new Error('missing run cost ledger') - const paid = await context.costLedger.runPaidCall({ - actor: 'memory-config-proposer', - channel: 'agent', - phase: context.costPhase, - model: 'fixture-model', - maximumCharge: { externallyEnforcedMaximumUsd: 0.1 }, - execute: async () => JSON.stringify({ visibility: 'team' }), - receipt: () => ({ - model: 'fixture-model', - inputTokens: 0, - outputTokens: 0, - actualCostUsd: 0.1, - }), - }) - if (!paid.succeeded) throw paid.error - return [paid.value] - }, - }, - improvementRef: 'interrupted-proposer-recovery/v1', - budget: { maxSteps: 1 }, - maxTotalCostUsd: 0.2, - runDir, - storage, - createCandidate: ({ config, candidateId }) => ({ - ref: `visibility:${config.visibility}:v1`, - policy: { read: [config.visibility], write: config.visibility }, - createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), - }), - } - - await expect(runAgentMemoryImprovement(options)).rejects.toThrow('failed to persist') - const interruptedLedger = createRunCostLedger({ - storage, - runDir, - costCeilingUsd: 0.2, + it('requires an explicit controller policy for custom storage', async () => { + const options = baseOptions({ + experimentId: 'custom-memory-storage', + runDir: '/runs/custom-memory-storage', + storage: inMemoryCampaignStorage(), }) - expect(interruptedLedger.listPending()).toEqual([ - expect.objectContaining({ actor: 'memory-config-proposer', state: 'interrupted' }), - ]) - - const result = await runAgentMemoryImprovement(options) - const resumedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 0.2 }) + delete (options as { controllerMode?: string }).controllerMode - expect(proposerCalls).toBe(2) - expect(result.totalCostUsd).toBe(0.2) - expect(resumedLedger.summary()).toMatchObject({ - totalCalls: 2, - unresolvedCalls: 0, - totalCostUsd: 0.2, - accountingComplete: true, - }) - expect(resumedLedger.list()[0]).toMatchObject({ - actor: 'memory-config-proposer', - costUsd: 0.1, - error: expect.stringContaining('charged the reserved maximum'), - }) + await expect(runAgentMemoryImprovementRaw(options)).rejects.toThrow( + "requires acquireRunLease or controllerMode='process-local'", + ) }) - it('rejects train and holdout histories with the same id', async () => { - const sequence = improvementSequence('duplicate', 'train') + it('requires a per-evaluation maximum before enabling paid work', async () => { await expect( - runAgentMemoryImprovement({ - experimentId: 'overlap', - trainSequences: [sequence], - holdoutSequences: [{ ...sequence, split: 'holdout' }], - seeds: [{ config: {}, track: 'baseline', proposer: 'seed' }], - proposer: { - kind: 'unused', - async propose() { - return [] - }, - }, - improvementRef: 'overlap-test/v1', - budget: { maxSteps: 1 }, - runDir: '/runs/overlap', - storage: inMemoryCampaignStorage(), - createCandidate: () => ({ - ref: 'unused:v1', - createAdapter: () => createScopedTestAdapter('unused'), + runAgentMemoryImprovement( + baseOptions({ + experimentId: 'missing-evaluation-maximum', + runDir: '/runs/missing-evaluation-maximum', + maxOptimizationCostUsd: 1, }), - }), - ).rejects.toThrow('train/holdout overlap: duplicate') + ), + ).rejects.toThrow('maximumEvaluationCostUsd is required when a spend limit is configured') }) - it('rejects a holdout history copied under a different id', async () => { - const train = improvementSequence('train-original', 'train') - await expect( - runAgentMemoryImprovement({ - experimentId: 'renamed-overlap', - trainSequences: [train], - holdoutSequences: [{ ...train, id: 'renamed-holdout', split: 'holdout' }], - seeds: [{ config: {}, track: 'baseline', proposer: 'seed' }], - proposer: { - kind: 'unused', - async propose() { - return [] + it('holds activation when the method cannot fully account for optimization cost', async () => { + let activationCalls = 0 + const result = await runAgentMemoryImprovement( + baseOptions({ + experimentId: 'incomplete-method-cost', + runDir: '/runs/incomplete-method-cost', + method: selectingMethod([{ visibility: 'private' }, { visibility: 'team' }], undefined, { + totalCostUsd: 0, + accountingComplete: false, + incompleteReasons: ['external optimizer usage unavailable'], + }), + activation: { + ref: 'memory-policy/live:v2', + async readCurrent() { + return { visibility: 'private' } + }, + async compareAndSet() { + activationCalls += 1 }, }, - improvementRef: 'renamed-overlap/v1', - budget: { maxSteps: 0 }, - runDir: '/runs/renamed-overlap', - storage: inMemoryCampaignStorage(), - createCandidate: () => ({ - ref: 'unused:v1', - createAdapter: () => createScopedTestAdapter('unused'), - }), }), - ).rejects.toThrow('histories duplicate content') + ) + + expect(result.winnerConfig).toEqual({ visibility: 'team' }) + expect(result.decision.status).toBe('hold') + expect(result.decision.reasons).toContain('optimization or final cost accounting is incomplete') + expect(result.activation.status).toBe('not-eligible') + expect(activationCalls).toBe(0) }) }) -function improvementSequence(id: string, split: 'train' | 'holdout', includeStaleTarget = true) { +function baseOptions( + overrides: Partial> = {}, +): RunAgentMemoryImprovementOptions { + return { + experimentId: 'memory-improvement', + baselineConfig: { visibility: 'private' }, + method: selectingMethod([{ visibility: 'private' }, { visibility: 'team' }]), + trainSequences: [improvementSequence('train-a', 'train')], + selectionSequences: [improvementSequence('selection-a', 'validation')], + finalSequences: [ + improvementSequence('final-a', 'test'), + improvementSequence('final-b', 'test'), + ], + createCandidate: ({ config, candidateId }) => ({ + ref: `visibility:${config.visibility}:v2`, + policy: { read: [config.visibility], write: config.visibility }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), + }), + improvementRef: 'memory-improvement:v2', + runDir: '/runs/memory-improvement', + storage: inMemoryCampaignStorage(), + controllerMode: 'process-local', + significance: { minProductiveRuns: 2, resamples: 200, seed: 7 }, + ...overrides, + } +} + +function selectingMethod( + configs: readonly TConfig[], + inputs?: string[][], + cost = { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, +): OptimizationMethod { + return { + name: 'fixture-selection', + async optimize(input) { + inputs?.push([ + ...input.trainScenarios.map((scenario) => scenario.id), + ...input.selectionScenarios.map((scenario) => scenario.id), + ]) + expect('testScenarios' in input).toBe(false) + const scored = await Promise.all( + configs.map(async (config) => { + const surface = canonicalJson(config) + await runCampaign({ + ...input.runOptions, + scenarios: [...input.trainScenarios], + dispatch: (scenario, context) => input.dispatchWithSurface(surface, scenario, context), + judges: [...input.judges], + runDir: `${input.runDir}/fixture/${surfaceHash(surface)}/train`, + seed: input.seed, + }) + const selection = await runCampaign({ + ...input.runOptions, + scenarios: [...input.selectionScenarios], + dispatch: (scenario, context) => input.dispatchWithSurface(surface, scenario, context), + judges: [...input.judges], + runDir: `${input.runDir}/fixture/${surfaceHash(surface)}/selection`, + seed: input.seed, + }) + return { surface, score: campaignMeanComposite(selection) } + }), + ) + scored.sort((left, right) => right.score - left.score) + return { + winnerSurface: scored[0]!.surface, + cost, + } + }, + } +} + +function improvementSequence( + id: string, + split: 'train' | 'validation' | 'test', +): AgentMemorySequence { return { id, - family: 'first-party' as const, + family: 'first-party', split, steps: [ { @@ -611,7 +343,7 @@ function improvementSequence(id: string, split: 'train' | 'holdout', includeStal writes: [ { id: `${id}-event`, - kind: 'fact' as const, + kind: 'fact', text: `${id} launch date is Friday`, metadata: { eventId: `${id}-event`, actorId: 'researcher' }, }, @@ -625,17 +357,13 @@ function improvementSequence(id: string, split: 'train' | 'holdout', includeStal id: 'launch-date', query: `${id} launch date`, requiredFacts: [{ id: 'current', anyOf: [`${id} launch date is Friday`] }], - ...(includeStaleTarget - ? { - forbiddenFacts: [ - { - id: 'stale', - anyOf: [`${id} launch date is Thursday`], - obsolete: true, - }, - ], - } - : {}), + forbiddenFacts: [ + { + id: 'stale', + anyOf: [`${id} launch date is Thursday`], + obsolete: true, + }, + ], expectedEventIds: [`${id}-event`], expectedActorIds: ['researcher'], }, diff --git a/tests/rag-improvement-loop.test.ts b/tests/rag-improvement-loop.test.ts index c0502c6..dbe3f9a 100644 --- a/tests/rag-improvement-loop.test.ts +++ b/tests/rag-improvement-loop.test.ts @@ -1,9 +1,17 @@ import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' +import { + inMemoryCampaignStorage, + type OptimizationMethod, +} from '@tangle-network/agent-eval/campaign' import { afterEach, describe, expect, it } from 'vitest' -import { type RetrievalEvalScenario, runRagKnowledgeImprovementLoop } from '../src/index' +import { + type RagAnswerEvalArtifact, + type RagAnswerEvalScenario, + type RetrievalEvalScenario, + runRagKnowledgeImprovementLoop, +} from '../src/index' const tempRoots: string[] = [] @@ -13,6 +21,86 @@ afterEach(async () => { }) describe('RAG knowledge improvement loop', () => { + it('runs a complete method over retrieval and answer configuration without exposing final data', async () => { + const methodInputs: string[][] = [] + const method: OptimizationMethod = { + name: 'fixture-rag-method', + async optimize(input) { + methodInputs.push([ + ...input.trainScenarios.map((scenario) => scenario.id), + ...input.selectionScenarios.map((scenario) => scenario.id), + ]) + expect('testScenarios' in input).toBe(false) + return { + winnerSurface: '{"answerMode":"grounded","k":2}', + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + } + }, + } + const scenario = (id: string): RagAnswerEvalScenario => ({ + id, + kind: 'rag-answer-eval', + query: `${id} refund window`, + expectedClaims: [`${id} refunds are allowed within 30 days`], + requiredContext: [{ id: `${id}-policy` }], + requireCitations: true, + }) + + const result = await runRagKnowledgeImprovementLoop({ + goal: 'Optimize retrieval and grounded answers together', + enabledPhases: ['rag-optimization', 'gap-diagnosis'], + requiredPhases: ['rag-optimization'], + optimization: { + baseline: { answerMode: 'unsupported', k: 1 }, + method, + trainScenarios: [scenario('rag-train')], + selectionScenarios: [scenario('rag-selection')], + finalScenarios: [scenario('rag-final-a'), scenario('rag-final-b')], + async run({ config, scenario: item }) { + const claim = `${item.id} refunds are allowed within 30 days` + if (config.answerMode !== 'grounded') { + return { + query: item.query, + answer: `${item.id} refunds are never allowed`, + contexts: [], + } + } + return { + query: item.query, + answer: claim, + contexts: [{ id: `${item.id}-policy`, text: claim, rank: 1 }], + claims: [{ id: `${item.id}-claim`, text: claim, citationIds: [`${item.id}-cite`] }], + citations: [ + { + id: `${item.id}-cite`, + claimId: `${item.id}-claim`, + contextId: `${item.id}-policy`, + quote: claim, + }, + ], + } + }, + runDir: 'memory://full-rag-optimization-test', + storage: inMemoryCampaignStorage(), + expectUsage: 'off', + resamples: 200, + }, + diagnose({ optimization }) { + expect(optimization?.winnerConfig).toEqual({ answerMode: 'grounded', k: 2 }) + return [] + }, + }) + + expect(methodInputs).toEqual([['rag-train', 'rag-selection']]) + expect(result.optimization?.winner.surface).toBe('{"answerMode":"grounded","k":2}') + expect(result.optimization?.comparison.testScenarioIds).toEqual(['rag-final-a', 'rag-final-b']) + expect(result.optimization?.comparison.best.lift).toBeGreaterThan(0) + expect(result.phases.map((phase) => `${phase.phase}:${phase.status}`)).toEqual([ + 'rag-optimization:completed', + 'gap-diagnosis:completed', + ]) + }) + it('exposes retrieval, diagnosis, acquisition, update, answer eval, and promotion phases', async () => { const calls: string[] = [] const trainScenario: RetrievalEvalScenario = { @@ -21,19 +109,24 @@ describe('RAG knowledge improvement loop', () => { query: 'needs second result', expected: { kind: 'page', pageId: 'gold' }, } - const holdoutScenario: RetrievalEvalScenario = { - id: 'q-holdout', + const makeScenario = (id: string): RetrievalEvalScenario => ({ + id, kind: 'retrieval-eval', - query: 'held out needs second result', + query: `${id} needs second result`, expected: { kind: 'page', pageId: 'gold' }, - } + }) const result = await runRagKnowledgeImprovementLoop({ goal: 'Improve support RAG', retrieval: { baseline: { k: 1 }, - scenarios: [trainScenario], - holdoutScenarios: [holdoutScenario], + trainScenarios: [trainScenario], + selectionScenarios: [ + makeScenario('q-selection-a'), + makeScenario('q-selection-b'), + makeScenario('q-selection-c'), + ], + finalScenarios: [makeScenario('q-final-a'), makeScenario('q-final-b')], searchSpace: { k: [1, 2] }, retrieve: async ({ k }) => ({ hits: [ @@ -41,13 +134,11 @@ describe('RAG knowledge improvement loop', () => { ...(k >= 2 ? [{ pageId: 'gold', path: 'knowledge/gold.md', rank: 2 }] : []), ], }), - targetRecall: 1, - deltaThreshold: 0.01, - populationSize: 1, - maxGenerations: 1, + boundedSearch: { targetRecall: 1, configurationConcurrency: 1 }, runDir: 'memory://rag-lifecycle-retrieval-test', storage: inMemoryCampaignStorage(), expectUsage: 'off', + resamples: 200, }, diagnose({ retrieval }) { calls.push('diagnose') @@ -109,6 +200,50 @@ describe('RAG knowledge improvement loop', () => { ]) }) + it('does not run full RAG optimization when the phase is disabled', async () => { + let methodCalled = false + const method: OptimizationMethod = { + name: 'disabled-rag-method', + async optimize(input) { + methodCalled = true + return { + winnerSurface: input.baselineSurface, + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + } + }, + } + const scenario = (id: string): RagAnswerEvalScenario => ({ + id, + kind: 'rag-answer-eval', + query: id, + }) + + const result = await runRagKnowledgeImprovementLoop({ + goal: 'Run diagnosis only', + enabledPhases: ['gap-diagnosis'], + optimization: { + baseline: { k: 1 }, + method, + trainScenarios: [scenario('disabled-train')], + selectionScenarios: [scenario('disabled-selection')], + finalScenarios: [scenario('disabled-final-a'), scenario('disabled-final-b')], + run: async ({ scenario: item }) => ({ + query: item.query, + answer: 'unused', + contexts: [], + }), + runDir: 'memory://disabled-rag-optimization-test', + storage: inMemoryCampaignStorage(), + expectUsage: 'off', + }, + diagnose: () => [], + }) + + expect(methodCalled).toBe(false) + expect(result.optimization).toBeUndefined() + expect(result.phases.map((phase) => phase.phase)).toEqual(['gap-diagnosis']) + }) + it('can apply acquired source text and write blocks through the existing research loop', async () => { const root = await mkdtemp(join(tmpdir(), 'agent-knowledge-rag-loop-')) tempRoots.push(root) diff --git a/tests/retrieval-eval.test.ts b/tests/retrieval-eval.test.ts index 7c2a3a1..5ab3a21 100644 --- a/tests/retrieval-eval.test.ts +++ b/tests/retrieval-eval.test.ts @@ -1,14 +1,17 @@ -import { inMemoryCampaignStorage, runCampaign } from '@tangle-network/agent-eval/campaign' +import { + inMemoryCampaignStorage, + type OptimizationMethod, + runCampaign, +} from '@tangle-network/agent-eval/campaign' import { describe, expect, it } from 'vitest' import { + buildBoundedRetrievalConfigs, buildRetrievalEvalDispatch, - buildRetrievalParameterCandidates, type KnowledgeIndex, type RetrievalEvalArtifact, type RetrievalEvalScenario, retrievalConfigFromSurface, retrievalConfigSurface, - retrievalParameterSweepProposer, retrievalRecallJudge, runRetrievalImprovementLoop, scoreRetrievalArtifact, @@ -183,78 +186,190 @@ describe('retrieval eval', () => { expect(campaign.aggregates.cost.totalCalls).toBe(1) }) - it('builds parameter candidates and delegates proposal to agent-eval', async () => { + it('enumerates a bounded retrieval grid and rejects spaces above its explicit limit', () => { const baseline = { k: 5, hybrid: false, reranker: null, chunk: { overlap: 100 } } - const candidates = buildRetrievalParameterCandidates( + const configurations = buildBoundedRetrievalConfigs( { 'chunk.overlap': [100, 200], hybrid: [false, true], k: [5, 10], }, - { baseline }, + { baseline, maxConfigurations: 8 }, ) - expect(candidates.map((candidate) => candidate.label)).toEqual([ - 'chunk.overlap=200', - 'hybrid=true', - 'k=10', - ]) + expect(configurations).toHaveLength(7) + expect(configurations).toContainEqual({ + k: 10, + hybrid: true, + reranker: null, + chunk: { overlap: 200 }, + }) + expect(() => + buildBoundedRetrievalConfigs( + { + k: [1, 2, 3], + hybrid: [false, true], + }, + { baseline, maxConfigurations: 5 }, + ), + ).toThrow(/more than 5 configurations/) + }) - const proposer = retrievalParameterSweepProposer({ candidates }) - const proposals = await proposer.propose({ - currentSurface: retrievalConfigSurface(baseline), - history: [], - findings: [], - populationSize: 2, - generation: 0, - signal, + it('runs a complete OptimizationMethod without exposing final cases to it', async () => { + const seen: string[][] = [] + const method: OptimizationMethod = { + name: 'official-compatible-fixture', + async optimize(input) { + seen.push([ + ...input.trainScenarios.map((scenario) => scenario.id), + ...input.selectionScenarios.map((scenario) => scenario.id), + ]) + return { + winnerSurface: '{\n "k": 2\n}', + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + } + }, + } + const result = await runRetrievalImprovementLoop({ + baseline: { k: 1 }, + method, + trainScenarios: [retrievalScenario('train', 'train query')], + selectionScenarios: [retrievalScenario('selection', 'selection query')], + finalScenarios: [ + retrievalScenario('final-a', 'final query a'), + retrievalScenario('final-b', 'final query b'), + ], + retrieve: retrievalFixture, + runDir: '/runs/retrieval-method-test', + storage: inMemoryCampaignStorage(), + expectUsage: 'off', + resamples: 200, }) - const surfaces = proposals.map((proposal) => - typeof proposal === 'string' ? proposal : 'surface' in proposal ? proposal.surface : proposal, - ) - expect(surfaces).toHaveLength(2) - expect(JSON.parse(surfaces[0] as string)).toMatchObject({ chunk: { overlap: 200 } }) - expect(JSON.parse(surfaces[1] as string)).toMatchObject({ hybrid: true }) + expect(seen).toEqual([['train', 'selection']]) + expect(result.winnerConfig).toEqual({ k: 2 }) + expect(result.winner.surface).toBe('{"k":2}') + expect(result.winner.surfaceHash).not.toBe(result.baseline.surfaceHash) + expect(result.comparison.best.scenarioScores.map((row) => row.scenarioId)).toEqual([ + 'final-a', + 'final-b', + ]) }) - it('runs an agent-eval loop that auto-selects the better retrieval config', async () => { - const trainScenario: RetrievalEvalScenario = { - id: 'q-train', - kind: 'retrieval-eval', - query: 'needs second result', - expected: { kind: 'page', pageId: 'gold' }, + it('rejects renamed duplicate scenarios before starting the method', async () => { + let methodCalled = false + const method: OptimizationMethod = { + name: 'must-not-run', + async optimize(input) { + methodCalled = true + return { + winnerSurface: input.baselineSurface, + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + } + }, } - const holdoutScenario: RetrievalEvalScenario = { - id: 'q-holdout', - kind: 'retrieval-eval', - query: 'held out needs second result', - expected: { kind: 'page', pageId: 'gold' }, + + await expect( + runRetrievalImprovementLoop({ + baseline: { k: 1 }, + method, + trainScenarios: [ + retrievalScenario('duplicate-train-a', 'same query'), + retrievalScenario('duplicate-train-b', 'same query'), + ], + selectionScenarios: [retrievalScenario('selection', 'selection query')], + finalScenarios: [ + retrievalScenario('final-a', 'final query a'), + retrievalScenario('final-b', 'final query b'), + ], + retrieve: retrievalFixture, + runDir: '/runs/retrieval-duplicate-test', + storage: inMemoryCampaignStorage(), + expectUsage: 'off', + }), + ).rejects.toThrow( + "train partition duplicates scenario content at 'duplicate-train-a'/'duplicate-train-b'", + ) + expect(methodCalled).toBe(false) + }) + + it('rejects a non-object method winner before retrieval runs', async () => { + let retrievalCalls = 0 + const method: OptimizationMethod = { + name: 'invalid-config-winner', + async optimize() { + return { + winnerSurface: 'null', + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + } + }, } - const result = await runRetrievalImprovementLoop({ - baseline: { k: 1 }, - scenarios: [trainScenario], - holdoutScenarios: [holdoutScenario], - searchSpace: { k: [1, 2] }, - retrieve: async ({ k }) => ({ - hits: [ - { pageId: 'distractor', path: 'knowledge/distractor.md', rank: 1 }, - ...(k >= 2 ? [{ pageId: 'gold', path: 'knowledge/gold.md', rank: 2 }] : []), + await expect( + runRetrievalImprovementLoop({ + baseline: { k: 1 }, + method, + trainScenarios: [retrievalScenario('invalid-train', 'train query')], + selectionScenarios: [retrievalScenario('invalid-selection', 'selection query')], + finalScenarios: [ + retrievalScenario('invalid-final-a', 'final query a'), + retrievalScenario('invalid-final-b', 'final query b'), ], + retrieve: async (input) => { + retrievalCalls += 1 + return retrievalFixture(input) + }, + runDir: '/runs/retrieval-invalid-config-test', + storage: inMemoryCampaignStorage(), + expectUsage: 'off', }), - targetRecall: 1, - deltaThreshold: 0.01, - populationSize: 1, - maxGenerations: 1, - runDir: 'memory://retrieval-loop-test', - storage: inMemoryCampaignStorage(), - expectUsage: 'off', - }) + ).rejects.toThrow('serialized knowledge candidate must be a JSON object') + expect(retrievalCalls).toBe(0) + }) + + it('uses the neutral bounded method for a small finite retrieval space', async () => { + const storage = inMemoryCampaignStorage() + const retrievedK: number[] = [] + const trainScenarios = [retrievalScenario('train', 'train query')] + const selectionScenarios = [ + retrievalScenario('selection-a', 'selection query a'), + retrievalScenario('selection-b', 'selection query b'), + retrievalScenario('selection-c', 'selection query c'), + ] + const finalScenarios = [ + retrievalScenario('final-a', 'final query a'), + retrievalScenario('final-b', 'final query b'), + ] + const run = () => + runRetrievalImprovementLoop({ + baseline: { k: 1 }, + configurations: [{ k: 2 }, { k: 3 }], + boundedSearch: { configurationConcurrency: 1, targetRecall: 1 }, + trainScenarios, + selectionScenarios, + finalScenarios, + retrieve: async (input) => { + retrievedK.push(input.k) + return retrievalFixture(input) + }, + runDir: '/runs/retrieval-bounded-test', + storage, + expectUsage: 'off', + resamples: 200, + }) + const result = await run() expect(result.winnerConfig).toMatchObject({ k: 2 }) + expect(result.boundedConfigurations).toEqual([{ k: 2 }, { k: 3 }]) + expect(retrievedK).not.toContain(3) expect(result.trainScenarios).toHaveLength(1) - expect(result.holdoutScenarios).toHaveLength(1) + expect(result.selectionScenarios).toHaveLength(3) + expect(result.finalScenarios).toHaveLength(2) + + const callsAfterFirstRun = retrievedK.length + const resumed = await run() + expect(retrievedK).toHaveLength(callsAfterFirstRun) + expect(resumed.winner.surfaceHash).toBe(result.winner.surfaceHash) }) it('fails loudly on invalid config surfaces and empty expected labels', () => { @@ -278,3 +393,21 @@ describe('retrieval eval', () => { ).toThrow(/has no expected targets/) }) }) + +function retrievalScenario(id: string, query: string): RetrievalEvalScenario { + return { + id, + kind: 'retrieval-eval', + query, + expected: { kind: 'page', pageId: 'gold' }, + } +} + +async function retrievalFixture({ k }: { k: number }) { + return { + hits: [ + { pageId: 'distractor', path: 'knowledge/distractor.md', rank: 1 }, + ...(k >= 2 ? [{ pageId: 'gold', path: 'knowledge/gold.md', rank: 2 }] : []), + ], + } +} diff --git a/tests/support/memory.ts b/tests/support/memory.ts index 9381514..982ea4b 100644 --- a/tests/support/memory.ts +++ b/tests/support/memory.ts @@ -1,3 +1,4 @@ +import type { JsonValue } from '@tangle-network/agent-eval/campaign' import { type AgentMemoryAdapter, type AgentMemoryHit, @@ -23,7 +24,7 @@ export function runAgentMemoryExperiment(options: RunAgentMemoryExperimentOption return runAgentMemoryExperimentRaw(withProcessLocalController(options)) } -export function runAgentMemoryImprovement( +export function runAgentMemoryImprovement( options: RunAgentMemoryImprovementOptions, ) { return runAgentMemoryImprovementRaw(withProcessLocalController(options))