diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9ddd1f..6eb8039 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,3 +38,38 @@ jobs: - name: Verify packed package lifecycle run: pnpm verify:package + + official-optimizers: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install JavaScript dependencies + run: pnpm install --frozen-lockfile + + - name: Install official Python optimizers + run: | + python -m venv .venv-official + .venv-official/bin/python -m pip install --upgrade pip + RPC_VERSION=$(node -e "const v=require('./package.json').dependencies['@tangle-network/agent-eval']; if (!/^\\d+\\.\\d+\\.\\d+$/.test(v)) throw new Error('agent-eval dependency must be exact'); console.log(v)") + .venv-official/bin/python -m pip install \ + "agent-eval-rpc==$RPC_VERSION" \ + "gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f" \ + "skillopt @ git+https://github.com/microsoft/SkillOpt.git@61735e3922efc2b90c6d6cab561e62e98452ca90" + + - name: Run packed package through official GEPA and SkillOpt + run: pnpm verify:official-optimizers + env: + AGENT_EVAL_TEST_PYTHON: ${{ github.workspace }}/.venv-official/bin/python diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index df0ec7e..2a8b304 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -20,6 +20,10 @@ jobs: cache: pnpm registry-url: https://registry.npmjs.org + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install deps run: pnpm install --frozen-lockfile @@ -38,6 +42,21 @@ jobs: - name: Verify packed package lifecycle run: pnpm run verify:package + - name: Install official Python optimizers + run: | + python -m venv .venv-official + .venv-official/bin/python -m pip install --upgrade pip + RPC_VERSION=$(node -e "const v=require('./package.json').dependencies['@tangle-network/agent-eval']; if (!/^\\d+\\.\\d+\\.\\d+$/.test(v)) throw new Error('agent-eval dependency must be exact'); console.log(v)") + .venv-official/bin/python -m pip install \ + "agent-eval-rpc==$RPC_VERSION" \ + "gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f" \ + "skillopt @ git+https://github.com/microsoft/SkillOpt.git@61735e3922efc2b90c6d6cab561e62e98452ca90" + + - name: Run packed package through official GEPA and SkillOpt + run: pnpm run verify:official-optimizers + env: + AGENT_EVAL_TEST_PYTHON: ${{ github.workspace }}/.venv-official/bin/python + - name: Verify tag/version lock run: | NPM_VERSION=$(node -p "require('./package.json').version") diff --git a/CHANGELOG.md b/CHANGELOG.md index f6f1afa..7a42a0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,23 +1,32 @@ # Changelog -## Unreleased +## 5.0.0 ### Breaking Changes - Retrieval improvement now requires independent train, selection, and final scenarios plus an explicit complete `OptimizationMethod`. +- Serialized retrieval and RAG optimization now requires an immutable `executionRef` covering candidate execution and scoring behavior. - Memory configuration improvement now requires a baseline configuration, a complete `OptimizationMethod`, and independent train, selection, and final histories. +- The RAG lifecycle promotion callback is now `decidePromotion` and runs only after final evidence passes regression, provenance, and cost checks. +- Knowledge improvement requires an immutable `implementationRef`, separates repeatable development evaluation from single-use final evaluation, and refuses to resume after interrupted final scoring. +- Memory candidate factories no longer receive scenario, repetition, or seed identity and must report observed external charges through `recordExternalCost()`. +- Answer-quality hooks require immutable evaluator identity, final scenario identity, and complete cost evidence. - Removed the public retrieval and memory proposer-search options; candidate generation and selection now belong to `agent-eval` methods. ### Added - Added a shared serialized-candidate adapter for running complete `agent-eval` optimization methods with canonical candidate identity and untouched final comparison. - Added full RAG configuration optimization and KB maintenance policy optimization. -- Added an explicit `OptimizationMethod` factory for bounded retrieval configuration enumeration over small finite spaces. +- Added direct support for official GEPA and SkillOpt methods through the shared `OptimizationMethod` contract. +- Added durable per-configuration memory candidate identities to prevent stale result reuse. +- Added live activation verification so resumed memory runs reject configuration drift. +- Added private execution contexts that expose memory operations, cancellation, and cost metering without evaluation labels. ### Changed -- Updated `@tangle-network/agent-eval` to `0.123.8`. +- Updated `@tangle-network/agent-eval` to `0.126.3` and `@tangle-network/agent-interface` to `0.32.0`. - Kept memory provider evaluations resumable and branch-isolated while moving search ownership to the supplied method. +- Restricted immutable references to lowercase SHA-256 and full Git commit identities. ## 4.1.0 diff --git a/README.md b/README.md index 0b5c6c8..5799c37 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Supply application callbacks for those decisions, or use `@tangle-network/agent- ## Install ```bash -pnpm add @tangle-network/agent-knowledge +pnpm add @tangle-network/agent-knowledge @tangle-network/agent-eval@0.126.3 ``` Requires Node.js 20.19 or later. @@ -121,6 +121,7 @@ import { const result = await improveKnowledgeBase({ root: './support-kb', goal: 'Answer refund questions using the current policy', + implementationRef: 'git:0123456789abcdef0123456789abcdef01234567', runId: 'refund-policy-2026-07', maxCandidates: 3, updateKnowledge: async ({ candidateRoot }) => { @@ -142,9 +143,15 @@ if (result.candidate && result.evaluation?.passed) { The update can come from a research agent, a coding agent, a source connector, or deterministic code. The comparison must return the product measures that decide whether the change helped. -Use the same `runId` to resume an interrupted run. +Use the same `runId` and `implementationRef` to resume an interrupted run. +Change `implementationRef` whenever callbacks, evaluation policy, models, indexes, or external configuration change. +Reusing a run ID with a different implementation reference fails before cached work or callbacks run. Different run IDs create separate candidate workspaces, so workers can explore in parallel. Promotion checks the original base hash and rejects a stale candidate instead of replacing newer work. +Candidate retries use `evaluateDevelopment` when provided, otherwise they use deterministic validation, readiness, and KB quality checks. +Development evaluation must use only train or selection data. +The configured `evaluate` callback and final RAG phases run once, on the first candidate that passes those development checks. +A failed final evaluation ends the run instead of selecting another candidate against final data. Candidate promotion currently requires Linux because it relies on Linux directory descriptors for exact file identity. @@ -155,7 +162,6 @@ Use the narrowest API that matches the job: | API | What it does | |---|---| | `runRetrievalImprovementLoop` | Runs one complete `OptimizationMethod` over serialized retrieval configuration. | -| `boundedRetrievalConfigMethod` | Builds a complete method that enumerates a small finite retrieval grid, limited to 128 configurations by default. | | `runRagOptimization` | Optimizes retrieval and answer behavior as one serialized RAG configuration. | | `optimizeKnowledgeBasePolicy` | Optimizes a KB maintenance policy, then applies only the selected policy to an isolated candidate. | | `scoreKnowledgeBaseIndex` | Measures KB structure, citations, source freshness, and configured quality thresholds. | @@ -164,36 +170,56 @@ Use the narrowest API that matches the job: | `improveKnowledgeBase` | Adds resumable state, isolated candidates, exact promotion, and conflict detection around that process. | ```ts -const method = boundedRetrievalConfigMethod({ - searchSpace: { k: [5, 10], reranker: [false, true] }, -}) +import type { + OptimizationMethod, +} from '@tangle-network/agent-eval/campaign' +import { + runRetrievalImprovementLoop, + type RetrievalEvalArtifact, + type RetrievalEvalScenario, +} from '@tangle-network/agent-knowledge' -const result = await runRetrievalImprovementLoop({ - baseline: { k: 5, reranker: false }, - method, - trainScenarios, - selectionScenarios, - finalScenarios, - retrieve: ({ scenario, config, k }) => search(scenario.query, { ...config, k }), - runDir: 'refund-retrieval-v1', - expectUsage: 'off', // Local search does not make a billable model call. -}) +async function tuneRetrieval( + method: OptimizationMethod, +) { + return runRetrievalImprovementLoop({ + executionRef: 'git:0123456789abcdef0123456789abcdef01234567', + baseline: { k: 5, reranker: false }, + method, + trainScenarios, + selectionScenarios, + finalScenarios, + retrieve: ({ scenario, config, k }) => search(scenario.query, { ...config, k }), + runDir: 'refund-retrieval', + expectUsage: 'off', // Local search does not make a billable model call. + }) +} ``` The method receives train and selection cases. `agent-eval` keeps final cases out of the search and measures the exact selected configuration on them afterward. Every optimization call requires an explicit complete method. -Pass an applicable official method, a custom method, or `boundedRetrievalConfigMethod()` for a small finite retrieval grid. +Create an official method with `gepaOptimizationMethod()` or `skillOptOptimizationMethod()` from `@tangle-network/agent-eval/campaign`, or pass another complete public `OptimizationMethod`. Reuse the run directory only with the method's compatible resume mode. Use separate run directories to explore branches in parallel. Optimizer-specific identity and resume settings stay on the supplied method; this package does not reinterpret them. -The supplied method owns `evaluationVersion`, engine or skill inputs, and resume compatibility. -`agent-eval` component surfaces stay outside this adapter; each knowledge candidate has one canonical serialized identity. -See the [`agent-eval` method guide](https://github.com/tangle-network/agent-eval/blob/main/docs/campaign-proposers.md) for official GEPA and Omni methods. +The supplied method owns its engine inputs and resume state. +`executionRef` owns retrieval, index, judge, model, and external-service identity. +Use `git:<40 lowercase hex>` or `sha256:<64 lowercase hex>` and change it whenever any candidate execution or scoring behavior changes. +Each candidate has one canonical serialized identity. +See the [`agent-eval` method guide](https://github.com/tangle-network/agent-eval/blob/main/docs/campaign-proposers.md) for official GEPA and SkillOpt methods. SkillOpt is skill-only; use it here only when the serialized candidate is itself a skill. +Read reported spend from `result.comparison.totalCost` and upstream source and run identity from `result.comparison.best.provenance`. +Official external methods must report observed package identity. +Custom in-process methods have no external package identity, so their behavior must be covered by `executionRef`. +Treat `accountingComplete: false` as incomplete evidence for activation. Retrieval and answer generation remain callbacks. This lets the same evaluation code work with local search, vector databases, hybrid search, rerankers, and hosted RAG services. +Adaptive diagnosis, acquisition, and update callbacks finish before retrieval or RAG final scoring starts. +Only answer evaluation, the terminal promotion decision, and the returned result can observe selected configurations. +Answer-quality evidence must name at least two final scenario IDs, immutable dataset and evaluator references, non-empty finite metrics, and observed cost accounting. +Promotion also requires `answerQualityCostCeiling`. ## Integrate memory systems @@ -203,8 +229,15 @@ Install the provider you use, create its client, and pass that client to the ada The memory APIs support scoped reads and writes, isolated branches, ordered histories, independent train, selection, and final comparisons, and adapter experiments. Use them to compare a provider against no memory or another provider on the same tasks before changing production behavior. -`runAgentMemoryImprovement` accepts a complete `OptimizationMethod`, evaluates each serialized configuration in an isolated provider branch, and activates only a final-data winner through compare-and-set. -Paid memory improvement defaults to zero-dollar optimization and final limits; set both limits and a per-evaluation maximum before enabling paid work. +`runAgentMemoryImprovement` accepts a complete `OptimizationMethod`, evaluates each serialized configuration in an isolated provider branch, and activates only a winner that passes a separate final comparison. +Set `implementationRef` to `git:<40 lowercase hex>` or `sha256:<64 lowercase hex>` covering the installed implementation, method configuration, candidate construction, execution behavior, and external configuration so incompatible state cannot resume. +The run records one immutable candidate reference for each memory configuration and refuses cached results if that reference changes. +Each improvement candidate declares a maximum for one sequence and one recovery attempt. +The adapter must enforce that maximum with its provider before starting external work. +The adapter callback must call `recordExternalCost()` with each observed charge. +Positive external work without a receipt is recorded as incomplete cost accounting, not as the configured maximum. +Use `0` only for a free local path. +Paid memory improvement defaults to a zero-dollar total limit; set `maxTotalCostUsd` and `maximumEvaluationCostUsd` before enabling paid work. ## Run benchmarks diff --git a/docs/eval/rag-eval-roadmap.md b/docs/eval/rag-eval-roadmap.md index 0ff5a7e..ce90da0 100644 --- a/docs/eval/rag-eval-roadmap.md +++ b/docs/eval/rag-eval-roadmap.md @@ -21,12 +21,12 @@ Done: - `runRetrievalImprovementLoop()` runs a complete `agent-eval` optimization method over retrieval configs. - `runRagOptimization()` does the same for a serialized retrieval and answer configuration. -- `boundedRetrievalConfigMethod()` builds an explicitly supplied method for finite retrieval grids of at most 128 configurations by default. - `runRagKnowledgeImprovementLoop()` exposes the whole RAG lifecycle as typed phases: retrieval tuning, gap diagnosis, knowledge acquisition, knowledge update, answer-quality eval, and promotion. - Retrieval scenarios can label pages, page paths, sources, source anchors, and source spans. - The retrieval judge reports recall, MRR, nDCG, and precision@k; `agent-eval` reports cost separately. - Selection and final data remain independent, and the optimization method never receives final cases. +- Every reusable optimization requires an immutable execution reference covering retrieval, index, model, judge, and external-service behavior. - The integration is tested with complete methods for retrieval and full RAG configuration. - The lifecycle loop is tested both with pluggable phase hooks and with a real local KB update through `runKnowledgeResearchLoop()`. - `ragAnswerQualityJudge()` and `createRagAnswerQualityHook()` score context precision/recall/relevance/sufficiency, faithfulness, answer relevance/correctness, citation support, abstention, and unsupported-answer rate. diff --git a/package.json b/package.json index c8db7cf..c64e635 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-knowledge", - "version": "4.1.0", + "version": "5.0.0", "description": "Build, search, evaluate, and improve source-backed knowledge bases.", "homepage": "https://github.com/tangle-network/agent-knowledge#readme", "repository": { @@ -69,11 +69,12 @@ "lint": "biome check src tests", "format": "biome format --write src tests", "check:skills": "node scripts/check-skills.mjs", - "verify:package": "pnpm run check:skills && node scripts/verify-package.mjs" + "verify:package": "pnpm run check:skills && node scripts/verify-package.mjs", + "verify:official-optimizers": "node scripts/verify-official-optimizers.mjs" }, "dependencies": { - "@tangle-network/agent-eval": "^0.123.8", - "@tangle-network/agent-interface": "^0.31.0", + "@tangle-network/agent-eval": "0.126.3", + "@tangle-network/agent-interface": "^0.32.0", "proper-lockfile": "4.1.2", "zod": "^4.4.3" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a437886..7173db6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,11 +13,11 @@ importers: .: dependencies: '@tangle-network/agent-eval': - specifier: ^0.123.8 - version: 0.123.8(typescript@5.9.3) + specifier: 0.126.3 + version: 0.126.3(typescript@5.9.3) '@tangle-network/agent-interface': - specifier: ^0.31.0 - version: 0.31.0 + specifier: ^0.32.0 + version: 0.32.0 proper-lockfile: specifier: 4.1.2 version: 4.1.2 @@ -63,8 +63,8 @@ packages: peerDependencies: zod: ^4.0.0 - '@ax-llm/ax@23.0.0': - resolution: {integrity: sha512-CWL/vM9RfS0wvVvRbApjYfwhgLa5UOdJno5y2Ime+lWSLYBEwHOhiBJonx08jhh01KoyJYHWS+QcYeInqR7Fsw==} + '@ax-llm/ax@23.0.5': + resolution: {integrity: sha512-Kgs+P4hPHMyppOnlEQTSFUJTuuLVWfvvV4QakHwLv3VRlw2T98okZkiwicdN0AKGH/uTf0ktF0Kbq1s9c7AxcA==} hasBin: true peerDependencies: zod: ^3.24.0 || ^4.0.0 @@ -965,11 +965,11 @@ packages: '@tangle-network/agent-core@0.3.8': resolution: {integrity: sha512-bZfVpdiFjXbcQwSxSQABdtXEmUuapWChCcXrHkP6fAnwqEy9hWEfeZLlMZUy+1XaOfpTaMLUOEAwYhXfN018wQ==} - '@tangle-network/agent-core@0.4.11': - resolution: {integrity: sha512-5B1IjrJ8xDR7w8Hv/MSk2ixul6NEJQ5Ftzo+z6l7ipeFpc0yaf1+Kkml4HDUEmGW/rqdrNpBf9/MpT7i/SY0PA==} + '@tangle-network/agent-core@0.4.20': + resolution: {integrity: sha512-gJzZh5PqPJtWW6kMEfm3IP7CGibvHdVXM4uqCAuCy+Kvg9TlVVkzXqZPRt9gU44mjdw5hDGoVzZotFsPoHAlFw==} - '@tangle-network/agent-eval@0.123.8': - resolution: {integrity: sha512-uWbtpUUy7cmoawrDcsNh1q9jEAulSOYC/N/qKG3BT+Eezz7bhKQQcLhT0S/m66kcyjXTrre5v2N23KVan8b9TA==} + '@tangle-network/agent-eval@0.126.3': + resolution: {integrity: sha512-VCd2drX0nvYK7OCwRHell+08eKdtH5MTyOGB/fLc4vy3Ev8X/SgzJ41o8GvRDyuF1kaiS7JATjl8exBh5aLHSg==} engines: {node: '>=20'} hasBin: true @@ -979,11 +979,8 @@ packages: '@tangle-network/agent-interface@0.17.1': resolution: {integrity: sha512-B7dRJTo0HSUtgBCB1VMwkTFYkLUaRr/4BcRglrQuGhGUwOzKv1RYyMejOVh5M3a5AagY9N79f7GYbjcA3UmnIA==} - '@tangle-network/agent-interface@0.26.0': - resolution: {integrity: sha512-/z4HavFr/9AbaHxi/13bFP9iSUt8oitZbw4wS9g9KAt2TeN2ec5/A2C106udWkKIETZLnu465TfU+K4KDVNkKw==} - - '@tangle-network/agent-interface@0.31.0': - resolution: {integrity: sha512-OvP8OebhbFd4d/Mxt1QDPAckJdBIA9omxoXU17dBbyEwhZKsiM6rbuOPaPDcv0Sf4koNLI25CfOYuzqlDR6ypQ==} + '@tangle-network/agent-interface@0.32.0': + resolution: {integrity: sha512-8GUiqdr9MZ+iedkx7KVL6jbiuW6jh7CzKPS3VDXbWherJjFbuf+ULsMKmc+wnfG7LvtjJgZGr5y+cZm+JN1YDA==} '@tangle-network/sandbox@0.9.7': resolution: {integrity: sha512-9pCwJ5MlF7RUpp0AQKQDFyR0yu+E0udEhWkqhrlb/RuoJxlt72zVPuzO4FnMb1MZTkfjStmomC3k5xQyqi1YSA==} @@ -2551,7 +2548,7 @@ snapshots: openapi3-ts: 4.5.0 zod: 4.4.3 - '@ax-llm/ax@23.0.0(zod@4.4.3)': + '@ax-llm/ax@23.0.5(zod@4.4.3)': dependencies: '@opentelemetry/api': 1.9.1 optionalDependencies: @@ -3227,18 +3224,18 @@ snapshots: '@tangle-network/agent-interface': 0.17.1 zod: 4.4.3 - '@tangle-network/agent-core@0.4.11': + '@tangle-network/agent-core@0.4.20': dependencies: - '@tangle-network/agent-interface': 0.26.0 + '@tangle-network/agent-interface': 0.32.0 zod: 4.4.3 - '@tangle-network/agent-eval@0.123.8(typescript@5.9.3)': + '@tangle-network/agent-eval@0.126.3(typescript@5.9.3)': dependencies: '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) - '@ax-llm/ax': 23.0.0(zod@4.4.3) + '@ax-llm/ax': 23.0.5(zod@4.4.3) '@hono/node-server': 2.0.1(hono@4.12.30) - '@tangle-network/agent-core': 0.4.11 - '@tangle-network/agent-interface': 0.31.0 + '@tangle-network/agent-core': 0.4.20 + '@tangle-network/agent-interface': 0.32.0 '@tangle-network/tcloud': 0.4.14(typescript@5.9.3)(zod@4.4.3) hono: 4.12.30 zod: 4.4.3 @@ -3259,12 +3256,7 @@ snapshots: dependencies: zod: 4.4.3 - '@tangle-network/agent-interface@0.26.0': - dependencies: - '@noble/hashes': 1.8.0 - zod: 4.4.3 - - '@tangle-network/agent-interface@0.31.0': + '@tangle-network/agent-interface@0.32.0': dependencies: '@noble/hashes': 1.8.0 zod: 4.4.3 diff --git a/scripts/verify-official-optimizers.mjs b/scripts/verify-official-optimizers.mjs new file mode 100644 index 0000000..84e79ae --- /dev/null +++ b/scripts/verify-official-optimizers.mjs @@ -0,0 +1,125 @@ +import { spawnSync } from 'node:child_process' +import { + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' + +const python = process.env.AGENT_EVAL_TEST_PYTHON +if (!python) { + throw new Error('AGENT_EVAL_TEST_PYTHON must point to Python with GEPA and SkillOpt installed') +} + +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..') +const sourcePackage = JSON.parse(readFileSync(join(repoRoot, 'package.json'), 'utf8')) +const agentEvalVersion = sourcePackage.dependencies?.['@tangle-network/agent-eval'] +if (!/^\d+\.\d+\.\d+$/.test(agentEvalVersion)) { + throw new Error('@tangle-network/agent-eval must be pinned to one exact version') +} +const pythonRpcVersion = run( + python, + ['-c', "from importlib.metadata import version; print(version('agent-eval-rpc'))"], + repoRoot, +).trim() +if (pythonRpcVersion !== agentEvalVersion) { + throw new Error( + `official optimizer Python bridge ${pythonRpcVersion} does not match agent-eval ${agentEvalVersion}`, + ) +} +const tempRoot = mkdtempSync(join(tmpdir(), 'agent-knowledge-official-')) + +try { + const packDir = join(tempRoot, 'pack') + const appDir = join(tempRoot, 'app') + mkdirSync(packDir, { recursive: true }) + mkdirSync(appDir, { recursive: true }) + writeFileSync( + join(appDir, 'package.json'), + `${JSON.stringify( + { + name: 'agent-knowledge-official-optimizer-verification', + private: true, + type: 'module', + }, + null, + 2, + )}\n`, + ) + + run('pnpm', ['build'], repoRoot) + run('npm', ['pack', '--ignore-scripts=false', '--pack-destination', packDir], repoRoot) + const tarballs = readdirSync(packDir).filter((name) => name.endsWith('.tgz')) + if (tarballs.length !== 1) { + throw new Error(`expected one package tarball, found ${tarballs.length}`) + } + run( + 'npm', + [ + 'install', + '--ignore-scripts=false', + '--no-package-lock', + '--no-save', + '--no-audit', + '--no-fund', + '--cache', + join(tempRoot, 'npm-cache'), + '--prefer-online', + join(packDir, tarballs[0]), + ], + appDir, + ) + + const installed = join(appDir, 'node_modules') + run( + 'pnpm', + [ + 'exec', + 'vitest', + 'run', + 'tests/official-optimization.integration.test.ts', + '--maxWorkers=1', + ], + repoRoot, + { + AGENT_EVAL_TEST_PYTHON: python, + AGENT_KNOWLEDGE_PACKAGE_URL: pathToFileURL( + join(installed, '@tangle-network', 'agent-knowledge', 'dist', 'index.js'), + ).href, + AGENT_EVAL_CAMPAIGN_URL: pathToFileURL( + join(installed, '@tangle-network', 'agent-eval', 'dist', 'campaign', 'index.js'), + ).href, + }, + ) +} finally { + rmSync(tempRoot, { recursive: true, force: true }) +} + +function run(command, args, cwd, env = {}) { + const result = spawnSync(command, args, { + cwd, + encoding: 'utf8', + env: { ...process.env, ...env }, + stdio: ['ignore', 'pipe', 'pipe'], + }) + if (result.error || result.status !== 0) { + throw new Error( + [ + `command failed: ${command} ${args.join(' ')}`, + result.error?.message, + result.stdout?.trim(), + result.stderr?.trim(), + ] + .filter(Boolean) + .join('\n'), + ) + } + process.stdout.write(result.stdout) + process.stderr.write(result.stderr) + return result.stdout +} diff --git a/scripts/verify-package.mjs b/scripts/verify-package.mjs index 3664b7a..babc5c1 100644 --- a/scripts/verify-package.mjs +++ b/scripts/verify-package.mjs @@ -19,8 +19,27 @@ const publicImports = [ `${packageName}/sources`, `${packageName}/benchmarks`, ] -const requiredRootExports = ['createFileSystemSearchProvider'] +const requiredRootExports = [ + 'createFileSystemSearchProvider', + 'optimizeKnowledgeBasePolicy', + 'runRagOptimization', + 'runRetrievalImprovementLoop', + 'runSerializedKnowledgeOptimization', +] +const forbiddenRootExports = [ + 'boundedRetrievalConfigMethod', + 'buildBoundedRetrievalConfigs', + 'buildRetrievalParameterCandidates', + 'retrievalParameterSweepProposer', +] +const requiredMemoryExports = ['runAgentMemoryImprovement'] +const requiredAgentEvalExports = ['gepaOptimizationMethod', 'skillOptOptimizationMethod'] const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..') +const sourcePackage = JSON.parse(readFileSync(join(repoRoot, 'package.json'), 'utf8')) +const agentEvalVersion = sourcePackage.dependencies?.['@tangle-network/agent-eval'] +if (!/^\d+\.\d+\.\d+$/.test(agentEvalVersion)) { + throw new Error('@tangle-network/agent-eval must be pinned to one exact version') +} const tempRoot = mkdtempSync(join(tmpdir(), 'agent-knowledge-package-')) try { @@ -55,7 +74,11 @@ try { '--no-save', '--no-audit', '--no-fund', + '--cache', + join(tempRoot, 'npm-cache'), + '--prefer-online', sourceTarball, + `@tangle-network/agent-eval@${agentEvalVersion}`, ], appDir, ) @@ -64,6 +87,20 @@ try { const installedPackage = JSON.parse( readFileSync(join(installedPackageDir, 'package.json'), 'utf8'), ) + const installedAgentEval = JSON.parse( + readFileSync( + join(appDir, 'node_modules', '@tangle-network', 'agent-eval', 'package.json'), + 'utf8', + ), + ) + if ( + installedPackage.dependencies?.['@tangle-network/agent-eval'] !== agentEvalVersion || + installedAgentEval.version !== agentEvalVersion + ) { + throw new Error( + `agent-eval version mismatch: dependency=${installedPackage.dependencies?.['@tangle-network/agent-eval']} installed=${installedAgentEval.version} expected=${agentEvalVersion}`, + ) + } const installedSkill = readFileSync( join(installedPackageDir, 'skills', 'build-with-agent-knowledge', 'SKILL.md'), 'utf8', @@ -90,7 +127,18 @@ try { `for (const name of ${JSON.stringify(requiredRootExports)}) {`, ` if (typeof root[name] !== 'function') throw new Error('missing root export: ' + name)`, `}`, - `for (const specifier of ${JSON.stringify(publicImports.slice(1))}) await import(specifier)`, + `for (const name of ${JSON.stringify(forbiddenRootExports)}) {`, + ` if (name in root) throw new Error('obsolete root export: ' + name)`, + `}`, + `const memory = await import(${JSON.stringify(`${packageName}/memory`)})`, + `for (const name of ${JSON.stringify(requiredMemoryExports)}) {`, + ` if (typeof memory[name] !== 'function') throw new Error('missing memory export: ' + name)`, + `}`, + `const campaign = await import('@tangle-network/agent-eval/campaign')`, + `for (const name of ${JSON.stringify(requiredAgentEvalExports)}) {`, + ` if (typeof campaign[name] !== 'function') throw new Error('missing agent-eval optimizer: ' + name)`, + `}`, + `for (const specifier of ${JSON.stringify(publicImports.slice(1).filter((specifier) => !specifier.endsWith('/memory')))}) await import(specifier)`, ].join(';'), ], appDir, diff --git a/src/benchmarks/memory-recovery.ts b/src/benchmarks/memory-recovery.ts index 8219fc7..47001f6 100644 --- a/src/benchmarks/memory-recovery.ts +++ b/src/benchmarks/memory-recovery.ts @@ -391,7 +391,6 @@ function parseMemoryBenchmarkAttemptEvent( const valid = typeof event === 'object' && event !== null && - event.schema === 3 && (event.status === 'started' || event.status === 'cleaned') && isNonEmptyString(event.attemptId) && isNonEmptyString(event.candidateId) && @@ -516,7 +515,6 @@ function isRecordValue(value: unknown): value is Record { } export interface MemoryAdapterBenchmarkAttemptEvent { - schema: 3 status: 'started' | 'cleaned' attemptId: string candidateId: string diff --git a/src/benchmarks/memory-responder.ts b/src/benchmarks/memory-responder.ts index 9a7841f..6c64362 100644 --- a/src/benchmarks/memory-responder.ts +++ b/src/benchmarks/memory-responder.ts @@ -55,7 +55,6 @@ export function createMemoryAdapterBenchmarkResponder(options: { options.scope, ) const attempt: MemoryAdapterBenchmarkAttemptEvent = { - schema: 3, status: 'started', attemptId, candidateId: options.candidateId, diff --git a/src/immutable-ref.ts b/src/immutable-ref.ts new file mode 100644 index 0000000..ee07dd6 --- /dev/null +++ b/src/immutable-ref.ts @@ -0,0 +1,8 @@ +const SHA256_REF = /^sha256:[a-f0-9]{64}$/ +const GIT_REF = /^git:[a-f0-9]{40}$/ + +export function assertImmutableRef(value: unknown, label: string): asserts value is string { + if (typeof value !== 'string' || (!SHA256_REF.test(value) && !GIT_REF.test(value))) { + throw new Error(`${label} must be lowercase sha256:<64 hex> or git:<40 hex>`) + } +} diff --git a/src/kb-improvement/contracts.ts b/src/kb-improvement/contracts.ts index 2e122c7..c0152ea 100644 --- a/src/kb-improvement/contracts.ts +++ b/src/kb-improvement/contracts.ts @@ -93,6 +93,8 @@ export interface KnowledgeImprovementCandidateRecord { candidateHash?: string evidenceHash?: string promotionPlanHash?: string + /** Durable one-way boundary preventing final-case reuse after interruption. */ + finalEvaluationStartedAt?: string status: KnowledgeImprovementStatus createdAt: string updatedAt: string @@ -102,6 +104,7 @@ export interface KnowledgeImprovementRunState { runId: string root: string goal: string + implementationRef: string status: KnowledgeImprovementStatus baseHash: string createdAt: string @@ -150,6 +153,7 @@ export interface KnowledgeImprovementActivationPersistence { } export const digestSchema = z.string().regex(/^[a-f0-9]{64}$/) +export const immutableRefSchema = z.string().regex(/^(?:sha256:[a-f0-9]{64}|git:[a-f0-9]{40})$/) export const runIdSchema = z.string().min(1).max(2_048) @@ -262,6 +266,7 @@ const candidateRecordSchema = z candidateHash: digestSchema.optional(), evidenceHash: digestSchema.optional(), promotionPlanHash: digestSchema.optional(), + finalEvaluationStartedAt: z.iso.datetime().optional(), status: improvementStatusSchema, createdAt: z.iso.datetime(), updatedAt: z.iso.datetime(), @@ -273,6 +278,7 @@ export const KnowledgeImprovementRunStateSchema = z runId: runIdSchema, root: z.string().min(1), goal: z.string().min(1), + implementationRef: immutableRefSchema, status: improvementStatusSchema, baseHash: digestSchema, createdAt: z.iso.datetime(), @@ -382,6 +388,7 @@ export const KnowledgeImprovementEvidenceSchema = z candidateId: safePathSegmentSchema, iteration: z.number().int().positive(), goalHash: digestSchema, + implementationRef: immutableRefSchema, baseHash: digestSchema, candidateHash: digestSchema, promotionPlanHash: digestSchema, @@ -496,6 +503,11 @@ export type KnowledgeImprovementUpdate = ( export interface KnowledgeImprovementOptions { root: string goal: string + /** + * Immutable identity covering callbacks, evaluation policy, models, indexes, + * external services, and all other behavior that can affect this run. + */ + implementationRef: string runId?: string ownerId?: string leaseTtlMs?: number @@ -515,9 +527,13 @@ export interface KnowledgeImprovementOptions { acquireKnowledge?: NonNullable updateKnowledge?: KnowledgeImprovementUpdate evaluateAnswers?: NonNullable - decidePromotion?: NonNullable + answerQualityCostCeiling?: RunRagKnowledgeImprovementLoopOptions['answerQualityCostCeiling'] + decidePromotion?: NonNullable enabledPhases?: readonly RagKnowledgeImprovementPhase[] requiredPhases?: readonly RagKnowledgeImprovementPhase[] + /** Repeatable candidate screening that must not use final cases. */ + evaluateDevelopment?: KnowledgeImprovementEvaluator + /** Single-use final evaluator. A failure ends the run. */ evaluate?: KnowledgeImprovementEvaluator signal?: AbortSignal now?: () => Date diff --git a/src/kb-improvement/evaluation.ts b/src/kb-improvement/evaluation.ts index 90e3593..b6b9a55 100644 --- a/src/kb-improvement/evaluation.ts +++ b/src/kb-improvement/evaluation.ts @@ -4,7 +4,9 @@ import { writeJsonDurableWithinRoot } from '../durable-fs' import type { EvalKnowledgeBundleBuildResult, KnowledgeReadinessSpec } from '../eval-readiness' import { knowledgeFileTransactionPlanHash } from '../file-transaction' import { sha256 } from '../ids' +import { assertImmutableRef } from '../immutable-ref' import { buildKnowledgeIndex } from '../indexer' +import { ragAnswerEvidenceRejectionReasons } from '../rag-answer-evidence' import { type KnowledgeBaseQualityReport, scoreKnowledgeBaseIndex } from '../rag-eval' import { type RagKnowledgeImprovementPhase, @@ -27,7 +29,12 @@ import { KnowledgeImprovementEvidenceSchema, UPDATE_PHASES, } from './contracts' -import { appendLedger, candidateEvidenceRelativePath, withCandidateWorkspace } from './state' +import { + appendLedger, + candidateEvidenceRelativePath, + saveState, + withCandidateWorkspace, +} from './state' import { knowledgeFilePlanEntries } from './transition' import { assertCandidateEvidence, @@ -39,6 +46,27 @@ import { } from './workspace' export function assertKnowledgeImprovementOptions(options: KnowledgeImprovementOptions): void { + assertImmutableRef(options.implementationRef, 'knowledge improvement implementationRef') + if ( + options.answerQualityCostCeiling !== undefined && + (!Number.isFinite(options.answerQualityCostCeiling) || options.answerQualityCostCeiling < 0) + ) { + throw new Error( + 'knowledge improvement answerQualityCostCeiling must be a non-negative finite number', + ) + } + if (options.ragOptimization) { + assertImmutableRef( + options.ragOptimization.executionRef, + 'knowledge improvement RAG executionRef', + ) + } + if (options.retrieval) { + assertImmutableRef( + options.retrieval.executionRef, + 'knowledge improvement retrieval executionRef', + ) + } if (options.step && options.knowledgeResearch?.step) { throw new Error('improveKnowledgeBase accepts either step or knowledgeResearch.step, not both') } @@ -64,6 +92,7 @@ export async function measureCandidate( candidate: KnowledgeImprovementCandidateRecord evaluation: KnowledgeImprovementMetric lifecycle?: RunRagKnowledgeImprovementLoopResult + finalEvaluated: boolean }> { return withCandidateWorkspace(runDir, candidate, async (candidateRoot) => { const currentCandidateHash = await hashKnowledgeBase(candidateRoot) @@ -76,8 +105,16 @@ export async function measureCandidate( const evidence = await assertCandidateEvidence( runDir, candidateRefFor(runId, state, candidate), + state.implementationRef, ) - return { candidate, evaluation: evidence.evaluation } + return { + candidate, + evaluation: evidence.evaluation, + ...(evidence.lifecycle === null + ? {} + : { lifecycle: evidence.lifecycle as RunRagKnowledgeImprovementLoopResult }), + finalEvaluated: shouldRunEvaluationStage(options), + } } clearCandidateMeasurement(candidate) @@ -93,16 +130,45 @@ export async function measureCandidate( if (updateLifecycle) lifecycles.push(updateLifecycle) } return withFrozenCandidateWorkspace(runDir, candidate, candidateRoot, async (snapshot) => { + let lifecycle = mergeLifecycleResults(options.goal, lifecycles) + const development = await evaluateCandidate( + runDir, + state, + candidate, + snapshot, + lifecycle, + options, + now, + false, + ) + if (!development.evaluation.passed || !shouldRunEvaluationStage(options)) { + return { + ...development, + ...(lifecycle ? { lifecycle } : {}), + finalEvaluated: false, + } + } + + candidate.finalEvaluationStartedAt = now().toISOString() + candidate.updatedAt = candidate.finalEvaluationStartedAt + state.updatedAt = candidate.finalEvaluationStartedAt + await saveState(runDir, state, options.onState) + await appendLedger(runDir, { + type: 'candidate.final-evaluation-started', + runId: state.runId, + candidateId: candidate.candidateId, + }) const evaluationLifecycle = await runCandidateEvaluationLifecycle( runId, runDir, candidate, snapshot.root, + snapshot.hash, options, now, ) if (evaluationLifecycle) lifecycles.push(evaluationLifecycle) - const lifecycle = mergeLifecycleResults(options.goal, lifecycles) + lifecycle = mergeLifecycleResults(options.goal, lifecycles) const measured = await evaluateCandidate( runDir, state, @@ -111,8 +177,13 @@ export async function measureCandidate( lifecycle, options, now, + true, ) - return { ...measured, ...(lifecycle ? { lifecycle } : {}) } + return { + ...measured, + ...(lifecycle ? { lifecycle } : {}), + finalEvaluated: true, + } }) }) } @@ -143,6 +214,7 @@ async function runCandidateEvaluationLifecycle( runDir: string, candidate: KnowledgeImprovementCandidateRecord, candidateRoot: string, + candidateHash: string, options: KnowledgeImprovementOptions, now: () => Date, ): Promise { @@ -154,6 +226,10 @@ async function runCandidateEvaluationLifecycle( optimization: options.ragOptimization ? { ...options.ragOptimization, + executionRef: candidateExecutionRef( + options.ragOptimization.executionRef, + candidateHash, + ), runDir: options.ragOptimization.runDir ?? join(runDir, 'rag-optimization', candidate.candidateId), @@ -174,13 +250,15 @@ async function runCandidateEvaluationLifecycle( retrieval: options.retrieval ? { ...options.retrieval, + executionRef: candidateExecutionRef(options.retrieval.executionRef, candidateHash), index: candidateIndex, runDir: options.retrieval.runDir ?? join(runDir, 'retrieval', candidate.candidateId), } : undefined, diagnose: options.diagnose, evaluateAnswers: options.evaluateAnswers, - promote: options.decidePromotion, + answerQualityCostCeiling: options.answerQualityCostCeiling, + decidePromotion: options.decidePromotion, enabledPhases: selectedStagePhases(options, EVALUATION_PHASES), requiredPhases: selectedStageRequiredPhases(options, EVALUATION_PHASES), signal: options.signal, @@ -189,6 +267,10 @@ async function runCandidateEvaluationLifecycle( ) } +function candidateExecutionRef(executionRef: string, candidateHash: string): string { + return `sha256:${sha256(`${executionRef}\n${candidateHash}`)}` +} + function candidateKnowledgeResearchOptions( candidateRoot: string, options: KnowledgeImprovementOptions, @@ -250,6 +332,7 @@ function shouldRunEvaluationStage(options: KnowledgeImprovementOptions): boolean options.diagnose || options.evaluateAnswers || options.decidePromotion || + options.evaluate || selectedStageRequiredPhases(options, EVALUATION_PHASES).length > 0, ) } @@ -302,6 +385,7 @@ async function evaluateCandidate( lifecycle: RunRagKnowledgeImprovementLoopResult | undefined, options: KnowledgeImprovementOptions, now: () => Date, + useConfiguredEvaluator: boolean, ): Promise<{ candidate: KnowledgeImprovementCandidateRecord evaluation: KnowledgeImprovementMetric @@ -318,23 +402,27 @@ async function evaluateCandidate( ...options.kbQuality, }) const candidateHash = snapshot.hash + const evaluator = useConfiguredEvaluator ? options.evaluate : options.evaluateDevelopment + const configuredMetric = evaluator + ? evaluator({ + runId: state.runId, + iteration: candidate.iteration, + root: options.root, + baselineRoot, + candidateRoot: snapshot.root, + baselineIndex, + candidateIndex, + baseHash: state.baseHash, + candidateHash, + validation, + readiness, + kbQuality, + lifecycle, + signal: options.signal, + }) + : undefined const metric = - options.evaluate?.({ - runId: state.runId, - iteration: candidate.iteration, - root: options.root, - baselineRoot, - candidateRoot: snapshot.root, - baselineIndex, - candidateIndex, - baseHash: state.baseHash, - candidateHash, - validation, - readiness, - kbQuality, - lifecycle, - signal: options.signal, - }) ?? + configuredMetric ?? defaultKnowledgeImprovementMetric( validation, readiness, @@ -342,7 +430,11 @@ async function evaluateCandidate( kbQuality, lifecycle, ) - const evaluation = applyLifecycleFailures(normalizeMetric(await metric), lifecycle) + const evaluation = applyLifecycleFailures( + normalizeMetric(await metric), + lifecycle, + options.answerQualityCostCeiling, + ) const measuredHash = await hashKnowledgeBase(snapshot.root) if (measuredHash !== candidateHash) { throw new Error( @@ -361,6 +453,7 @@ async function evaluateCandidate( candidateId: candidate.candidateId, iteration: candidate.iteration, goalHash: sha256(state.goal), + implementationRef: state.implementationRef, baseHash: candidate.baseHash, candidateHash, promotionPlanHash: candidate.promotionPlanHash, @@ -436,18 +529,20 @@ function defaultKnowledgeImprovementMetric( function applyLifecycleFailures( metric: KnowledgeImprovementMetric, lifecycle: RunRagKnowledgeImprovementLoopResult | undefined, + answerQualityCostCeiling: number | undefined, ): KnowledgeImprovementMetric { + const answerQualityFailures = lifecycle?.answerQuality + ? ragAnswerEvidenceRejectionReasons(lifecycle.answerQuality, answerQualityCostCeiling) + : [] const reasons = [ metric.notes, - lifecycle?.answerQuality && !lifecycle.answerQuality.passed - ? 'answer quality failed' - : undefined, + ...answerQualityFailures, lifecycle?.promotion && !lifecycle.promotion.promoted ? `promotion decision held: ${lifecycle.promotion.reason}` : undefined, ].filter((reason): reason is string => Boolean(reason)) const forcedFailure = - Boolean(lifecycle?.answerQuality && !lifecycle.answerQuality.passed) || + answerQualityFailures.length > 0 || Boolean(lifecycle?.promotion && !lifecycle.promotion.promoted) return { ...metric, diff --git a/src/kb-improvement/optimization.ts b/src/kb-improvement/optimization.ts index 81bbc11..0069268 100644 --- a/src/kb-improvement/optimization.ts +++ b/src/kb-improvement/optimization.ts @@ -1,10 +1,11 @@ import type { DispatchContext, - JsonValue, OptimizationMethod, Scenario, } from '@tangle-network/agent-eval/campaign' -import { stableId } from '../ids' +import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' +import { sha256, stableId } from '../ids' +import { assertImmutableRef } from '../immutable-ref' import { type RunSerializedKnowledgeOptimizationOptions, type RunSerializedKnowledgeOptimizationResult, @@ -20,7 +21,14 @@ import { hashKnowledgeBase } from './workspace' type PolicyCandidateOptions = Omit< KnowledgeImprovementOptions, - 'root' | 'goal' | 'runId' | 'maxCandidates' | 'step' | 'knowledgeResearch' | 'updateKnowledge' + | 'root' + | 'goal' + | 'implementationRef' + | 'runId' + | 'maxCandidates' + | 'step' + | 'knowledgeResearch' + | 'updateKnowledge' > type PolicyOptimizationBaseOptions< @@ -29,7 +37,12 @@ type PolicyOptimizationBaseOptions< TArtifact, > = Omit< RunSerializedKnowledgeOptimizationOptions, - 'baseline' | 'method' | 'trainScenarios' | 'selectionScenarios' | 'finalScenarios' + | 'baseline' + | 'method' + | 'trainScenarios' + | 'selectionScenarios' + | 'finalScenarios' + | 'executionRef' > export interface OptimizeKnowledgeBasePolicyOptions< @@ -44,7 +57,7 @@ export interface OptimizeKnowledgeBasePolicyOptions< trainScenarios: readonly TScenario[] selectionScenarios: readonly TScenario[] finalScenarios: readonly TScenario[] - /** Stable version for applyPolicy and its external dependencies. */ + /** Commit or content identity for evaluation, applyPolicy, and external dependencies. */ policyApplicationRef: string /** Optional namespace for parallel materialization of the same measured policy. */ candidateRunLabel?: string @@ -99,9 +112,7 @@ export async function optimizeKnowledgeBasePolicy< if (typeof goal !== 'string' || !goal.trim()) { throw new Error('optimizeKnowledgeBasePolicy goal must be non-empty') } - if (typeof policyApplicationRef !== 'string' || !policyApplicationRef.trim()) { - throw new Error('optimizeKnowledgeBasePolicy policyApplicationRef must be non-empty') - } + assertImmutableRef(policyApplicationRef, 'optimizeKnowledgeBasePolicy policyApplicationRef') if ( candidateRunLabel !== undefined && (typeof candidateRunLabel !== 'string' || !candidateRunLabel.trim()) @@ -111,6 +122,7 @@ export async function optimizeKnowledgeBasePolicy< const baseHash = await hashKnowledgeBase(root) const optimization = await runSerializedKnowledgeOptimization({ ...optimizationOptions, + executionRef: policyApplicationRef, baseline: baselinePolicy, method, trainScenarios, @@ -132,6 +144,9 @@ export async function optimizeKnowledgeBasePolicy< ...(candidate ?? {}), root, goal, + implementationRef: `sha256:${sha256( + `${policyApplicationRef}\n${winner.surfaceHash}\n${optimization.methodName}`, + )}`, runId, maxCandidates: 1, updateKnowledge: async (input) => { diff --git a/src/kb-improvement/run.ts b/src/kb-improvement/run.ts index 898a980..953fef7 100644 --- a/src/kb-improvement/run.ts +++ b/src/kb-improvement/run.ts @@ -71,6 +71,7 @@ async function improveKnowledgeBaseInRun( runId, root: options.root, goal: options.goal, + implementationRef: options.implementationRef, status: 'running', baseHash, createdAt: now().toISOString(), @@ -84,6 +85,9 @@ async function improveKnowledgeBaseInRun( if (state.goal !== options.goal) { throw new Error('knowledge improvement state does not match the requested goal') } + if (state.implementationRef !== options.implementationRef) { + throw new Error('knowledge improvement state does not match the requested implementationRef') + } const promotedCandidateId = state.promotedCandidateId const promotedCandidate = state.status === 'promoted' @@ -105,6 +109,7 @@ async function improveKnowledgeBaseInRun( const evidence = await assertCandidateEvidence( runDir, candidateRefFor(runId, promotedState, promoted), + promotedState.implementationRef, ) return { runId, @@ -130,6 +135,22 @@ async function improveKnowledgeBaseInRun( let lifecycle: RunRagKnowledgeImprovementLoopResult | undefined while (candidate || state.candidates.length < maxCandidates) { + if (candidate?.status === 'running' && candidate.finalEvaluationStartedAt) { + state = await blockRun( + runDir, + state, + `candidate '${candidate.candidateId}' was interrupted after final evaluation started; refusing to reuse final cases`, + options.onState, + now, + ) + return { + runId, + state, + candidate, + promoted: false, + blocked: true, + } + } if (!candidate) { const currentHash = await hashKnowledgeBase(options.root) if (currentHash !== state.baseHash) { @@ -189,6 +210,7 @@ async function improveKnowledgeBaseInRun( lastRejectedCandidate = candidate lastRejectedEvaluation = evaluation candidate = undefined + if (measured.finalEvaluated) break } if (!candidate) { @@ -206,7 +228,11 @@ async function improveKnowledgeBaseInRun( } } - const evidence = await assertCandidateEvidence(runDir, candidateRefFor(runId, state, candidate)) + const evidence = await assertCandidateEvidence( + runDir, + candidateRefFor(runId, state, candidate), + state.implementationRef, + ) return { runId, state, diff --git a/src/kb-improvement/transition.ts b/src/kb-improvement/transition.ts index 73c1511..400c7b5 100644 --- a/src/kb-improvement/transition.ts +++ b/src/kb-improvement/transition.ts @@ -46,6 +46,7 @@ import { withKnowledgeImprovementRun, } from './state' import { + assertCandidateEvidence, assertStateIdentity, candidateIdentityFor, hashKnowledgeBase, @@ -139,10 +140,9 @@ async function applyKnowledgeCandidateTarget( ) { throw new Error('knowledge candidate approval does not match the measured candidate') } - const action = target === 'candidate' ? 'promotion' : 'restore' const desiredHash = target === 'candidate' ? candidateRef.candidateHash : candidateRef.baseHash - const purpose = knowledgeCandidateTransitionPurpose(candidateRef, target) + const purpose = knowledgeCandidateTransitionPurpose(candidateRef, state.implementationRef, target) const recoveryOwner = input.activation ? `knowledge-improvement-activation:${input.activation.activation.digest}` : 'knowledge-improvement-candidate-transition' @@ -157,6 +157,9 @@ async function applyKnowledgeCandidateTarget( const transactionRoot = mutationLock.transactionRoot const recovery = mutationLock.recovery?.purpose === purpose ? mutationLock.recovery : undefined + if (!recovery) { + await assertCandidateEvidence(runDir, candidateRef, state.implementationRef) + } const existingActivation = input.activation ? await loadKnowledgeActivationRecord( runDir, @@ -500,10 +503,11 @@ function candidateTransitionResult( function knowledgeCandidateTransitionPurpose( candidate: KnowledgeImprovementCandidateRef, + implementationRef: string, target: KnowledgeImprovementTarget, ): string { const action = target === 'candidate' ? 'promotion' : 'restore' - return `knowledge-${action}:${contentHash(candidate)}` + return `knowledge-${action}:${contentHash({ candidate, implementationRef })}` } export async function knowledgeFilePlanEntries( diff --git a/src/kb-improvement/workspace.ts b/src/kb-improvement/workspace.ts index d830d66..5e261a0 100644 --- a/src/kb-improvement/workspace.ts +++ b/src/kb-improvement/workspace.ts @@ -164,7 +164,7 @@ export async function withMeasuredCandidateSnapshot( if (canonicalJson(expectedRef) !== canonicalJson(candidateRef)) { throw new Error('knowledge candidate approval does not match the measured candidate') } - const evidence = await assertCandidateEvidence(runDir, candidateRef) + const evidence = await assertCandidateEvidence(runDir, candidateRef, state.implementationRef) const relativePath = join( 'candidates', candidate.candidateId, @@ -209,6 +209,7 @@ async function withIsolatedKnowledgeCopy( export async function assertCandidateEvidence( runDir: string, candidate: KnowledgeImprovementCandidateRef, + expectedImplementationRef: string, ): Promise { const evidence = KnowledgeImprovementEvidenceSchema.parse( JSON.parse( @@ -230,6 +231,7 @@ export async function assertCandidateEvidence( evidence.runId !== candidate.runId || evidence.candidateId !== candidate.candidateId || evidence.goalHash !== candidate.goalHash || + evidence.implementationRef !== expectedImplementationRef || evidence.baseHash !== candidate.baseHash || evidence.candidateHash !== candidate.candidateHash || evidence.promotionPlanHash !== candidate.promotionPlanHash || diff --git a/src/memory/attempt-log.ts b/src/memory/attempt-log.ts index ea0c105..9f69384 100644 --- a/src/memory/attempt-log.ts +++ b/src/memory/attempt-log.ts @@ -6,7 +6,6 @@ export interface AttemptJournalEvent { } interface RecoveryAttemptEvent { - schema: 1 attemptId: string generation: number recordedAt: string @@ -162,7 +161,6 @@ export function reserveRecoveryAttempts(input: { path, label, event: { - schema: 1, attemptId, generation, recordedAt: (input.now ?? (() => new Date()))().toISOString(), @@ -263,7 +261,6 @@ function parseRecoveryAttemptEvent( const valid = typeof event === 'object' && event !== null && - event.schema === 1 && typeof event.attemptId === 'string' && event.attemptId.length > 0 && Number.isSafeInteger(event.generation) && diff --git a/src/memory/experiment.ts b/src/memory/experiment.ts index 71b9b73..07961b4 100644 --- a/src/memory/experiment.ts +++ b/src/memory/experiment.ts @@ -6,6 +6,12 @@ export { export { runAgentMemoryExperiment } from './experiment/run' export type { AgentMemoryAttemptEvent, + AgentMemoryExecutionContext, + AgentMemoryExecutionCostMeter, + AgentMemoryExecutionCostReceipt, + AgentMemoryExecutionPaidCallInput, + AgentMemoryExecutionPaidCallResult, + AgentMemoryExecutionStep, AgentMemoryExperimentCandidate, AgentMemoryExperimentRankingRow, AgentMemoryExperimentRunLease, diff --git a/src/memory/experiment/cell.ts b/src/memory/experiment/cell.ts index 585410c..f560efe 100644 --- a/src/memory/experiment/cell.ts +++ b/src/memory/experiment/cell.ts @@ -5,6 +5,8 @@ import { stableId } from '../../ids' import { type AgentMemoryBranch, createAgentMemoryBranch } from '../branch' import { resolveMemoryCleanupTimeoutMs, runBoundedMemoryLifecycle } from '../lifecycle' import type { AgentMemoryAdapter } from '../types' +import { createAgentMemoryCostRecorder } from './cost' +import { createAgentMemoryExecutionContext } from './execution-context' import { countDimensions, mean, meanDimensions } from './metrics' import { appendMemoryAttemptEvent, memoryAttemptCostCallId, memoryAttemptEvent } from './recovery' import { @@ -16,6 +18,7 @@ import { trackExternalMemoryCalls, } from './runtime' import type { + AgentMemoryExecutionStep, AgentMemoryExperimentCandidate, AgentMemorySequence, AgentMemorySequenceArtifact, @@ -78,6 +81,11 @@ export async function runSequenceCell(input: { appendMemoryAttemptEvent(storage, attemptLogPath, attempt) let externalCallAttempted = false + const costRecorder = createAgentMemoryCostRecorder({ + candidateRef: candidate.ref, + maximumCostUsd: costUsd, + operation: `${candidate.id}: memory sequence`, + }) const appendCleanedAttempt = (priorError?: unknown): void => { try { appendMemoryAttemptEvent(storage, attemptLogPath, { @@ -105,14 +113,16 @@ export async function runSequenceCell(input: { try { const created = await candidate.createAdapter({ branchId, - sequence: scenario.sequence, - rep: context.rep, - seed: context.seed, purpose: 'execute', signal: context.signal, + maximumCostUsd: costUsd, markExternalCall: () => { externalCallAttempted = true }, + recordExternalCost: (actualCostUsd) => { + externalCallAttempted = true + costRecorder.record(actualCostUsd) + }, }) if (!created) throw new Error(`${candidate.id}: createAdapter returned no execution adapter`) rawAdapter = created @@ -135,18 +145,24 @@ export async function runSequenceCell(input: { baseScope: memoryExperimentBaseScope(options, candidate, scenario.sequenceId), }) const probes: AgentMemorySequenceProbeResult[] = [] - for (const step of scenario.sequence.steps) { + for (const [ordinal, step] of scenario.sequence.steps.entries()) { context.signal.throwIfAborted() await lease.assertOwned() await writeStep(memory, step) await lease.assertOwned() - await options.executeStep?.({ - memory, - candidateId: candidate.id, - sequence: scenario.sequence, - step, - context, - }) + if (options.executeStep) { + const execution = createAgentMemoryExecutionContext(context, scenario.sequence) + try { + await options.executeStep({ + memory, + candidateId: candidate.id, + step: executionStep(step, ordinal), + context: execution.context, + }) + } finally { + execution.dispose() + } + } context.signal.throwIfAborted() await lease.assertOwned() const stepProbes = await probeStep(memory, scenario.sequence, step) @@ -295,28 +311,27 @@ export async function runSequenceCell(input: { if (!artifact) throw new Error(`${candidate.id}: memory sequence produced no result`) return artifact } - const receipt = { - model: candidate.id, - inputTokens: 0, - outputTokens: 0, - actualCostUsd: costUsd, - } as const const paid = await context.cost.runPaidCall({ callId: memoryAttemptCostCallId(attempt, 'execute', 0), actor: `agent-knowledge:memory-experiment:${candidate.id}`, - model: candidate.id, + model: candidate.ref, maximumCharge: { externallyEnforcedMaximumUsd: costUsd }, execute, - receipt: () => receipt, - receiptFromError: () => ({ - ...receipt, - actualCostUsd: externalCallAttempted ? costUsd : 0, - }), + receipt: () => costRecorder.receipt(externalCallAttempted), + receiptFromError: () => costRecorder.receipt(externalCallAttempted), }) if (!paid.succeeded) throw paid.error return paid.value } +function executionStep(step: AgentMemorySequenceStep, ordinal: number): AgentMemoryExecutionStep { + return { + ordinal, + ...(step.instruction !== undefined ? { instruction: step.instruction } : {}), + ...(step.scope !== undefined ? { scope: structuredClone(step.scope) } : {}), + } +} + async function writeStep(memory: AgentMemoryBranch, step: AgentMemorySequenceStep): Promise { const writes = (step.writes ?? []).map((write) => ({ ...write, diff --git a/src/memory/experiment/cost.ts b/src/memory/experiment/cost.ts new file mode 100644 index 0000000..7c1de9d --- /dev/null +++ b/src/memory/experiment/cost.ts @@ -0,0 +1,58 @@ +export interface AgentMemoryCostReceipt { + model: string + inputTokens: 0 + outputTokens: 0 + actualCostUsd?: number + costUnknown?: boolean +} + +export interface AgentMemoryCostRecorder { + record(actualCostUsd: number): void + receipt(externalCallAttempted: boolean): AgentMemoryCostReceipt +} + +export function createAgentMemoryCostRecorder(input: { + candidateRef: string + maximumCostUsd: number + operation: string +}): AgentMemoryCostRecorder { + let observedCostUsd = 0 + let receiptCount = 0 + + return { + record(actualCostUsd) { + if (!Number.isFinite(actualCostUsd) || actualCostUsd < 0) { + throw new Error( + `${input.operation}: observed external cost must be non-negative and finite`, + ) + } + const next = observedCostUsd + actualCostUsd + if (exceedsMaximum(next, input.maximumCostUsd)) { + throw new Error( + `${input.operation}: observed external cost ${next} exceeds declared maximum ${input.maximumCostUsd}`, + ) + } + observedCostUsd = next + receiptCount += 1 + }, + receipt(externalCallAttempted) { + const base = { + model: input.candidateRef, + inputTokens: 0 as const, + outputTokens: 0 as const, + } + if (input.maximumCostUsd === 0 || !externalCallAttempted) { + return { ...base, actualCostUsd: 0 } + } + if (receiptCount === 0) { + return { ...base, costUnknown: true } + } + return { ...base, actualCostUsd: observedCostUsd } + }, + } +} + +function exceedsMaximum(actual: number, maximum: number): boolean { + const tolerance = Number.EPSILON * Math.max(1, Math.abs(actual), Math.abs(maximum)) * 8 + return actual - maximum > tolerance +} diff --git a/src/memory/experiment/execution-context.ts b/src/memory/experiment/execution-context.ts new file mode 100644 index 0000000..06f5f69 --- /dev/null +++ b/src/memory/experiment/execution-context.ts @@ -0,0 +1,167 @@ +import type { CostReceipt, PaidCallResult } from '@tangle-network/agent-eval' +import type { DispatchContext } from '@tangle-network/agent-eval/campaign' +import type { + AgentMemoryExecutionContext, + AgentMemoryExecutionCostMeter, + AgentMemoryExecutionCostReceipt, + AgentMemoryExecutionPaidCallInput, + AgentMemoryExecutionPaidCallResult, + AgentMemorySequence, +} from './types' + +export interface OwnedAgentMemoryExecutionContext { + context: AgentMemoryExecutionContext + dispose(): void +} + +export function createAgentMemoryExecutionContext( + dispatch: DispatchContext, + sequence: AgentMemorySequence, +): OwnedAgentMemoryExecutionContext { + const privateValues = memoryEvaluationPrivateValues(dispatch.cellId, sequence) + const signal = relayAbortWithoutReason(dispatch.signal) + const cost: AgentMemoryExecutionCostMeter = Object.freeze({ + async runPaidCall( + input: AgentMemoryExecutionPaidCallInput, + ): Promise> { + const result = await dispatch.cost.runPaidCall({ + ...input, + execute: (sourceSignal, callId) => + withRedactedAbortSignal(sourceSignal, (redactedSignal) => + input.execute(redactedSignal, callId), + ), + }) + return sanitizePaidCallResult(result, privateValues) + }, + }) + return { + context: Object.freeze({ signal: signal.signal, cost }), + dispose: signal.dispose, + } +} + +function sanitizePaidCallResult( + result: PaidCallResult, + privateValues: readonly string[], +): AgentMemoryExecutionPaidCallResult { + if (result.succeeded) { + return Object.freeze({ + succeeded: true, + callId: result.callId, + value: result.value, + receipt: sanitizeCostReceipt(result.receipt, privateValues), + }) + } + return Object.freeze({ + succeeded: false, + ...(result.callId === undefined ? {} : { callId: result.callId }), + error: sanitizeError(result.error, privateValues), + ...(result.receipt === undefined + ? {} + : { receipt: sanitizeCostReceipt(result.receipt, privateValues) }), + }) +} + +function sanitizeCostReceipt( + receipt: CostReceipt, + privateValues: readonly string[], +): AgentMemoryExecutionCostReceipt { + const { tags: _tags, phase: _phase, error, ...visible } = receipt + return Object.freeze({ + ...visible, + ...(error === undefined ? {} : { error: redactPrivateValues(error, privateValues) }), + }) +} + +function sanitizeError(error: Error, privateValues: readonly string[]): Error { + const sanitized = new Error(redactPrivateValues(error.message, privateValues)) + sanitized.name = redactPrivateValues(error.name, privateValues) + return sanitized +} + +async function withRedactedAbortSignal( + source: AbortSignal, + run: (signal: AbortSignal) => Promise, +): Promise { + const relay = relayAbortWithoutReason(source) + try { + return await run(relay.signal) + } finally { + relay.dispose() + } +} + +function relayAbortWithoutReason(source: AbortSignal): { + signal: AbortSignal + dispose(): void +} { + const controller = new AbortController() + const abort = () => controller.abort(memoryExecutionAbortError()) + if (source.aborted) abort() + else source.addEventListener('abort', abort, { once: true }) + return { + signal: controller.signal, + dispose() { + source.removeEventListener('abort', abort) + }, + } +} + +function memoryExecutionAbortError(): Error { + const error = new Error('memory execution aborted') + error.name = 'AbortError' + return error +} + +function memoryEvaluationPrivateValues( + cellId: string, + sequence: AgentMemorySequence, +): readonly string[] { + const values = new Set([cellId, sequence.id]) + collectUnknownStrings(sequence.metadata, values) + for (const step of sequence.steps) { + values.add(step.id) + collectUnknownStrings(step.metadata, values) + for (const probe of step.probes ?? []) { + values.add(probe.id) + values.add(probe.query) + if (probe.referenceAnswer !== undefined) values.add(probe.referenceAnswer) + for (const matcher of [...(probe.requiredFacts ?? []), ...(probe.forbiddenFacts ?? [])]) { + values.add(matcher.id) + for (const expected of matcher.anyOf) values.add(expected) + } + for (const expected of probe.expectedEventIds ?? []) values.add(expected) + for (const expected of probe.expectedActorIds ?? []) values.add(expected) + } + } + return [...values].filter(Boolean).sort((left, right) => right.length - left.length) +} + +function collectUnknownStrings( + value: unknown, + output: Set, + seen = new WeakSet(), +): void { + if (typeof value === 'string') { + output.add(value) + return + } + if (!value || typeof value !== 'object' || seen.has(value)) return + seen.add(value) + if (Array.isArray(value)) { + for (const item of value) collectUnknownStrings(item, output, seen) + return + } + for (const [key, item] of Object.entries(value)) { + output.add(key) + collectUnknownStrings(item, output, seen) + } +} + +function redactPrivateValues(value: string, privateValues: readonly string[]): string { + let redacted = value + for (const privateValue of privateValues) { + redacted = redacted.split(privateValue).join('[redacted]') + } + return redacted +} diff --git a/src/memory/experiment/recovery.ts b/src/memory/experiment/recovery.ts index 9ea2f20..b64ed06 100644 --- a/src/memory/experiment/recovery.ts +++ b/src/memory/experiment/recovery.ts @@ -19,6 +19,7 @@ import { sleepForMemoryRecovery, } from '../lifecycle' import type { AgentMemoryAdapter } from '../types' +import { createAgentMemoryCostRecorder } from './cost' import { AgentMemoryCleanupError, clearSequenceScopes, @@ -130,6 +131,11 @@ export async function recoverAbandonedMemoryAttempts(input: { throw new Error(`missing recovery generation for memory branch '${attempt.branchId}'`) } let externalRecoveryAttempted = false + const costRecorder = createAgentMemoryCostRecorder({ + candidateRef: candidate.ref, + maximumCostUsd: recoveryCostUsd, + operation: `${candidate.id}: memory recovery`, + }) const recover = async (): Promise => { await recoverMemoryAttempt({ options: input.options, @@ -140,6 +146,10 @@ export async function recoverAbandonedMemoryAttempts(input: { onExternalCall: () => { externalRecoveryAttempted = true }, + recordExternalCost: (actualCostUsd) => { + externalRecoveryAttempted = true + costRecorder.record(actualCostUsd) + }, }) appendMemoryAttemptEvent(input.storage, input.attemptLogPath, { ...attempt, @@ -152,29 +162,17 @@ export async function recoverAbandonedMemoryAttempts(input: { await recover() } else { const tags = memoryRecoveryCostTags(input.runDir, candidate.id, attempt.branchId) - const receipt = { - model: candidate.id, - inputTokens: 0, - outputTokens: 0, - actualCostUsd: recoveryCostUsd, - } as const const paid = await input.costLedger.runPaidCall({ callId: memoryAttemptCostCallId(attempt, 'recovery', recoveryGeneration), channel: 'driver', phase: `${input.options.costPhase ?? 'memory.experiment'}.recovery`, actor: `agent-knowledge:memory-recovery:${candidate.id}`, - model: candidate.id, + model: candidate.ref, tags, maximumCharge: { externallyEnforcedMaximumUsd: recoveryCostUsd }, execute: recover, - receipt: () => ({ - ...receipt, - actualCostUsd: externalRecoveryAttempted ? recoveryCostUsd : 0, - }), - receiptFromError: () => ({ - ...receipt, - actualCostUsd: externalRecoveryAttempted ? recoveryCostUsd : 0, - }), + receipt: () => costRecorder.receipt(externalRecoveryAttempted), + receiptFromError: () => costRecorder.receipt(externalRecoveryAttempted), }) if (!paid.succeeded) throw paid.error } @@ -197,8 +195,9 @@ async function recoverMemoryAttempt(input: { attempt: AgentMemoryAttemptEvent lease: OwnedMemoryExperimentRunLease onExternalCall(): void + recordExternalCost(actualCostUsd: number): void }): Promise { - const { options, candidate, sequence, attempt, lease, onExternalCall } = input + const { options, candidate, sequence, attempt, lease, onExternalCall, recordExternalCost } = input const cleanupBranches = attempt.cleanupBranches const cleanupTimeoutMs = resolveMemoryCleanupTimeoutMs( options.cleanupTimeoutMs, @@ -213,12 +212,11 @@ async function recoverMemoryAttempt(input: { const creation = Promise.resolve().then(() => candidate.createAdapter({ branchId: attempt.branchId, - sequence, - rep: attempt.rep, - seed: attempt.seed, purpose: 'recovery', signal: abortController.signal, + maximumCostUsd: candidate.externalRecoveryCostUsdPerAttempt ?? 0, markExternalCall: onExternalCall, + recordExternalCost, }), ) releaseMemoryAdapterCreatedAfterAbort({ @@ -377,7 +375,6 @@ export function memoryAttemptEvent(input: { now?: () => Date }): AgentMemoryAttemptEvent { return { - schema: 2, status: input.status, branchId: input.branchId, candidateId: input.candidate.id, @@ -429,7 +426,6 @@ function parseMemoryAttemptEvent( const valid = typeof event === 'object' && event !== null && - event.schema === 2 && (event.status === 'started' || event.status === 'cleaned') && typeof event.branchId === 'string' && event.branchId.length > 0 && diff --git a/src/memory/experiment/run.ts b/src/memory/experiment/run.ts index 92ffccf..4e57a2f 100644 --- a/src/memory/experiment/run.ts +++ b/src/memory/experiment/run.ts @@ -80,6 +80,19 @@ export async function runAgentMemoryExperiment( `${candidate.id}: externalRecoveryCostUsdPerAttempt must be a non-negative finite number`, ) } + if ( + candidate.externalCostAccounting !== undefined && + candidate.externalCostAccounting !== 'exact' + ) { + throw new Error(`${candidate.id}: externalCostAccounting must be 'exact'`) + } + if ( + ((candidate.externalCostUsdPerSequence ?? 0) > 0 || + (candidate.externalRecoveryCostUsdPerAttempt ?? 0) > 0) && + candidate.externalCostAccounting !== 'exact' + ) { + throw new Error(`${candidate.id}: positive external charges require exact cost accounting`) + } assertNonEmptyString(candidate.ref, `${candidate.id} ref`) } @@ -316,6 +329,7 @@ function memoryExperimentDispatchRef(options: RunAgentMemoryExperimentOptions): baseScope: candidate.baseScope ?? null, externalCostUsdPerSequence: candidate.externalCostUsdPerSequence ?? 0, externalRecoveryCostUsdPerAttempt: candidate.externalRecoveryCostUsdPerAttempt ?? 0, + externalCostAccounting: candidate.externalCostAccounting ?? 'exact', })) .sort((a, b) => a.id.localeCompare(b.id)), }), diff --git a/src/memory/experiment/types.ts b/src/memory/experiment/types.ts index 5e150fd..99a2898 100644 --- a/src/memory/experiment/types.ts +++ b/src/memory/experiment/types.ts @@ -1,8 +1,8 @@ +import type { CostChannel, CostReceipt, RunPaidCallInput } from '@tangle-network/agent-eval' import type { CampaignResult, CampaignStorage, CostLedgerHandle, - DispatchContext, Scenario, } from '@tangle-network/agent-eval/campaign' import type { @@ -50,6 +50,48 @@ export interface AgentMemorySequenceStep { metadata?: Record } +/** Runtime-visible step fields. Evaluation labels and dataset identity are excluded. */ +export interface AgentMemoryExecutionStep { + ordinal: number + instruction?: string + scope?: AgentMemoryScope +} + +export type AgentMemoryExecutionPaidCallInput = Omit< + RunPaidCallInput, + 'channel' | 'phase' | 'tags' | 'signal' +> & { + channel?: CostChannel +} + +export type AgentMemoryExecutionCostReceipt = Omit + +export type AgentMemoryExecutionPaidCallResult = + | { + succeeded: true + callId: string + value: T + receipt: AgentMemoryExecutionCostReceipt + } + | { + succeeded: false + callId?: string + error: Error + receipt?: AgentMemoryExecutionCostReceipt + } + +export interface AgentMemoryExecutionCostMeter { + runPaidCall( + input: AgentMemoryExecutionPaidCallInput, + ): Promise> +} + +/** Execution capabilities with all campaign and evaluation identity removed. */ +export interface AgentMemoryExecutionContext { + readonly signal: AbortSignal + readonly cost: AgentMemoryExecutionCostMeter +} + export interface AgentMemorySequence { id: string family: KnowledgeBenchmarkFamily | string @@ -79,19 +121,22 @@ export interface AgentMemoryExperimentCandidate { /** Local construction is free; call markExternalCall before billable provisioning or reconnects. */ createAdapter(input: { branchId: string - sequence: AgentMemorySequence - rep: number - seed: number purpose: 'execute' | 'recovery' signal: AbortSignal + /** Maximum the adapter must enforce with its provider before external work. */ + maximumCostUsd: number markExternalCall(): void + /** Record each observed provider charge. Required for complete positive-cost accounting. */ + recordExternalCost(actualCostUsd: number): void }): AgentMemoryAdapter | null | Promise policy?: AgentMemorySharingPolicy baseScope?: AgentMemoryScope - /** Conservative external provider charge for one complete history. */ + /** Maximum the adapter must enforce for one complete history. Zero declares a free path. */ externalCostUsdPerSequence?: number - /** Conservative extra provider charge when recovering one interrupted history. */ + /** Maximum the adapter must enforce for one recovery attempt. Zero declares free recovery. */ externalRecoveryCostUsdPerAttempt?: number + /** Requires observed provider receipts for positive external charges. */ + externalCostAccounting?: 'exact' /** Release resources and, when cleanupBranches is false, delete the isolated state. */ disposeAdapter?(adapter: AgentMemoryAdapter): Promise } @@ -156,9 +201,8 @@ export interface RunAgentMemoryExperimentOptions { executeStep?: (input: { memory: AgentMemoryBranch candidateId: string - sequence: AgentMemorySequence - step: AgentMemorySequenceStep - context: DispatchContext + step: AgentMemoryExecutionStep + context: AgentMemoryExecutionContext }) => Promise /** Required with executeStep; identify the runtime/profile behavior in cache keys. */ executeStepRef?: string @@ -196,7 +240,6 @@ export interface RunAgentMemoryExperimentOptions { export type AgentMemoryExperimentRunLease = AgentMemoryRunLease export interface AgentMemoryAttemptEvent { - schema: 2 status: 'started' | 'cleaned' branchId: string candidateId: string diff --git a/src/memory/graphiti.ts b/src/memory/graphiti.ts index ea9c69a..3e1456d 100644 --- a/src/memory/graphiti.ts +++ b/src/memory/graphiti.ts @@ -565,5 +565,5 @@ export function graphitiMemoryAdapterIdentity( if (typeof options.backendRef !== 'string' || !options.backendRef.trim()) { throw new Error('Graphiti backendRef must be a non-empty string') } - return stableId('graphiti', canonicalJson(stripUndefined(options))) + return `sha256:${sha256(canonicalJson(stripUndefined(options)))}` } diff --git a/src/memory/improvement/activation.ts b/src/memory/improvement/activation.ts index e90411c..10e9eb9 100644 --- a/src/memory/improvement/activation.ts +++ b/src/memory/improvement/activation.ts @@ -1,8 +1,5 @@ -import { - type CampaignStorage, - type JsonValue, - surfaceHash, -} from '@tangle-network/agent-eval/campaign' +import { type CampaignStorage, surfaceHash } from '@tangle-network/agent-eval/campaign' +import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' import { appendDurableJournalEvent } from '../attempt-log' import { runBoundedMemoryLifecycle } from '../lifecycle' import { memoryConfigCodec } from './evaluation' @@ -65,6 +62,30 @@ export function readMemoryActivationJournal( return { prepared, ...(activated ? { activated } : {}) } } +export async function assertActivatedMemoryWinner(input: { + options: RunAgentMemoryImprovementOptions + lease: OwnedRunLease + result: Pick, 'winnerSurfaceHash'> +}): Promise { + const activationDriver = input.options.activation + if (!activationDriver) { + throw new Error('an activated memory journal requires its activation driver') + } + await input.lease.assertOwned() + const currentConfig = await runBoundedMemoryLifecycle({ + operation: `${activationDriver.ref}: confirm active memory configuration`, + timeoutMs: input.options.activationTimeoutMs ?? 60_000, + run: () => activationDriver.readCurrent(), + }) + await input.lease.assertOwned() + const currentHash = surfaceHash(memoryConfigCodec(input.options).serialize(currentConfig)) + if (currentHash !== input.result.winnerSurfaceHash) { + throw new Error( + `memory activation target '${activationDriver.ref}' drifted from measured winner '${input.result.winnerSurfaceHash}' to '${currentHash}'`, + ) + } +} + export async function activateMemoryWinner(input: { options: RunAgentMemoryImprovementOptions storage: CampaignStorage @@ -195,6 +216,19 @@ function parseMemoryActivationEvent( if (event.status !== 'prepared' && event.status !== 'activated') { throw new Error(`invalid memory activation journal '${path}' line ${line} status`) } + const expectedKeys = [ + ...Object.keys(expected), + 'status', + 'recordedAt', + ...(event.status === 'activated' ? ['outcome'] : []), + ] + const actualKeys = Object.keys(event) + if ( + actualKeys.length !== expectedKeys.length || + actualKeys.some((key) => !expectedKeys.includes(key)) + ) { + throw new Error(`invalid memory activation journal '${path}' line ${line} fields`) + } if (typeof event.recordedAt !== 'string' || !Number.isFinite(Date.parse(event.recordedAt))) { throw new Error(`invalid memory activation journal '${path}' line ${line} recordedAt`) } diff --git a/src/memory/improvement/candidate.ts b/src/memory/improvement/candidate.ts index be07971..d23bbca 100644 --- a/src/memory/improvement/candidate.ts +++ b/src/memory/improvement/candidate.ts @@ -1,4 +1,8 @@ -import type { CampaignStorage, JsonValue } from '@tangle-network/agent-eval/campaign' +import { dirname, join } from 'node:path' +import { canonicalJson } from '@tangle-network/agent-eval' +import type { CampaignStorage } from '@tangle-network/agent-eval/campaign' +import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' +import { assertImmutableRef } from '../../immutable-ref' import type { AgentMemoryExperimentCandidate, RunAgentMemoryExperimentOptions } from '../experiment' import type { OwnedRunLease, RunAgentMemoryImprovementOptions } from './types' @@ -13,6 +17,15 @@ export async function buildCandidate( candidateId: id, surfaceHash: hash, }) + assertImmutableRef(built.ref, `${id}: ref`) + assertDeclaredCost(built.externalCostUsdPerSequence, `${id}: externalCostUsdPerSequence`) + assertDeclaredCost( + built.externalRecoveryCostUsdPerAttempt, + `${id}: externalRecoveryCostUsdPerAttempt`, + ) + if (built.externalCostAccounting !== 'exact') { + throw new Error(`${id}: memory improvement requires exact external cost accounting`) + } return { ...built, id, @@ -20,6 +33,43 @@ export async function buildCandidate( } } +export async function buildRegisteredCandidate( + options: RunAgentMemoryImprovementOptions, + storage: CampaignStorage, + runDir: string, + config: TConfig, + hash: string, +): Promise { + const candidate = await buildCandidate(options, config, hash) + const path = join(runDir, 'memory-candidate-identities', `${hash}.json`) + const record = { surfaceHash: hash, candidateRef: candidate.ref } + const stored = storage.read(path) + if (stored !== undefined) { + let parsed: unknown + try { + parsed = JSON.parse(stored) + } catch (error) { + throw new Error(`invalid memory candidate identity '${path}'`, { cause: error }) + } + if (canonicalJson(parsed as JsonValue) !== canonicalJson(record)) { + throw new Error( + `memory config '${hash}' changed candidate identity within the same improvement run`, + ) + } + return candidate + } + if (storage.exists(path)) throw new Error(`cannot read memory candidate identity '${path}'`) + storage.ensureDir(dirname(path)) + storage.write(path, `${JSON.stringify(record, null, 2)}\n`) + return candidate +} + +function assertDeclaredCost(value: unknown, label: string): asserts value is number { + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { + throw new Error(`${label} must be a declared non-negative finite number`) + } +} + export function experimentOptions( options: RunAgentMemoryImprovementOptions, storage: CampaignStorage, diff --git a/src/memory/improvement/evaluation.ts b/src/memory/improvement/evaluation.ts index a3ae0a3..3803a46 100644 --- a/src/memory/improvement/evaluation.ts +++ b/src/memory/improvement/evaluation.ts @@ -3,17 +3,18 @@ import { canonicalJson } from '@tangle-network/agent-eval' import { type CampaignCostMeter, type CampaignStorage, - type JsonValue, surfaceHash, } from '@tangle-network/agent-eval/campaign' +import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' import { stableId } from '../../ids' import type { SerializedCandidateCodec } from '../../optimization' import { + type AgentMemoryExperimentCandidate, type AgentMemorySequence, type AgentMemorySequenceArtifact, runAgentMemoryExperiment, } from '../experiment' -import { buildCandidate, experimentOptions } from './candidate' +import { experimentOptions } from './candidate' import { memorySequenceFingerprint, parseMemoryConfig, serializeMemoryConfig } from './identity' import type { AgentMemoryFinalEvaluation, @@ -24,8 +25,8 @@ import type { } from './types' interface StoredMemoryArtifact { - schema: 1 surfaceHash: string + candidateRef: string sequenceFingerprint: string sequenceId: string rep: number @@ -55,7 +56,7 @@ export async function evaluateMemoryCandidate(input: storage: CampaignStorage runDir: string lease: OwnedRunLease - config: TConfig + candidate: AgentMemoryExperimentCandidate surfaceHash: string scenario: MemoryConfigScenario rep: number @@ -74,6 +75,7 @@ export async function evaluateMemoryCandidate(input: ) const stored = readStoredMemoryArtifact(input.storage, artifactPath, { surfaceHash: input.surfaceHash, + candidateRef: input.candidate.ref, scenario: input.scenario, rep: input.rep, seed: input.seed, @@ -89,7 +91,6 @@ export async function evaluateMemoryCandidate(input: return stored } - const candidate = await buildCandidate(input.options, input.config, input.surfaceHash) const evaluationCostLimit = input.options.maximumEvaluationCostUsd ?? 0 const evaluationId = stableId( 'memory_eval', @@ -102,7 +103,7 @@ export async function evaluateMemoryCandidate(input: ) const paid = await input.cost.runPaidCall({ actor: 'agent-knowledge:memory-config-evaluation', - model: candidate.ref, + model: input.candidate.ref, signal: input.signal, maximumCharge: { externallyEnforcedMaximumUsd: evaluationCostLimit, @@ -113,7 +114,7 @@ export async function evaluateMemoryCandidate(input: experimentId: `${input.options.experimentId}:${evaluationId}`, experimentRunId: evaluationId, sequences: [input.scenario.sequence], - candidates: [candidate], + candidates: [input.candidate], runDir: join(input.runDir, 'evaluations', evaluationId), seed: input.seed, reps: 1, @@ -122,7 +123,7 @@ export async function evaluateMemoryCandidate(input: costPhase: `memory.config.${input.surfaceHash}`, }) const cell = experiment.campaign.cells[0] - if (!cell || cell.error || cell.artifact.candidateId !== candidate.id) { + if (!cell || cell.error || cell.artifact.candidateId !== input.candidate.id) { throw new Error( `${input.surfaceHash}/${input.scenario.sequenceId}: memory config evaluation did not complete`, ) @@ -135,7 +136,7 @@ export async function evaluateMemoryCandidate(input: } }, receipt: (value) => ({ - model: candidate.ref, + model: input.candidate.ref, inputTokens: value.cost.inputTokens, outputTokens: value.cost.outputTokens, ...(value.cost.reasoningTokens !== undefined @@ -169,7 +170,9 @@ export function loadFinalEvaluation(input: { storage: CampaignStorage runDir: string baselineSurfaceHash: string + baselineCandidateRef: string winnerSurfaceHash: string + winnerCandidateRef: string }): AgentMemoryFinalEvaluation { const pairs: AgentMemoryFinalPair[] = [] const reps = input.options.reps ?? 1 @@ -179,12 +182,22 @@ export function loadFinalEvaluation(input: { const baseline = readStoredMemoryArtifact( input.storage, memoryFinalArtifactPath(input.runDir, input.baselineSurfaceHash, scenario, rep), - { surfaceHash: input.baselineSurfaceHash, scenario, rep }, + { + surfaceHash: input.baselineSurfaceHash, + candidateRef: input.baselineCandidateRef, + scenario, + rep, + }, ) const winner = readStoredMemoryArtifact( input.storage, memoryFinalArtifactPath(input.runDir, input.winnerSurfaceHash, scenario, rep), - { surfaceHash: input.winnerSurfaceHash, scenario, rep }, + { + surfaceHash: input.winnerSurfaceHash, + candidateRef: input.winnerCandidateRef, + scenario, + rep, + }, ) if (!baseline || !winner) { throw new Error( @@ -198,10 +211,14 @@ export function loadFinalEvaluation(input: { manifestHash: surfaceHash( canonicalJson({ baselineSurfaceHash: input.baselineSurfaceHash, + baselineCandidateRef: input.baselineCandidateRef, winnerSurfaceHash: input.winnerSurfaceHash, + winnerCandidateRef: input.winnerCandidateRef, pairs, }), ), + baselineCandidateRef: input.baselineCandidateRef, + winnerCandidateRef: input.winnerCandidateRef, pairs, } } @@ -251,6 +268,7 @@ function readStoredMemoryArtifact( path: string, expected: { surfaceHash: string + candidateRef: string scenario: MemoryConfigScenario rep: number seed?: number @@ -272,8 +290,17 @@ function readStoredMemoryArtifact( } const value = record as Partial if ( - value.schema !== 1 || + !hasExactKeys(record, [ + 'surfaceHash', + 'candidateRef', + 'sequenceFingerprint', + 'sequenceId', + 'rep', + 'seed', + 'artifact', + ]) || value.surfaceHash !== expected.surfaceHash || + value.candidateRef !== expected.candidateRef || value.sequenceFingerprint !== memorySequenceFingerprint(expected.scenario.sequence) || value.sequenceId !== expected.scenario.sequenceId || value.rep !== expected.rep || @@ -311,6 +338,7 @@ function writeStoredMemoryArtifact( function storedMemoryArtifactRecord( input: { surfaceHash: string + candidate: AgentMemoryExperimentCandidate scenario: MemoryConfigScenario rep: number seed: number @@ -318,8 +346,8 @@ function storedMemoryArtifactRecord( artifact: AgentMemorySequenceArtifact, ): StoredMemoryArtifact { return { - schema: 1, surfaceHash: input.surfaceHash, + candidateRef: input.candidate.ref, sequenceFingerprint: memorySequenceFingerprint(input.scenario.sequence), sequenceId: input.scenario.sequenceId, rep: input.rep, @@ -374,3 +402,8 @@ function isNonnegativeIntegerRecord(value: unknown): value is Record Number.isSafeInteger(entry) && entry >= 0) ) } + +function hasExactKeys(value: object, expected: readonly string[]): boolean { + const keys = Object.keys(value) + return keys.length === expected.length && keys.every((key) => expected.includes(key)) +} diff --git a/src/memory/improvement/identity.ts b/src/memory/improvement/identity.ts index 1968f58..cf3dcfd 100644 --- a/src/memory/improvement/identity.ts +++ b/src/memory/improvement/identity.ts @@ -1,16 +1,10 @@ import { join } from 'node:path' import { canonicalJson } from '@tangle-network/agent-eval' -import { - type CampaignStorage, - type JsonValue, - surfaceHash, -} from '@tangle-network/agent-eval/campaign' +import { type CampaignStorage, surfaceHash } from '@tangle-network/agent-eval/campaign' +import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' import type { AgentMemorySequence } from '../experiment' import { normalizedPromotionPolicy } from './promotion' -import { - MEMORY_IMPROVEMENT_IMPLEMENTATION_REF, - type RunAgentMemoryImprovementOptions, -} from './types' +import type { RunAgentMemoryImprovementOptions } from './types' export function assertMemoryImprovementIdentity( options: RunAgentMemoryImprovementOptions, @@ -20,10 +14,8 @@ export function assertMemoryImprovementIdentity( ): void { const path = join(runDir, 'memory-improvement-manifest.json') const identity = { - schema: 7, - implementationRef: MEMORY_IMPROVEMENT_IMPLEMENTATION_REF, experimentId: options.experimentId, - improvementRef: options.improvementRef, + implementationRef: options.implementationRef, method: options.method.name, activationRef: options.activation?.ref ?? null, baselineConfig: serialize(options.baselineConfig), @@ -32,8 +24,7 @@ export function assertMemoryImprovementIdentity( promotionPolicy: normalizedPromotionPolicy(options), seed: options.seed ?? 42, reps: options.reps ?? 1, - maxOptimizationCostUsd: options.maxOptimizationCostUsd ?? null, - maxFinalCostUsd: options.maxFinalCostUsd ?? null, + maxTotalCostUsd: options.maxTotalCostUsd ?? null, maximumEvaluationCostUsd: options.maximumEvaluationCostUsd ?? null, allowIncompleteCostAccounting: options.allowIncompleteCostAccounting ?? false, trainSequences: options.trainSequences, @@ -63,7 +54,7 @@ export function assertMemoryImprovementIdentity( canonicalJson((manifest as Record).identity) !== canonicalJson(identity) ) { throw new Error( - `memory improvement run '${runDir}' does not match its persisted inputs or improvementRef`, + `memory improvement run '${runDir}' does not match its persisted inputs or implementation`, ) } } diff --git a/src/memory/improvement/output.ts b/src/memory/improvement/output.ts index cef4403..67dc92f 100644 --- a/src/memory/improvement/output.ts +++ b/src/memory/improvement/output.ts @@ -1,4 +1,5 @@ -import type { CampaignStorage, JsonValue } from '@tangle-network/agent-eval/campaign' +import type { CampaignStorage } from '@tangle-network/agent-eval/campaign' +import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' import type { RunAgentMemoryImprovementResult } from './types' export function writeMemoryImprovementResult( diff --git a/src/memory/improvement/promotion.ts b/src/memory/improvement/promotion.ts index a710a27..18ad04f 100644 --- a/src/memory/improvement/promotion.ts +++ b/src/memory/improvement/promotion.ts @@ -1,5 +1,6 @@ -import type { JsonValue, PairedHoldout } from '@tangle-network/agent-eval/campaign' +import type { PairedHoldout } from '@tangle-network/agent-eval/campaign' import { heldoutSignificance } from '@tangle-network/agent-eval/campaign' +import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' import type { RunSerializedKnowledgeOptimizationResult } from '../../optimization' import type { AgentMemorySequence, AgentMemorySequenceProbe } from '../experiment' import { @@ -64,22 +65,20 @@ export function decidePromotion(input: { }, ) const reasons: string[] = [] - const optimizationCost = input.optimization.comparison.optimizationCost - const finalCost = input.optimization.comparison.testCost + const totalCost = input.optimization.comparison.totalCost if ( !input.optimization.comparison.totalCost.accountingComplete && !options.allowIncompleteCostAccounting ) { reasons.push('optimization or final cost accounting is incomplete') } - if (optimizationCost.totalCostUsd > (options.maxOptimizationCostUsd ?? 0)) { - reasons.push( - `optimization cost ${optimizationCost.totalCostUsd} exceeds the configured limit ${options.maxOptimizationCostUsd ?? 0}`, - ) + const optimizerSource = input.optimization.comparison.best.provenance?.source + if (optimizerSource && optimizerSource.evidence !== 'observed') { + reasons.push('external optimizer package identity was not observed') } - if (finalCost.totalCostUsd > (options.maxFinalCostUsd ?? 0)) { + if (totalCost.totalCostUsd > (options.maxTotalCostUsd ?? 0)) { reasons.push( - `final comparison cost ${finalCost.totalCostUsd} exceeds the configured limit ${options.maxFinalCostUsd ?? 0}`, + `total cost ${totalCost.totalCostUsd} exceeds the configured limit ${options.maxTotalCostUsd ?? 0}`, ) } if (!significance.significant) { @@ -129,8 +128,7 @@ export function normalizedPromotionPolicy( criticalDimensions: [...(options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS)], criticalDimensionTolerance: options.criticalDimensionTolerance ?? 0.05, minFinalScore: options.minFinalScore ?? 0, - maxOptimizationCostUsd: options.maxOptimizationCostUsd ?? 0, - maxFinalCostUsd: options.maxFinalCostUsd ?? 0, + maxTotalCostUsd: options.maxTotalCostUsd ?? 0, allowIncompleteCostAccounting: options.allowIncompleteCostAccounting ?? false, } } diff --git a/src/memory/improvement/run.ts b/src/memory/improvement/run.ts index 651f4fb..ff25e24 100644 --- a/src/memory/improvement/run.ts +++ b/src/memory/improvement/run.ts @@ -3,14 +3,23 @@ import { canonicalJson } from '@tangle-network/agent-eval' import { type CampaignStorage, fsCampaignStorage, - type JsonValue, resolveRunDir, surfaceHash, } from '@tangle-network/agent-eval/campaign' +import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' import { runSerializedKnowledgeOptimization } from '../../optimization' -import { type AgentMemorySequenceArtifact, agentMemorySequenceJudge } from '../experiment' +import { + type AgentMemoryExperimentCandidate, + type AgentMemorySequenceArtifact, + agentMemorySequenceJudge, +} from '../experiment' import { acquireAgentMemoryRunLease } from '../run-control' -import { activateMemoryWinner, readMemoryActivationJournal } from './activation' +import { + activateMemoryWinner, + assertActivatedMemoryWinner, + readMemoryActivationJournal, +} from './activation' +import { buildRegisteredCandidate } from './candidate' import { evaluateMemoryCandidate, loadFinalEvaluation, @@ -96,6 +105,16 @@ async function runAgentMemoryImprovementOwned( const finalScenarios = memoryConfigScenarios(options.finalSequences) const finalScenarioIds = new Set(finalScenarios.map((scenario) => scenario.id)) const pending = new Map>() + const candidates = new Map>() + const candidateFor = (config: TConfig, hash: string) => { + let candidate = candidates.get(hash) + if (!candidate) { + candidate = buildRegisteredCandidate(options, storage, runDir, config, hash) + candidates.set(hash, candidate) + void candidate.catch(() => candidates.delete(hash)) + } + return candidate + } const optimizationRunOptions = { ...(options.optimizationRunOptions ?? {}), storage, @@ -107,12 +126,10 @@ async function runAgentMemoryImprovementOwned( ...(options.dispatchTimeoutMs !== undefined ? { dispatchTimeoutMs: options.dispatchTimeoutMs } : {}), - ...(options.maxOptimizationCostUsd !== undefined - ? { costCeiling: options.maxOptimizationCostUsd } - : { costCeiling: 0 }), expectUsage: 'off' as const, } const optimization = await runSerializedKnowledgeOptimization({ + executionRef: options.implementationRef, baseline: options.baselineConfig, method: options.method, trainScenarios, @@ -120,7 +137,7 @@ async function runAgentMemoryImprovementOwned( finalScenarios, codec, scenarioFingerprint: (scenario) => memorySequenceFingerprint(scenario.sequence), - dispatchCandidate: ({ candidate, candidateSurfaceHash, scenario, context }) => { + dispatchCandidate: async ({ candidate, candidateSurfaceHash, scenario, context }) => { const key = memoryArtifactPath( runDir, candidateSurfaceHash, @@ -130,20 +147,22 @@ async function runAgentMemoryImprovementOwned( ) let operation = pending.get(key) if (!operation) { - operation = evaluateMemoryCandidate({ - options, - storage, - runDir, - lease, - config: candidate, - surfaceHash: candidateSurfaceHash, - scenario, - rep: context.rep, - seed: context.seed, - final: finalScenarioIds.has(scenario.id), - cost: context.cost, - signal: context.signal, - }) + operation = candidateFor(candidate, candidateSurfaceHash).then((builtCandidate) => + evaluateMemoryCandidate({ + options, + storage, + runDir, + lease, + candidate: builtCandidate, + surfaceHash: candidateSurfaceHash, + scenario, + rep: context.rep, + seed: context.seed, + final: finalScenarioIds.has(scenario.id), + cost: context.cost, + signal: context.signal, + }), + ) pending.set(key, operation) void operation.catch(() => pending.delete(key)) } @@ -156,7 +175,7 @@ async function runAgentMemoryImprovementOwned( seed: options.seed, reps: options.reps, resumable: options.resumable, - costCeiling: options.maxFinalCostUsd ?? 0, + costCeiling: options.maxTotalCostUsd ?? 0, maxConcurrency: options.sequenceConcurrency, dispatchTimeoutMs: options.dispatchTimeoutMs, expectUsage: 'off', @@ -164,13 +183,19 @@ async function runAgentMemoryImprovementOwned( now: options.now, }) await lease.assertOwned() + const [baselineCandidate, winnerCandidate] = await Promise.all([ + candidateFor(optimization.baseline.value, optimization.baseline.surfaceHash), + candidateFor(optimization.winner.value, optimization.winner.surfaceHash), + ]) const finalEvaluation = loadFinalEvaluation({ options, storage, runDir, baselineSurfaceHash: optimization.baseline.surfaceHash, + baselineCandidateRef: baselineCandidate.ref, winnerSurfaceHash: optimization.winner.surfaceHash, + winnerCandidateRef: winnerCandidate.ref, }) const unchanged = optimization.baseline.surfaceHash === optimization.winner.surfaceHash const decision = decidePromotion({ options, optimization, finalEvaluation, unchanged }) @@ -179,7 +204,7 @@ async function runAgentMemoryImprovementOwned( const activationId = `memory-activation-${surfaceHash( canonicalJson({ experimentId: options.experimentId, - improvementRef: options.improvementRef, + implementationRef: options.implementationRef, method: optimization.methodName, activationRef, baselineSurfaceHash: optimization.baseline.surfaceHash, @@ -192,7 +217,6 @@ async function runAgentMemoryImprovementOwned( const activationJournalPath = join(activationJournalDir, `${activationId}.jsonl`) const activationEligible = decision.status === 'promote' const activationEventIdentity = { - schema: 2 as const, activationId, experimentId: options.experimentId, activationRef, @@ -229,6 +253,9 @@ async function runAgentMemoryImprovementOwned( resultJsonPath, } satisfies RunAgentMemoryImprovementResult await lease.assertOwned() + if (activationJournal.activated) { + await assertActivatedMemoryWinner({ options, lease, result }) + } writeMemoryImprovementResult(storage, options.experimentId, result) if (activationEligible && !activationJournal.activated && options.activation) { diff --git a/src/memory/improvement/types.ts b/src/memory/improvement/types.ts index 41eaecd..dfd5cf0 100644 --- a/src/memory/improvement/types.ts +++ b/src/memory/improvement/types.ts @@ -2,11 +2,11 @@ import type { CampaignStorage, HeldoutSignificance, HeldoutSignificanceOptions, - JsonValue, OptimizationMethod, OptimizationMethodRunOptions, Scenario, } from '@tangle-network/agent-eval/campaign' +import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' import type { RunSerializedKnowledgeOptimizationResult } from '../../optimization' import type { AgentMemoryExperimentCandidate, @@ -52,6 +52,8 @@ export interface AgentMemoryFinalPair { export interface AgentMemoryFinalEvaluation { manifestHash: string + baselineCandidateRef: string + winnerCandidateRef: string pairs: readonly AgentMemoryFinalPair[] } @@ -86,7 +88,6 @@ export interface AgentMemoryActivationDriver { } export interface AgentMemoryActivationEvent { - schema: 2 status: 'prepared' | 'activated' activationId: string experimentId: string @@ -105,6 +106,22 @@ export interface AgentMemoryActivationJournalState { export type AgentMemoryImprovementRunLease = AgentMemoryRunLease +export type AgentMemoryImprovementCandidate = Omit< + AgentMemoryExperimentCandidate, + | 'id' + | 'externalCostUsdPerSequence' + | 'externalRecoveryCostUsdPerAttempt' + | 'externalCostAccounting' +> & + Required< + Pick< + AgentMemoryExperimentCandidate, + 'externalCostUsdPerSequence' | 'externalRecoveryCostUsdPerAttempt' + > + > & { + externalCostAccounting: 'exact' + } + export interface RunAgentMemoryImprovementOptions { experimentId: string baselineConfig: TConfig @@ -116,11 +133,12 @@ export interface RunAgentMemoryImprovementOptions { config: TConfig candidateId: string surfaceHash: string - }): - | Omit - | Promise> - /** Stable version or commit for method config, candidate construction, and execution behavior. */ - improvementRef: string + }): AgentMemoryImprovementCandidate | Promise + /** + * Immutable digest covering the installed implementation, method config, + * candidate construction, execution behavior, and external settings. + */ + implementationRef: string runDir: string repo?: string storage?: CampaignStorage @@ -136,10 +154,8 @@ export interface RunAgentMemoryImprovementOptions { cleanupTimeoutMs?: number maxRecoveryAttempts?: number maxRecoveryRetriesPerAttempt?: number - /** Method search spend limit. */ - maxOptimizationCostUsd?: number - /** Final comparison spend limit. */ - maxFinalCostUsd?: number + /** Total spend limit across method search and final comparison. Default 0. */ + maxTotalCostUsd?: number /** Enforced maximum for one config and one sequence. Required with either spend limit. */ maximumEvaluationCostUsd?: number /** Allow activation when a method cannot fully account for cost. Default false. */ @@ -190,6 +206,4 @@ export const DEFAULT_CRITICAL_DIMENSIONS = [ 'memory_event_recall', ] as const -export const MEMORY_IMPROVEMENT_IMPLEMENTATION_REF = 'agent-knowledge:memory-improvement:v3' - export type OwnedRunLease = OwnedAgentMemoryRunLease diff --git a/src/memory/improvement/validation.ts b/src/memory/improvement/validation.ts index 45d77ae..47af544 100644 --- a/src/memory/improvement/validation.ts +++ b/src/memory/improvement/validation.ts @@ -1,4 +1,5 @@ -import type { JsonValue } from '@tangle-network/agent-eval/campaign' +import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' +import { assertImmutableRef } from '../../immutable-ref' import type { AgentMemorySequence } from '../experiment' import { memorySequenceFingerprint } from './identity' import { DEFAULT_CRITICAL_DIMENSIONS, type RunAgentMemoryImprovementOptions } from './types' @@ -9,12 +10,12 @@ export function assertMemoryImprovementOptions( for (const [name, value] of [ ['experimentId', options.experimentId], ['runDir', options.runDir], - ['improvementRef', options.improvementRef], ] as const) { if (typeof value !== 'string' || !value.trim()) { throw new Error(`memory improvement ${name} must be a non-empty string`) } } + assertImmutableRef(options.implementationRef, 'memory improvement implementationRef') if ( !options.method || typeof options.method.name !== 'string' || @@ -27,9 +28,7 @@ export function assertMemoryImprovementOptions( throw new Error('memory improvement createCandidate must be a function') } if (options.activation !== undefined) { - if (typeof options.activation.ref !== 'string' || !options.activation.ref.trim()) { - throw new Error('memory improvement activation.ref must be a non-empty string') - } + assertImmutableRef(options.activation.ref, 'memory improvement activation.ref') if (typeof options.activation.readCurrent !== 'function') { throw new Error('memory improvement activation.readCurrent must be a function') } @@ -37,6 +36,9 @@ export function assertMemoryImprovementOptions( throw new Error('memory improvement activation.compareAndSet must be a function') } } + if (options.executeStep) { + assertImmutableRef(options.executeStepRef, 'memory improvement executeStepRef') + } if (options.serializeConfig !== undefined && typeof options.serializeConfig !== 'function') { throw new Error('memory improvement serializeConfig must be a function') } @@ -53,10 +55,7 @@ export function assertMemoryImprovementOptions( throw new Error(`memory improvement ${name} must be a positive safe integer`) } } - for (const [name, value] of [ - ['maxOptimizationCostUsd', options.maxOptimizationCostUsd], - ['maxFinalCostUsd', options.maxFinalCostUsd], - ] as const) { + for (const [name, value] of [['maxTotalCostUsd', options.maxTotalCostUsd]] as const) { if (value !== undefined && (!Number.isFinite(value) || value < 0)) { throw new Error(`memory improvement ${name} must be a non-negative finite number`) } @@ -67,10 +66,7 @@ export function assertMemoryImprovementOptions( ) { throw new Error('memory improvement maximumEvaluationCostUsd must be a positive finite number') } - if ( - ((options.maxOptimizationCostUsd ?? 0) > 0 || (options.maxFinalCostUsd ?? 0) > 0) && - options.maximumEvaluationCostUsd === undefined - ) { + if ((options.maxTotalCostUsd ?? 0) > 0 && options.maximumEvaluationCostUsd === undefined) { throw new Error( 'memory improvement maximumEvaluationCostUsd is required when a spend limit is configured', ) diff --git a/src/memory/mem0.ts b/src/memory/mem0.ts index d390fa6..a7f14ed 100644 --- a/src/memory/mem0.ts +++ b/src/memory/mem0.ts @@ -1,6 +1,6 @@ import { randomUUID } from 'node:crypto' import { canonicalJson } from '@tangle-network/agent-eval' -import { stableId } from '../ids' +import { sha256, stableId } from '../ids' import { defaultGetMemoryContext } from './adapter' import { memoryWriteResultToSourceRecord } from './source-record' import type { @@ -674,5 +674,5 @@ export function mem0MemoryAdapterIdentity( if (typeof options.backendRef !== 'string' || !options.backendRef.trim()) { throw new Error('Mem0 backendRef must be a non-empty string') } - return stableId('mem0', canonicalJson(compactRecord(options))) + return `sha256:${sha256(canonicalJson(compactRecord(options)))}` } diff --git a/src/optimization.ts b/src/optimization.ts index 614b0f2..1a6fdc7 100644 --- a/src/optimization.ts +++ b/src/optimization.ts @@ -3,13 +3,15 @@ import { type CompareOptimizationMethodsOptions, compareOptimizationMethods, type DispatchContext, - type JsonValue, type MutableSurface, type OptimizationMethod, type OptimizationMethodComparison, + type OptimizationMethodRunOptions, type Scenario, surfaceHash, } from '@tangle-network/agent-eval/campaign' +import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' +import { assertImmutableRef } from './immutable-ref' export interface SerializedCandidateCodec { serialize(candidate: TCandidate): string @@ -30,6 +32,8 @@ type ComparisonOptions = Omit< | 'selectionScenarios' | 'testScenarios' | 'dispatchWithSurface' + | 'dispatchRef' + | 'optimizationRunOptions' > export interface RunSerializedKnowledgeOptimizationOptions< @@ -37,6 +41,8 @@ export interface RunSerializedKnowledgeOptimizationOptions< TScenario extends Scenario, TArtifact, > extends ComparisonOptions { + /** Immutable identity for candidate execution, judges, models, indexes, and external settings. */ + executionRef: string baseline: TCandidate method: OptimizationMethod trainScenarios: readonly TScenario[] @@ -52,6 +58,7 @@ export interface RunSerializedKnowledgeOptimizationOptions< codec?: SerializedCandidateCodec /** Detects duplicated cases whose IDs differ within or across data partitions. */ scenarioFingerprint?: (scenario: TScenario) => string + optimizationRunOptions?: Omit, 'dispatchRef'> } export interface RunSerializedKnowledgeOptimizationResult { @@ -73,8 +80,15 @@ export async function runSerializedKnowledgeOptimization< options: RunSerializedKnowledgeOptimizationOptions, ): Promise> { assertCompleteOptimizationMethod(options.method) + assertImmutableRef(options.executionRef, 'knowledge optimization executionRef') const codec = options.codec ?? jsonCandidateCodec() const baseline = normalizeCandidate(options.baseline, codec, 'baseline') + const dispatchRef = [ + 'knowledge-optimization', + options.executionRef, + baseline.surfaceHash, + surfaceHash(options.method.name), + ].join(':') assertPartitionContent( options.trainScenarios, options.selectionScenarios, @@ -90,14 +104,12 @@ export async function runSerializedKnowledgeOptimization< selectionScenarios, finalScenarios, dispatchCandidate, + executionRef: _executionRef, codec: _codec, scenarioFingerprint: _scenarioFingerprint, ...comparisonOptions } = options const optimizationRunOptions = { - ...(comparisonOptions.dispatchRef !== undefined - ? { dispatchRef: comparisonOptions.dispatchRef } - : {}), ...(comparisonOptions.reps !== undefined ? { reps: comparisonOptions.reps } : {}), ...(comparisonOptions.resumable !== undefined ? { resumable: comparisonOptions.resumable } @@ -131,9 +143,11 @@ export async function runSerializedKnowledgeOptimization< ? { cellPlacement: comparisonOptions.cellPlacement } : {}), ...(comparisonOptions.optimizationRunOptions ?? {}), + dispatchRef, } const comparison = await compareOptimizationMethods({ ...comparisonOptions, + dispatchRef, optimizationRunOptions, methods: [method], baselineSurface: baseline.surface, diff --git a/src/rag-answer-evidence.ts b/src/rag-answer-evidence.ts new file mode 100644 index 0000000..38298d0 --- /dev/null +++ b/src/rag-answer-evidence.ts @@ -0,0 +1,79 @@ +import type { ComparisonCost } from '@tangle-network/agent-eval/campaign' +import { assertImmutableRef } from './immutable-ref' + +interface RagAnswerEvidence { + passed: boolean + metrics: Record + finalScenarioIds: readonly string[] + datasetRef: string + evaluatorRef: string + cost: ComparisonCost +} + +export function assertRagAnswerEvidence(result: RagAnswerEvidence): void { + if (!result || typeof result !== 'object') { + throw new Error('answer-quality evidence must be an object') + } + if (typeof result.passed !== 'boolean') { + throw new Error('answer-quality evidence requires a boolean passed verdict') + } + assertImmutableRef(result.datasetRef, 'answer-quality datasetRef') + assertImmutableRef(result.evaluatorRef, 'answer-quality evaluatorRef') + if ( + !Array.isArray(result.finalScenarioIds) || + result.finalScenarioIds.length < 2 || + new Set(result.finalScenarioIds).size !== result.finalScenarioIds.length || + result.finalScenarioIds.some((id) => typeof id !== 'string' || id.trim().length === 0) + ) { + throw new Error('answer-quality evidence requires at least 2 unique final scenario IDs') + } + if (!result.metrics || typeof result.metrics !== 'object' || Array.isArray(result.metrics)) { + throw new Error('answer-quality evidence requires non-empty finite metrics') + } + const metrics = Object.entries(result.metrics) + if ( + metrics.length === 0 || + metrics.some(([key, value]) => key.trim().length === 0 || !Number.isFinite(value)) + ) { + throw new Error('answer-quality evidence requires non-empty finite metrics') + } + if (!result.cost || typeof result.cost !== 'object') { + throw new Error('answer-quality evidence requires cost accounting') + } + if ( + !Number.isFinite(result.cost.totalCostUsd) || + result.cost.totalCostUsd < 0 || + typeof result.cost.accountingComplete !== 'boolean' || + !Array.isArray(result.cost.incompleteReasons) || + result.cost.incompleteReasons.some((reason) => typeof reason !== 'string' || !reason.trim()) + ) { + throw new Error('answer-quality evidence has invalid cost accounting') + } + if (result.cost.accountingComplete && result.cost.incompleteReasons.length > 0) { + throw new Error('complete answer-quality cost accounting cannot include incomplete reasons') + } +} + +export function ragAnswerEvidenceRejectionReasons( + evidence: RagAnswerEvidence, + costCeiling: number | undefined, +): string[] { + const reasons: string[] = [] + if (!evidence.passed) reasons.push('answer-quality evaluation failed') + if (!evidence.cost.accountingComplete) { + reasons.push('answer-quality final comparison has incomplete cost accounting') + } + if (costCeiling === undefined) { + reasons.push('answer-quality promotion requires answerQualityCostCeiling') + } else if (exceedsCostCeiling(evidence.cost.totalCostUsd, costCeiling)) { + reasons.push( + `answer-quality final comparison cost ${evidence.cost.totalCostUsd} exceeds ${costCeiling}`, + ) + } + return reasons +} + +function exceedsCostCeiling(totalCostUsd: number, costCeiling: number): boolean { + const tolerance = Number.EPSILON * Math.max(1, Math.abs(totalCostUsd), Math.abs(costCeiling)) * 8 + return totalCostUsd - costCeiling > tolerance +} diff --git a/src/rag-eval/calibration.ts b/src/rag-eval/calibration.ts index 9d58d01..e01a25d 100644 --- a/src/rag-eval/calibration.ts +++ b/src/rag-eval/calibration.ts @@ -1,3 +1,5 @@ +import { contentHash } from '@tangle-network/agent-eval' +import { assertImmutableRef } from '../immutable-ref' import type { RagAnswerQualityResult, RagGapFinding } from '../rag-improvement-loop' import type { RagAnswerMetricSummary, @@ -10,6 +12,16 @@ import { aggregateRagAnswerMetrics, ragAnswerQualityJudge, scoreRagAnswerArtifac export function createRagAnswerQualityHook( options: RagAnswerQualityHookOptions, ): () => Promise { + assertImmutableRef(options.evaluatorRef, 'RAG answer evaluatorRef') + const finalScenarioIds = options.scenarios.map((scenario) => scenario.id) + if ( + finalScenarioIds.length < 2 || + new Set(finalScenarioIds).size !== finalScenarioIds.length || + finalScenarioIds.some((id) => !id.trim()) + ) { + throw new Error('RAG answer quality requires at least 2 unique final scenarios') + } + const datasetRef = `sha256:${contentHash(options.scenarios)}` return async () => { const summaries: RagAnswerMetricSummary[] = [] const findings: RagGapFinding[] = [] @@ -33,9 +45,15 @@ export function createRagAnswerQualityHook( findings.push(...summary.findings) } const metrics = aggregateRagAnswerMetrics(summaries) + const cost = + typeof options.cost === 'function' ? await options.cost() : structuredClone(options.cost) return { passed: findings.length === 0, metrics, + finalScenarioIds, + datasetRef, + evaluatorRef: options.evaluatorRef, + cost, findings, metadata: { scenarioCount: options.scenarios.length }, } diff --git a/src/rag-eval/contracts.ts b/src/rag-eval/contracts.ts index 612d8af..2e6922b 100644 --- a/src/rag-eval/contracts.ts +++ b/src/rag-eval/contracts.ts @@ -1,4 +1,5 @@ -import type { JsonValue, JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign' +import type { ComparisonCost, JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign' +import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' import type { RagGapFinding } from '../rag-improvement-loop' export type RagEvalProvider = @@ -131,6 +132,10 @@ export interface RagAnswerEvalCase { export interface RagAnswerQualityHookOptions { scenarios: readonly RagAnswerEvalScenario[] + /** Immutable identity of generation, scoring, models, and external evaluator behavior. */ + evaluatorRef: string + /** Return observed spend after all generation and evaluation calls finish. */ + cost: ComparisonCost | (() => MaybePromise) run: (scenario: RagAnswerEvalScenario) => MaybePromise externalEvaluator?: ( item: RagAnswerEvalCase, diff --git a/src/rag-improvement-loop.ts b/src/rag-improvement-loop.ts index bcedede..8852d41 100644 --- a/src/rag-improvement-loop.ts +++ b/src/rag-improvement-loop.ts @@ -1,4 +1,6 @@ -import type { JsonValue } from '@tangle-network/agent-eval/campaign' +import type { ComparisonCost } from '@tangle-network/agent-eval/campaign' +import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' +import { assertRagAnswerEvidence, ragAnswerEvidenceRejectionReasons } from './rag-answer-evidence' import { type RunRagOptimizationOptions, type RunRagOptimizationResult, @@ -59,24 +61,35 @@ export interface RagKnowledgeImprovementPhaseResult { metadata?: Record } +export type RagOptimizationSelection = Pick< + RunRagOptimizationResult, + 'methodName' | 'baseline' | 'winner' | 'baselineConfig' | 'winnerConfig' +> + +export type RetrievalOptimizationSelection = Pick< + RunRetrievalImprovementLoopResult, + 'methodName' | 'baseline' | 'winner' | 'baselineConfig' | 'winnerConfig' +> + export interface RagPhaseInputBase { goal: string phases: readonly RagKnowledgeImprovementPhaseResult[] - optimization?: RunRagOptimizationResult + /** Selected candidate only. Adaptive update callbacks run before final scoring starts. */ + optimization?: RagOptimizationSelection signal?: AbortSignal } export interface RagDiagnosisInput extends RagPhaseInputBase { - retrieval?: RunRetrievalImprovementLoopResult + retrieval?: RetrievalOptimizationSelection } export interface RagKnowledgeAcquisitionInput extends RagPhaseInputBase { - retrieval?: RunRetrievalImprovementLoopResult + retrieval?: RetrievalOptimizationSelection findings: readonly RagGapFinding[] } export interface RagKnowledgeUpdateInput extends RagPhaseInputBase { - retrieval?: RunRetrievalImprovementLoopResult + retrieval?: RetrievalOptimizationSelection findings: readonly RagGapFinding[] acquisition?: KnowledgeResearchLoopDecision } @@ -89,7 +102,7 @@ export interface RagKnowledgeUpdateResult { } export interface RagAnswerQualityInput extends RagPhaseInputBase { - retrieval?: RunRetrievalImprovementLoopResult + retrieval?: RetrievalOptimizationSelection findings: readonly RagGapFinding[] acquisition?: KnowledgeResearchLoopDecision knowledgeUpdate?: RagKnowledgeUpdateResult @@ -98,12 +111,20 @@ export interface RagAnswerQualityInput extends RagPhaseInputBase { export interface RagAnswerQualityResult { passed: boolean metrics: Record + finalScenarioIds: readonly string[] + datasetRef: string + evaluatorRef: string + cost: ComparisonCost findings?: readonly RagGapFinding[] metadata?: Record } export interface RagPromotionInput extends RagPhaseInputBase { - retrieval?: RunRetrievalImprovementLoopResult + retrieval?: RetrievalOptimizationSelection + /** Full final-case result available only to the terminal promotion decision. */ + optimizationComparison?: RunRagOptimizationResult['comparison'] + /** Full final-case result available only to the terminal promotion decision. */ + retrievalComparison?: RunRetrievalImprovementLoopResult['comparison'] findings: readonly RagGapFinding[] acquisition?: KnowledgeResearchLoopDecision knowledgeUpdate?: RagKnowledgeUpdateResult @@ -133,7 +154,13 @@ export interface RunRagKnowledgeImprovementLoopOptions { knowledgeResearch?: RagKnowledgeResearchOptions updateKnowledge?: (input: RagKnowledgeUpdateInput) => MaybePromise evaluateAnswers?: (input: RagAnswerQualityInput) => MaybePromise - promote?: (input: RagPromotionInput) => MaybePromise + /** Maximum total answer-evaluation spend accepted for promotion. */ + answerQualityCostCeiling?: number + /** + * Makes a side-effect-free promotion decision after the library has rejected + * missing, regressing, unaccounted, or over-budget final evidence. + */ + decidePromotion?: (input: RagPromotionInput) => MaybePromise enabledPhases?: readonly RagKnowledgeImprovementPhase[] requiredPhases?: readonly RagKnowledgeImprovementPhase[] signal?: AbortSignal @@ -158,6 +185,7 @@ export async function runRagKnowledgeImprovementLoop( options: RunRagKnowledgeImprovementLoopOptions, ): Promise { assertConfiguredRequiredPhases(options) + assertOptionalCostCeiling(options.answerQualityCostCeiling, 'answerQualityCostCeiling') const now = options.now ?? (() => new Date()) const phases: RagKnowledgeImprovementPhaseResult[] = [] let optimization: RunRagOptimizationResult | undefined @@ -168,45 +196,6 @@ export async function runRagKnowledgeImprovementLoop( let answerQuality: RagAnswerQualityResult | undefined let promotion: RagPromotionResult | undefined - if ( - phaseEnabled(options, 'rag-optimization') && - (options.optimization || - options.enabledPhases?.includes('rag-optimization') || - options.requiredPhases?.includes('rag-optimization')) - ) { - if (options.optimization) { - optimization = await runPhase( - phases, - now, - 'rag-optimization', - async () => { - assertNotAborted(options.signal) - return runRagOptimization(options.optimization!) - }, - summarizeRagOptimization, - ) - } else { - skipPhase(phases, now, 'rag-optimization', 'no full RAG optimization options provided') - } - } - - if (phaseEnabled(options, 'retrieval-tuning')) { - if (options.retrieval) { - retrieval = await runPhase( - phases, - now, - 'retrieval-tuning', - async () => { - assertNotAborted(options.signal) - return runRetrievalImprovementLoop(options.retrieval!) - }, - summarizeRetrievalResult, - ) - } else { - skipPhase(phases, now, 'retrieval-tuning', 'no retrieval options provided') - } - } - if (phaseEnabled(options, 'gap-diagnosis')) { if (options.diagnose) { findings = [ @@ -219,9 +208,9 @@ export async function runRagKnowledgeImprovementLoop( return options.diagnose!({ goal: options.goal, phases, - optimization, + optimization: selectRagOptimization(optimization), signal: options.signal, - retrieval, + retrieval: selectRetrievalOptimization(retrieval), }) }, (diagnosed) => `${diagnosed.length} finding(s)`, @@ -243,9 +232,9 @@ export async function runRagKnowledgeImprovementLoop( return options.acquireKnowledge!({ goal: options.goal, phases, - optimization, + optimization: selectRagOptimization(optimization), signal: options.signal, - retrieval, + retrieval: selectRetrievalOptimization(retrieval), findings, }) }, @@ -267,9 +256,9 @@ export async function runRagKnowledgeImprovementLoop( return options.updateKnowledge!({ goal: options.goal, phases, - optimization, + optimization: selectRagOptimization(optimization), signal: options.signal, - retrieval, + retrieval: selectRetrievalOptimization(retrieval), findings, acquisition, }) @@ -292,6 +281,45 @@ export async function runRagKnowledgeImprovementLoop( } } + if ( + phaseEnabled(options, 'rag-optimization') && + (options.optimization || + options.enabledPhases?.includes('rag-optimization') || + options.requiredPhases?.includes('rag-optimization')) + ) { + if (options.optimization) { + optimization = await runPhase( + phases, + now, + 'rag-optimization', + async () => { + assertNotAborted(options.signal) + return runRagOptimization(options.optimization!) + }, + summarizeRagOptimization, + ) + } else { + skipPhase(phases, now, 'rag-optimization', 'no full RAG optimization options provided') + } + } + + if (phaseEnabled(options, 'retrieval-tuning')) { + if (options.retrieval) { + retrieval = await runPhase( + phases, + now, + 'retrieval-tuning', + async () => { + assertNotAborted(options.signal) + return runRetrievalImprovementLoop(options.retrieval!) + }, + summarizeRetrievalResult, + ) + } else { + skipPhase(phases, now, 'retrieval-tuning', 'no retrieval options provided') + } + } + if (phaseEnabled(options, 'answer-quality')) { if (options.evaluateAnswers) { answerQuality = await runPhase( @@ -300,16 +328,18 @@ export async function runRagKnowledgeImprovementLoop( 'answer-quality', async () => { assertNotAborted(options.signal) - return options.evaluateAnswers!({ + const result = await options.evaluateAnswers!({ goal: options.goal, phases, - optimization, + optimization: selectRagOptimization(optimization), signal: options.signal, - retrieval, + retrieval: selectRetrievalOptimization(retrieval), findings, acquisition, knowledgeUpdate, }) + assertRagAnswerEvidence(result) + return result }, summarizeAnswerQuality, ) @@ -320,19 +350,30 @@ export async function runRagKnowledgeImprovementLoop( } if (phaseEnabled(options, 'promotion')) { - if (options.promote) { + if (options.decidePromotion) { promotion = await runPhase( phases, now, 'promotion', async () => { assertNotAborted(options.signal) - return options.promote!({ + const evidenceRejection = rejectUnsafePromotionEvidence({ + optimization: optimization?.comparison, + optimizationCostCeiling: options.optimization?.costCeiling, + retrieval: retrieval?.comparison, + retrievalCostCeiling: options.retrieval?.costCeiling, + answerQuality, + answerQualityCostCeiling: options.answerQualityCostCeiling, + }) + if (evidenceRejection) return evidenceRejection + return options.decidePromotion!({ goal: options.goal, phases, - optimization, + optimization: selectRagOptimization(optimization), + optimizationComparison: optimization?.comparison, signal: options.signal, - retrieval, + retrieval: selectRetrievalOptimization(retrieval), + retrievalComparison: retrieval?.comparison, findings, acquisition, knowledgeUpdate, @@ -342,7 +383,7 @@ export async function runRagKnowledgeImprovementLoop( (result) => `${result.promoted ? 'promoted' : 'held'}: ${result.reason}`, ) } else { - skipPhase(phases, now, 'promotion', 'no promotion hook provided') + skipPhase(phases, now, 'promotion', 'no promotion decision hook provided') } } @@ -359,8 +400,70 @@ export async function runRagKnowledgeImprovementLoop( } } +function rejectUnsafePromotionEvidence(evidence: { + optimization?: RunRagOptimizationResult['comparison'] + optimizationCostCeiling?: number + retrieval?: RunRetrievalImprovementLoopResult['comparison'] + retrievalCostCeiling?: number + answerQuality?: RagAnswerQualityResult + answerQualityCostCeiling?: number +}): RagPromotionResult | undefined { + const reasons: string[] = [] + if (!evidence.optimization && !evidence.retrieval && !evidence.answerQuality) { + reasons.push('promotion requires final RAG, retrieval, or answer-quality evidence') + } + for (const [label, comparison, costCeiling] of [ + ['RAG', evidence.optimization, evidence.optimizationCostCeiling], + ['retrieval', evidence.retrieval, evidence.retrievalCostCeiling], + ] as const) { + if (!comparison) continue + if (!comparison.totalCost.accountingComplete) { + reasons.push(`${label} final comparison has incomplete cost accounting`) + } + const optimizerSource = comparison.best.provenance?.source + if (optimizerSource && optimizerSource.evidence !== 'observed') { + reasons.push(`${label} optimizer package identity was not observed`) + } + if (comparison.best.liftCi.low < 0) { + reasons.push(`${label} final comparison does not rule out a regression`) + } + if ( + costCeiling !== undefined && + exceedsCostCeiling(comparison.totalCost.totalCostUsd, costCeiling) + ) { + reasons.push( + `${label} final comparison cost ${comparison.totalCost.totalCostUsd} exceeds ${costCeiling}`, + ) + } + } + if (evidence.answerQuality) { + reasons.push( + ...ragAnswerEvidenceRejectionReasons( + evidence.answerQuality, + evidence.answerQualityCostCeiling, + ), + ) + } + if (reasons.length === 0) return undefined + return { + promoted: false, + reason: reasons.join('; '), + } +} + +function assertOptionalCostCeiling(value: number | undefined, label: string): void { + if (value !== undefined && (!Number.isFinite(value) || value < 0)) { + throw new Error(`${label} must be a non-negative finite number`) + } +} + +function exceedsCostCeiling(totalCostUsd: number, costCeiling: number): boolean { + const tolerance = Number.EPSILON * Math.max(1, Math.abs(totalCostUsd), Math.abs(costCeiling)) * 8 + return totalCostUsd - costCeiling > tolerance +} + function summarizeRagOptimization(result: RunRagOptimizationResult): string { - return `${result.methodName}; winner=${result.winner.surfaceHash}; final_lift=${result.comparison.best.lift.toFixed(3)}` + return `${result.methodName}; winner=${result.winner.surfaceHash}` } async function runKnowledgeResearchUpdate( @@ -442,7 +545,33 @@ function skipPhase( } function summarizeRetrievalResult(result: RunRetrievalImprovementLoopResult): string { - return `${result.methodName}; winner=${result.winner.surfaceHash}; final_lift=${result.comparison.best.lift.toFixed(3)}` + return `${result.methodName}; winner=${result.winner.surfaceHash}` +} + +function selectRagOptimization( + result: RunRagOptimizationResult | undefined, +): RagOptimizationSelection | undefined { + if (!result) return undefined + return { + methodName: result.methodName, + baseline: structuredClone(result.baseline), + winner: structuredClone(result.winner), + baselineConfig: structuredClone(result.baselineConfig), + winnerConfig: structuredClone(result.winnerConfig), + } +} + +function selectRetrievalOptimization( + result: RunRetrievalImprovementLoopResult | undefined, +): RetrievalOptimizationSelection | undefined { + if (!result) return undefined + return { + methodName: result.methodName, + baseline: structuredClone(result.baseline), + winner: structuredClone(result.winner), + baselineConfig: structuredClone(result.baselineConfig), + winnerConfig: structuredClone(result.winnerConfig), + } } function summarizeAcquisitionDecision(decision: KnowledgeResearchLoopDecision): string { @@ -500,7 +629,7 @@ function phaseConfigured( case 'answer-quality': return Boolean(options.evaluateAnswers) case 'promotion': - return Boolean(options.promote) + return Boolean(options.decidePromotion) } } @@ -519,7 +648,7 @@ function requiredPhaseMessage(phase: RagKnowledgeImprovementPhase): string { case 'answer-quality': return 'required phase answer-quality requires an evaluateAnswers hook' case 'promotion': - return 'required phase promotion requires a promote hook' + return 'required phase promotion requires a decidePromotion hook' } } diff --git a/src/rag-optimization.ts b/src/rag-optimization.ts index 0ae64de..4ea03e6 100644 --- a/src/rag-optimization.ts +++ b/src/rag-optimization.ts @@ -1,9 +1,9 @@ import type { DispatchContext, - JsonValue, JudgeConfig, OptimizationMethod, } from '@tangle-network/agent-eval/campaign' +import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' import { jsonObjectCandidateCodec, type RunSerializedKnowledgeOptimizationOptions, diff --git a/src/retrieval-eval.ts b/src/retrieval-eval.ts index abe0c0d..83d829e 100644 --- a/src/retrieval-eval.ts +++ b/src/retrieval-eval.ts @@ -1,11 +1,11 @@ import { canonicalJson } from '@tangle-network/agent-eval' import type { DispatchContext, - JsonValue, JudgeConfig, MutableSurface, Scenario, } from '@tangle-network/agent-eval/campaign' +import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' import { searchKnowledge } from './search' import type { KnowledgeIndex, KnowledgeSearchResult } from './types' diff --git a/src/retrieval-optimization.ts b/src/retrieval-optimization.ts index 053bfd7..7179efd 100644 --- a/src/retrieval-optimization.ts +++ b/src/retrieval-optimization.ts @@ -1,17 +1,10 @@ import { canonicalJson } from '@tangle-network/agent-eval' import { - type CampaignResult, - campaignMeanComposite, - costFromLedgerSummary, - createRunCostLedger, - fsCampaignStorage, - type JsonValue, type JudgeConfig, type OptimizationMethod, - type OptimizationMethodRunOptions, - runCampaign, surfaceHash, } from '@tangle-network/agent-eval/campaign' +import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' import { jsonObjectCandidateCodec, type RunSerializedKnowledgeOptimizationOptions, @@ -25,30 +18,10 @@ import { type RetrievalEvalRetriever, type RetrievalEvalScenario, type RetrievalMetricWeights, - retrievalConfigFromSurface, - retrievalConfigSurface, retrievalRecallJudge, } from './retrieval-eval' import type { KnowledgeIndex } from './types' -export type RetrievalParameterSearchSpace = Record - -export interface BuildBoundedRetrievalConfigsOptions { - baseline: RetrievalConfig - maxConfigurations?: number -} - -export interface BoundedRetrievalConfigMethodOptions { - name?: string - configurations?: readonly RetrievalConfig[] - searchSpace?: RetrievalParameterSearchSpace - maxConfigurations?: number - /** Configuration campaigns run in parallel. Default 4. */ - configurationConcurrency?: number - targetRecall?: number - runOptions?: OptimizationMethodRunOptions -} - type RetrievalOptimizationBaseOptions = Omit< RunSerializedKnowledgeOptimizationOptions< RetrievalConfig, @@ -88,131 +61,6 @@ export interface RunRetrievalImprovementLoopResult finalScenarios: readonly RetrievalEvalScenario[] } -export function buildBoundedRetrievalConfigs( - searchSpace: RetrievalParameterSearchSpace, - options: BuildBoundedRetrievalConfigsOptions, -): RetrievalConfig[] { - const maxConfigurations = options.maxConfigurations ?? 128 - if (!Number.isSafeInteger(maxConfigurations) || maxConfigurations <= 0) { - throw new Error('maxConfigurations must be a positive safe integer') - } - const entries = Object.entries(searchSpace).sort(([left], [right]) => left.localeCompare(right)) - if (entries.length === 0) { - throw new Error('bounded retrieval search requires at least one parameter') - } - let total = 1 - for (const [path, values] of entries) { - assertSafeConfigPath(path) - if (values.length === 0) { - throw new Error(`bounded retrieval search parameter '${path}' has no values`) - } - total *= values.length - if (!Number.isSafeInteger(total) || total > maxConfigurations) { - throw new Error( - `bounded retrieval search expands to more than ${maxConfigurations} configurations; use an OptimizationMethod for larger spaces`, - ) - } - } - - let configurations: RetrievalConfig[] = [structuredClone(options.baseline)] - for (const [path, values] of entries) { - configurations = configurations.flatMap((config) => - values.map((value) => setConfigPath(config, path, value)), - ) - } - const baselineSurface = retrievalConfigSurface(options.baseline) - const unique = new Map() - for (const configuration of configurations) { - const surface = retrievalConfigSurface(configuration) - if (surface !== baselineSurface) unique.set(surface, configuration) - } - return [...unique.values()] -} - -/** - * Exhaustively checks a small, finite retrieval grid through agent-eval. - * Pass the returned complete method explicitly to runRetrievalImprovementLoop(). - * Larger or generative spaces should use an official OptimizationMethod. - */ -export function boundedRetrievalConfigMethod( - options: BoundedRetrievalConfigMethodOptions, -): OptimizationMethod { - if (options.configurations && options.searchSpace) { - throw new Error('bounded retrieval method accepts configurations or searchSpace, not both') - } - if (!options.configurations && !options.searchSpace) { - throw new Error('bounded retrieval method requires configurations or searchSpace') - } - const name = options.name ?? 'bounded-retrieval-config' - return { - name, - async optimize(input) { - const baseline = retrievalConfigFromSurface(input.baselineSurface) - const maxConfigurations = options.maxConfigurations ?? 128 - const configurations = options.configurations - ? normalizeBoundedConfigurations(options.configurations, baseline, maxConfigurations) - : buildBoundedRetrievalConfigs(options.searchSpace!, { - baseline, - maxConfigurations, - }) - if (configurations.length === 0) { - return { - winnerSurface: input.baselineSurface, - cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, - durationMs: 0, - } - } - assertTargetRecall(options.targetRecall) - const concurrency = options.configurationConcurrency ?? 4 - assertConfigurationConcurrency(concurrency) - const startedAt = Date.now() - const runOptions = { - ...input.runOptions, - ...(options.runOptions ?? {}), - } - const storage = runOptions.storage ?? fsCampaignStorage() - const costLedger = createRunCostLedger({ - storage, - runDir: `${input.runDir}/bounded-cost`, - costCeilingUsd: runOptions.costCeiling, - }) - const surfaces = [ - input.baselineSurface, - ...configurations.map(retrievalConfigSurface), - ] as string[] - let winner: BoundedRetrievalMeasurement | undefined - for (let offset = 0; offset < surfaces.length; offset += concurrency) { - const batch = await Promise.all( - surfaces - .slice(offset, offset + concurrency) - .map((surface) => - measureBoundedRetrievalSurface(surface, input, runOptions, storage, costLedger), - ), - ) - for (const measurement of batch) { - if (!measurement.complete) continue - if (!winner || measurement.composite > winner.composite) winner = measurement - } - if ( - options.targetRecall !== undefined && - winner?.recall !== undefined && - winner.recall >= options.targetRecall - ) { - break - } - } - if (!winner) { - throw new Error('bounded retrieval search produced no complete selection measurement') - } - return { - winnerSurface: winner.surface, - cost: costFromLedgerSummary(costLedger.summary()), - durationMs: Date.now() - startedAt, - } - }, - } -} - export async function runRetrievalImprovementLoop( options: RunRetrievalImprovementLoopOptions, ): Promise { @@ -257,148 +105,6 @@ export async function runRetrievalImprovementLoop( } } -interface BoundedRetrievalMeasurement { - surface: string - composite: number - recall?: number - complete: boolean -} - -async function measureBoundedRetrievalSurface( - surface: string, - input: Parameters< - OptimizationMethod['optimize'] - >[0], - runOptions: OptimizationMethodRunOptions, - storage: ReturnType, - costLedger: ReturnType, -): Promise { - const hash = surfaceHash(surface) - const campaign = await runCampaign({ - ...runOptions, - scenarios: [...input.selectionScenarios], - dispatch: (scenario, context) => input.dispatchWithSurface(surface, scenario, context), - dispatchRef: `${runOptions.dispatchRef ?? 'bounded-retrieval'}:${hash}`, - judges: [...input.judges], - runDir: `${input.runDir}/bounded-candidates/${hash}`, - seed: input.seed, - storage, - costLedger, - costPhase: `${runOptions.costPhase ?? 'bounded-retrieval'}.${hash}`, - }) - const recall = campaignDimensionMean(campaign, 'recall') - return { - surface, - composite: campaignMeanComposite(campaign), - ...(recall !== undefined ? { recall } : {}), - complete: campaignIsComplete(campaign), - } -} - -function assertTargetRecall(targetRecall: number | undefined): void { - if ( - targetRecall !== undefined && - (!Number.isFinite(targetRecall) || targetRecall < 0 || targetRecall > 1) - ) { - throw new Error(`targetRecall must be between 0 and 1, got ${String(targetRecall)}`) - } -} - -function assertConfigurationConcurrency(value: number): void { - if (!Number.isSafeInteger(value) || value <= 0) { - throw new Error('bounded retrieval configurationConcurrency must be a positive safe integer') - } -} - -function campaignIsComplete( - campaign: CampaignResult, -): boolean { - const expectedCells = campaign.scenarios.length * campaign.reps - return ( - campaign.cells.length === expectedCells && - campaign.cells.every( - (cell) => - !cell.error && - Object.values(cell.judgeScores).some( - (score) => score.composite !== undefined && Number.isFinite(score.composite), - ), - ) - ) -} - -function campaignDimensionMean( - campaign: CampaignResult, - dimension: string, -): number | undefined { - const values: number[] = [] - for (const cell of campaign.cells) { - if (cell.error) continue - for (const score of Object.values(cell.judgeScores)) { - const value = score.dimensions[dimension] - if (value !== undefined && Number.isFinite(value)) values.push(value) - } - } - if (values.length === 0) return undefined - return values.reduce((sum, value) => sum + value, 0) / values.length -} - -function normalizeBoundedConfigurations( - configurations: readonly RetrievalConfig[], - baseline: RetrievalConfig, - maxConfigurations: number, -): RetrievalConfig[] { - if (!Number.isSafeInteger(maxConfigurations) || maxConfigurations <= 0) { - throw new Error('maxConfigurations must be a positive safe integer') - } - if (configurations.length > maxConfigurations) { - throw new Error( - `bounded retrieval search received ${configurations.length} configurations, exceeding maxConfigurations=${maxConfigurations}`, - ) - } - const baselineSurface = retrievalConfigSurface(baseline) - const unique = new Map() - for (const configuration of configurations) { - const surface = retrievalConfigSurface(configuration) - if (surface !== baselineSurface) unique.set(surface, structuredClone(configuration)) - } - return [...unique.values()] -} - -function setConfigPath(config: RetrievalConfig, path: string, value: JsonValue): RetrievalConfig { - assertSafeConfigPath(path) - const result = structuredClone(config) - const parts = path.split('.') - let current: Record = result - for (const part of parts.slice(0, -1)) { - const child = current[part] - if (child !== undefined && !isJsonObject(child)) { - throw new Error(`retrieval config path '${path}' crosses non-object '${part}'`) - } - const next = child ? structuredClone(child) : {} - current[part] = next - current = next - } - current[parts.at(-1)!] = structuredClone(value) - return result -} - -function assertSafeConfigPath(path: string): void { - const parts = path.split('.') - if ( - parts.length === 0 || - parts.some( - (part) => - !part || - part === '__proto__' || - part === 'prototype' || - part === 'constructor' || - !/^[A-Za-z0-9_-]+$/.test(part), - ) - ) { - throw new Error(`unsafe retrieval config path '${path}'`) - } -} - function retrievalScenarioFingerprint(scenario: RetrievalEvalScenario): string { return surfaceHash( canonicalJson({ @@ -408,7 +114,3 @@ function retrievalScenarioFingerprint(scenario: RetrievalEvalScenario): string { } as JsonValue), ) } - -function isJsonObject(value: unknown): value is Record { - return Boolean(value) && typeof value === 'object' && !Array.isArray(value) -} diff --git a/tests/benchmarks/recovery.test.ts b/tests/benchmarks/recovery.test.ts index 7b01420..99564b5 100644 --- a/tests/benchmarks/recovery.test.ts +++ b/tests/benchmarks/recovery.test.ts @@ -94,7 +94,6 @@ describe('memory adapter benchmark recovery', () => { storage.write( `${runDir}/memory-adapter-attempts.jsonl`, `${JSON.stringify({ - schema: 3, status: 'started', attemptId: 'retired-attempt', candidateId: 'retired', @@ -155,7 +154,6 @@ describe('memory adapter benchmark recovery', () => { storage.write( `${runDir}/memory-adapter-attempts.jsonl`, `${JSON.stringify({ - schema: 3, status: 'started', attemptId: 'unfinished-attempt', candidateId: 'memory', @@ -206,7 +204,6 @@ describe('memory adapter benchmark recovery', () => { storage.write( `${runDir}/memory-adapter-attempts.jsonl`, `${JSON.stringify({ - schema: 3, status: 'started', attemptId: 'unfinished-attempt', candidateId: 'memory', @@ -257,7 +254,6 @@ describe('memory adapter benchmark recovery', () => { storage.write( `${runDir}/memory-adapter-attempts.jsonl`, `${JSON.stringify({ - schema: 3, status: 'started', attemptId: 'unfinished-attempt', candidateId: 'memory', diff --git a/tests/immutable-ref.test.ts b/tests/immutable-ref.test.ts new file mode 100644 index 0000000..705e452 --- /dev/null +++ b/tests/immutable-ref.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from 'vitest' +import { assertImmutableRef } from '../src/immutable-ref' + +describe('assertImmutableRef', () => { + it.each([`sha256:${'a'.repeat(64)}`, `git:${'b'.repeat(40)}`])( + 'accepts canonical immutable ref %s', + (value) => { + expect(() => assertImmutableRef(value, 'ref')).not.toThrow() + }, + ) + + it.each([`sha256:${'A'.repeat(64)}`, `git:${'B'.repeat(40)}`])( + 'rejects non-canonical immutable ref %s', + (value) => { + expect(() => assertImmutableRef(value, 'ref')).toThrow(/must be lowercase/) + }, + ) +}) diff --git a/tests/kb-improvement/activation.test.ts b/tests/kb-improvement/activation.test.ts index c611183..625dd3d 100644 --- a/tests/kb-improvement/activation.test.ts +++ b/tests/kb-improvement/activation.test.ts @@ -4,7 +4,6 @@ import { describe, expect, it } from 'vitest' import { applyKnowledgeWriteBlocks, hashKnowledgeBase, - improveKnowledgeBase, inspectPendingKnowledgeMutation, type KnowledgeImprovementMutationReceipt, knowledgeImprovementCandidateRef, @@ -17,6 +16,7 @@ import { } from '../../src/index' import { withKnowledgeMutation } from '../../src/mutation-lock' import { + improveTestKnowledgeBase as improveKnowledgeBase, knowledgeActivation, knowledgeActivationResult, passingMetric, diff --git a/tests/kb-improvement/candidate.test.ts b/tests/kb-improvement/candidate.test.ts index a5504a2..3aa4537 100644 --- a/tests/kb-improvement/candidate.test.ts +++ b/tests/kb-improvement/candidate.test.ts @@ -2,16 +2,15 @@ import { mkdir, readFile, stat, writeFile } from 'node:fs/promises' import { dirname, join } from 'node:path' import { describe, expect, it } from 'vitest' import { - boundedRetrievalConfigMethod, buildEvalKnowledgeBundle, buildKnowledgeIndex, evaluateKnowledgeBaseReadiness, hashKnowledgeBase, - improveKnowledgeBase, knowledgeImprovementRunDir, } from '../../src/index' import { improveAndPromote, + improveTestKnowledgeBase as improveKnowledgeBase, mutableCandidateRoot, passingMetric, refundProposal, @@ -20,6 +19,7 @@ import { withEmptyRoot, withKb, } from '../support/kb-improvement' +import { fixedOptimizationMethod, testExecutionRef } from '../support/optimization' describe('improveKnowledgeBase', () => { it('leaves the live knowledge base unchanged unless promotion is explicit', async () => { @@ -54,6 +54,51 @@ describe('improveKnowledgeBase', () => { }) }) + it('rejects cached candidate evidence when implementation identity changes', async () => { + await withEmptyRoot(async (root) => { + const firstImplementation = + 'sha256:1111111111111111111111111111111111111111111111111111111111111111' + const secondImplementation = + 'sha256:2222222222222222222222222222222222222222222222222222222222222222' + let secondUpdateCalls = 0 + let secondEvaluationCalls = 0 + const common = { + root, + goal: 'Bind resumed evidence to implementation identity', + runId: 'implementation-bound-resume', + } + const first = await improveKnowledgeBase({ + ...common, + implementationRef: firstImplementation, + async updateKnowledge({ candidateRoot }) { + await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') + return { applied: true, summary: 'created candidate' } + }, + evaluate: passingMetric, + }) + expect(first.state.status).toBe('candidate-ready') + + await expect( + improveKnowledgeBase({ + ...common, + implementationRef: secondImplementation, + async updateKnowledge() { + secondUpdateCalls += 1 + return { applied: true, summary: 'must not run' } + }, + evaluate() { + secondEvaluationCalls += 1 + return passingMetric() + }, + }), + ).rejects.toThrow( + 'knowledge improvement state does not match the requested implementationRef', + ) + expect(secondUpdateCalls).toBe(0) + expect(secondEvaluationCalls).toBe(0) + }) + }) + it('rejects evaluator results without provenance', async () => { await withKb(async (root) => { await expect( @@ -248,6 +293,31 @@ describe('improveKnowledgeBase', () => { }) }) + it('blocks resume instead of evaluating final cases twice after interruption', async () => { + await withKb(async (root) => { + let finalEvaluationCalls = 0 + const options = { + root, + goal: 'Never reuse final cases after an interrupted evaluation', + runId: 'interrupted-final-evaluation', + evaluateDevelopment: passingMetric, + evaluate() { + finalEvaluationCalls += 1 + throw new Error('final evaluator interrupted') + }, + } + + await expect(improveKnowledgeBase(options)).rejects.toThrow('final evaluator interrupted') + expect(finalEvaluationCalls).toBe(1) + + const resumed = await improveKnowledgeBase(options) + + expect(resumed.blocked).toBe(true) + expect(resumed.state.blockedReason).toContain('refusing to reuse final cases') + expect(finalEvaluationCalls).toBe(1) + }) + }) + it('passes the candidate KB root into updateKnowledge callbacks', async () => { await withKb(async (root) => { const seen: Array<{ @@ -323,6 +393,7 @@ describe('improveKnowledgeBase', () => { proposalText: refundProposal(source.id), }), retrieval: { + executionRef: testExecutionRef('candidate-retrieval-execution'), baseline: { k: 1 }, trainScenarios: [ { @@ -366,11 +437,7 @@ describe('improveKnowledgeBase', () => { expected: { kind: 'page', pageId: 'refund-policy' }, }, ], - method: boundedRetrievalConfigMethod({ - searchSpace: { k: [1, 2] }, - targetRecall: 1, - configurationConcurrency: 1, - }), + method: fixedOptimizationMethod('{"k":2}'), retrieve: async ({ k }) => ({ hits: [ { pageId: 'distractor', path: 'knowledge/distractor.md', rank: 1 }, @@ -391,6 +458,36 @@ describe('improveKnowledgeBase', () => { }) }) + it('rejects mutable retrieval identity before updating a candidate KB', async () => { + await withKb(async (root) => { + let updateCalls = 0 + + await expect( + improveKnowledgeBase({ + root, + goal: 'Reject mutable retrieval identity', + runId: 'mutable-retrieval-identity', + updateKnowledge: async () => { + updateCalls += 1 + return { applied: true, summary: 'must not run' } + }, + retrieval: { + executionRef: 'deployment:latest', + baseline: { k: 1 }, + trainScenarios: [], + selectionScenarios: [], + finalScenarios: [], + method: fixedOptimizationMethod('{"k":2}'), + retrieve: async () => ({ hits: [] }), + expectUsage: 'off', + }, + }), + ).rejects.toThrow('knowledge improvement retrieval executionRef') + + expect(updateCalls).toBe(0) + }) + }) + it('rejects a candidate when answer quality fails', async () => { await withKb(async (root) => { const source = refundSource() @@ -408,6 +505,10 @@ describe('improveKnowledgeBase', () => { evaluateAnswers: () => ({ passed: false, metrics: { faithfulness: 0, answer_relevance: 0.8 }, + finalScenarioIds: ['refund-answer-a', 'refund-answer-b'], + datasetRef: testExecutionRef('refund-answer-dataset'), + evaluatorRef: testExecutionRef('refund-answer-evaluator'), + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, findings: [ { id: 'refund-answer:faithfulness', @@ -422,7 +523,36 @@ describe('improveKnowledgeBase', () => { expect(result.promoted).toBe(false) expect(result.state.status).toBe('rejected') expect(result.evaluation?.passed).toBe(false) - expect(result.evaluation?.notes).toContain('answer quality failed') + expect(result.evaluation?.notes).toContain('answer-quality evaluation failed') + }) + }) + + it('rejects candidate promotion when final answer cost is incomplete', async () => { + await withKb(async (root) => { + const result = await improveKnowledgeBase({ + root, + goal: 'Reject unaccounted final answer evaluation', + runId: 'answer-quality-incomplete-cost', + evaluateDevelopment: passingMetric, + evaluate: passingMetric, + evaluateAnswers: () => ({ + passed: true, + metrics: { faithfulness: 1 }, + finalScenarioIds: ['refund-answer-a', 'refund-answer-b'], + datasetRef: testExecutionRef('incomplete-answer-dataset'), + evaluatorRef: testExecutionRef('incomplete-answer-evaluator'), + cost: { + totalCostUsd: 0, + accountingComplete: false, + incompleteReasons: ['provider receipt unavailable'], + }, + }), + answerQualityCostCeiling: 1, + }) + + expect(result.state.status).toBe('rejected') + expect(result.evaluation?.passed).toBe(false) + expect(result.evaluation?.notes).toContain('incomplete cost accounting') }) }) diff --git a/tests/kb-improvement/integrity.test.ts b/tests/kb-improvement/integrity.test.ts index 8c9ad3d..2fafc36 100644 --- a/tests/kb-improvement/integrity.test.ts +++ b/tests/kb-improvement/integrity.test.ts @@ -14,7 +14,6 @@ import { tmpdir } from 'node:os' import { dirname, join, relative } from 'node:path' import { describe, expect, it } from 'vitest' import { - improveKnowledgeBase, knowledgeImprovementCandidateRef, knowledgeImprovementRunDir, loadKnowledgeImprovementState, @@ -23,6 +22,7 @@ import { } from '../../src/index' import { improveAndPromote, + improveTestKnowledgeBase as improveKnowledgeBase, passingMetric, refundProposal, refundSource, @@ -263,6 +263,30 @@ describe('improveKnowledgeBase', () => { }) }) + it('rejects candidate use when persisted implementation identity differs from evidence', async () => { + await withKb(async (root) => { + const staged = await improveKnowledgeBase({ + root, + goal: 'Bind approved evidence to implementation identity', + runId: 'candidate-implementation-evidence', + evaluate: passingMetric, + }) + const candidate = knowledgeImprovementCandidateRef(staged) + const runDir = knowledgeImprovementRunDir(root, staged.runId) + const statePath = join(runDir, 'state.json') + const state = JSON.parse(await readFile(statePath, 'utf8')) as { + implementationRef: string + } + state.implementationRef = + 'sha256:3333333333333333333333333333333333333333333333333333333333333333' + await writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`) + + await expect(promoteKnowledgeCandidate({ root, candidate })).rejects.toThrow( + 'knowledge candidate evidence does not match the approved candidate', + ) + }) + }) + it('fails loudly when persisted improvement state is malformed', async () => { await withKb(async (root) => { const runDir = knowledgeImprovementRunDir(root, 'malformed-state') diff --git a/tests/kb-improvement/optimization.test.ts b/tests/kb-improvement/optimization.test.ts index eba3b6b..326853c 100644 --- a/tests/kb-improvement/optimization.test.ts +++ b/tests/kb-improvement/optimization.test.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto' import { mkdir, readFile, writeFile } from 'node:fs/promises' import { dirname, join } from 'node:path' import { @@ -7,6 +8,8 @@ import { } from '@tangle-network/agent-eval/campaign' import { describe, expect, it } from 'vitest' import { + addSourceText, + applyKnowledgeWriteBlocks, hashKnowledgeBase, improveKnowledgeBase, optimizeKnowledgeBasePolicy, @@ -14,7 +17,14 @@ import { type RagAnswerEvalScenario, scenarioContentFingerprint, } from '../../src/index' -import { mutableCandidateRoot, passingMetric, withKb } from '../support/kb-improvement' +import { + mutableCandidateRoot, + passingMetric, + refundProposal, + refundSource, + refundSpec, + withKb, +} from '../support/kb-improvement' interface PolicyScenario extends Scenario { kind: 'kb-policy-eval' @@ -27,6 +37,10 @@ interface PolicyArtifact { type Policy = { evidence: 'none' | 'required'; maxSources: number } +function immutableRef(value: string): string { + return `sha256:${createHash('sha256').update(value).digest('hex')}` +} + describe('optimizeKnowledgeBasePolicy', () => { it('runs full RAG evaluation against the isolated candidate KB', async () => { await withKb(async (root) => { @@ -50,6 +64,7 @@ describe('optimizeKnowledgeBasePolicy', () => { const result = await improveKnowledgeBase({ root, goal: 'Evaluate RAG against candidate knowledge', + implementationRef: immutableRef('candidate-rag-improvement'), runId: 'candidate-rag-optimization', async updateKnowledge({ candidateRoot }) { const path = join(candidateRoot, 'knowledge', 'candidate-policy.md') @@ -68,6 +83,7 @@ describe('optimizeKnowledgeBasePolicy', () => { return { applied: true, summary: 'wrote candidate knowledge' } }, ragOptimization: { + executionRef: immutableRef('candidate-rag-execution'), baseline: { mode: 'unsupported' }, method, trainScenarios: [scenario('candidate-rag-train')], @@ -121,6 +137,106 @@ describe('optimizeKnowledgeBasePolicy', () => { }) }) + it('uses development checks for retries and runs final evaluation once', async () => { + await withKb(async (root) => { + let methodCalls = 0 + let promotionCalls = 0 + let developmentEvaluatorCalls = 0 + let finalEvaluatorCalls = 0 + const updatedIterations: number[] = [] + const finalDispatches: string[] = [] + const scenario = (id: string): RagAnswerEvalScenario => ({ + id, + kind: 'rag-answer-eval', + query: id, + }) + const method: OptimizationMethod = { + name: 'single-final-method', + async optimize() { + methodCalls += 1 + return { + winnerSurface: '{"mode":"candidate"}', + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + } + }, + } + + const result = await improveKnowledgeBase({ + root, + goal: 'Retry development candidates without reusing final cases', + implementationRef: immutableRef('single-final-improvement'), + runId: 'single-final-improvement', + maxCandidates: 3, + async updateKnowledge({ candidateRoot, iteration }) { + updatedIterations.push(iteration) + if (iteration === 1) return { applied: false, summary: 'left required knowledge absent' } + const source = refundSource() + const added = await addSourceText(candidateRoot, source) + await applyKnowledgeWriteBlocks(candidateRoot, refundProposal(added.id)) + return { applied: true, summary: `updated candidate ${iteration}` } + }, + readinessSpecs: [refundSpec], + strict: true, + ragOptimization: { + executionRef: immutableRef('single-final-rag'), + baseline: { mode: 'baseline' }, + method, + trainScenarios: [scenario('single-final-train')], + selectionScenarios: [scenario('single-final-selection')], + finalScenarios: [scenario('single-final-a'), scenario('single-final-b')], + async run({ scenario: item }) { + if (item.id.startsWith('single-final-') && !item.id.endsWith('train')) { + if (item.id === 'single-final-a' || item.id === 'single-final-b') { + finalDispatches.push(item.id) + } + } + return { query: item.query, answer: 'answer', contexts: [] } + }, + judges: [ + { + name: 'single-final-quality', + dimensions: [{ key: 'quality', description: 'answer quality' }], + score: () => ({ composite: 1, dimensions: { quality: 1 } }), + }, + ], + storage: inMemoryCampaignStorage(), + expectUsage: 'off', + resamples: 200, + }, + requiredPhases: ['rag-optimization', 'promotion'], + evaluateDevelopment({ iteration }) { + developmentEvaluatorCalls += 1 + return { + score: iteration >= 2 ? 1 : 0, + passed: iteration >= 2, + provenance: { + evaluator: 'single-final-development', + version: '1', + method: 'deterministic', + }, + } + }, + evaluate() { + finalEvaluatorCalls += 1 + return passingMetric() + }, + decidePromotion() { + promotionCalls += 1 + return { promoted: false, reason: 'adversarial final rejection' } + }, + }) + + expect(updatedIterations).toEqual([1, 2]) + expect(methodCalls).toBe(1) + expect(promotionCalls).toBe(1) + expect(developmentEvaluatorCalls).toBe(2) + expect(finalEvaluatorCalls).toBe(1) + expect(new Set(finalDispatches)).toEqual(new Set(['single-final-a', 'single-final-b'])) + expect(result.state.status).toBe('rejected') + expect(result.state.candidates).toHaveLength(2) + }) + }) + it('runs a complete method and applies only the exact winner to an isolated candidate', async () => { await withKb(async (root) => { const methodInputs: string[][] = [] @@ -152,7 +268,7 @@ describe('optimizeKnowledgeBasePolicy', () => { trainScenarios: [scenario('policy-train')], selectionScenarios: [scenario('policy-selection')], finalScenarios: [scenario('policy-final-a'), scenario('policy-final-b')], - policyApplicationRef: 'write-maintenance-policy:v1', + policyApplicationRef: immutableRef('write-maintenance-policy'), dispatchCandidate: async ({ candidate }) => ({ score: candidate.evidence === 'required' && candidate.maxSources >= 2 ? 1 : 0, }), @@ -199,7 +315,7 @@ describe('optimizeKnowledgeBasePolicy', () => { expect(result.improvement.lifecycle?.knowledgeUpdate?.metadata?.optimization).toEqual({ method: 'fixture-kb-policy-method', policySurfaceHash: result.optimization.winner.surfaceHash, - policyApplicationRef: 'write-maintenance-policy:v1', + policyApplicationRef: immutableRef('write-maintenance-policy'), }) await expect( readFile(join(root, 'knowledge', 'maintenance-policy.md'), 'utf8'), @@ -240,7 +356,7 @@ describe('optimizeKnowledgeBasePolicy', () => { trainScenarios: [scenario('changing-train')], selectionScenarios: [scenario('changing-selection')], finalScenarios: [scenario('changing-final-a'), scenario('changing-final-b')], - policyApplicationRef: 'changing-policy:v1', + policyApplicationRef: immutableRef('changing-policy'), dispatchCandidate: async () => ({ score: 1 }), judges: [ { diff --git a/tests/kb-improvement/promotion.test.ts b/tests/kb-improvement/promotion.test.ts index 819b24c..6223897 100644 --- a/tests/kb-improvement/promotion.test.ts +++ b/tests/kb-improvement/promotion.test.ts @@ -3,7 +3,6 @@ import { dirname, join, relative } from 'node:path' import { describe, expect, it } from 'vitest' import { hashKnowledgeBase, - improveKnowledgeBase, knowledgeImprovementCandidateRef, knowledgeImprovementRunDir, loadKnowledgeImprovementEvents, @@ -13,6 +12,7 @@ import { withKnowledgeImprovementComparison, } from '../../src/index' import { + improveTestKnowledgeBase as improveKnowledgeBase, mutableCandidateRoot, passingMetric, refundProposal, diff --git a/tests/memory/experiment-cost.test.ts b/tests/memory/experiment-cost.test.ts index fb216a2..56ae3f9 100644 --- a/tests/memory/experiment-cost.test.ts +++ b/tests/memory/experiment-cost.test.ts @@ -103,6 +103,7 @@ describe('agent memory experiment cost and resume', () => { id: 'paid-memory', ref: 'paid-memory:v1', externalCostUsdPerSequence: 0.25, + externalCostAccounting: 'exact', createAdapter() { throw new Error('provider setup rejected') }, @@ -140,8 +141,10 @@ describe('agent memory experiment cost and resume', () => { id: 'paid-memory', ref: 'paid-memory:v1', externalCostUsdPerSequence: 0.25, - createAdapter({ markExternalCall }) { + externalCostAccounting: 'exact', + createAdapter({ markExternalCall, recordExternalCost }) { markExternalCall() + recordExternalCost(0.25) throw new Error('provider provisioning failed') }, }, @@ -155,6 +158,65 @@ describe('agent memory experiment cost and resume', () => { expect(costLedger.summary()).toMatchObject({ totalCalls: 1, totalCostUsd: 0.25 }) }) + it('marks positive external spend incomplete when the provider emits no receipt', async () => { + const storage = inMemoryCampaignStorage() + const costLedger = createRunCostLedger({ + storage, + runDir: '/runs/missing-provider-receipt', + costCeilingUsd: 1, + }) + + const result = await runAgentMemoryExperiment({ + experimentId: 'missing-provider-receipt', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'durable fact' }], + probes: [ + { + id: 'recall', + query: 'durable', + scope: { agentId: 'worker' }, + referenceAnswer: 'durable fact', + }, + ], + }, + ], + }, + ], + candidates: [ + { + id: 'paid-memory', + ref: 'paid-memory:v1', + externalCostUsdPerSequence: 0.25, + externalCostAccounting: 'exact', + createAdapter({ maximumCostUsd, markExternalCall }) { + expect(maximumCostUsd).toBe(0.25) + markExternalCall() + return createScopedTestAdapter('missing-provider-receipt') + }, + }, + ], + runDir: '/runs/missing-provider-receipt', + storage, + costLedger, + }) + + expect(result.campaign.aggregates.cost).toMatchObject({ + accountingComplete: false, + totalCostUsd: 0, + }) + expect(costLedger.summary()).toMatchObject({ + accountingComplete: false, + totalCostUsd: 0, + }) + }) + it('records provider cleanup before a paid-call receipt can be interrupted', async () => { const storage = inMemoryCampaignStorage() const append = storage.append!.bind(storage) @@ -193,8 +255,10 @@ describe('agent memory experiment cost and resume', () => { id: 'memory', ref: 'memory:v1', externalCostUsdPerSequence: 0.1, - createAdapter({ purpose }) { + externalCostAccounting: 'exact', + createAdapter({ purpose, recordExternalCost }) { purposes.push(purpose) + recordExternalCost(0.1) return createScopedTestAdapter(`memory:${purpose}`) }, }, @@ -204,11 +268,9 @@ describe('agent memory experiment cost and resume', () => { costCeiling: 1, }) - const interrupted = await run() - expect(interrupted.campaign.aggregates).toMatchObject({ - cellsFailed: 1, - cost: { accountingComplete: false, unresolvedCalls: 1 }, - }) + await expect(run()).rejects.toThrow( + /paid calls for failed cell .* did not settle .* no complete failure receipt was produced/, + ) expect( storage.read('/runs/execute-receipt-crash/memory-attempts.jsonl')?.trim().split('\n'), ).toHaveLength(2) @@ -272,7 +334,11 @@ describe('agent memory experiment cost and resume', () => { id, ref: `${id}:v1`, externalCostUsdPerSequence: 0.1, - createAdapter: () => createAdapter(id), + externalCostAccounting: 'exact' as const, + createAdapter: ({ recordExternalCost }) => { + recordExternalCost(0.1) + return createAdapter(id) + }, })), runDir: '/runs/parallel-shared-cost', storage: inMemoryCampaignStorage(), @@ -320,7 +386,11 @@ describe('agent memory experiment cost and resume', () => { id: 'memory', ref: 'memory:v1', externalCostUsdPerSequence: 0.1, - createAdapter: ({ sequence }) => createScopedTestAdapter(sequence.id), + externalCostAccounting: 'exact', + createAdapter: ({ branchId, recordExternalCost }) => { + recordExternalCost(0.1) + return createScopedTestAdapter(branchId) + }, }, ], runDir: '/runs/sequential-shared-cost', @@ -356,7 +426,11 @@ describe('agent memory experiment cost and resume', () => { id: 'memory', ref: 'memory:v1', externalCostUsdPerSequence: 0.1, - createAdapter: () => createScopedTestAdapter('memory'), + externalCostAccounting: 'exact' as const, + createAdapter: ({ recordExternalCost }) => { + recordExternalCost(0.1) + return createScopedTestAdapter('memory') + }, } const scenarioId = buildAgentMemorySequenceScenarios([sequence], [candidate])[0]!.id const abandonedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) diff --git a/tests/memory/experiment-execution.test.ts b/tests/memory/experiment-execution.test.ts index 5e08ba4..dd084cd 100644 --- a/tests/memory/experiment-execution.test.ts +++ b/tests/memory/experiment-execution.test.ts @@ -71,9 +71,9 @@ describe('agent memory experiment execution', () => { executeStepRef: 'test-runtime/v1', executeStep: async ({ memory, step }) => { const order = stepOrder.get(memory.branchId) ?? [] - order.push(step.id) + order.push(String(step.ordinal)) stepOrder.set(memory.branchId, order) - if (step.id === 'research') { + if (step.ordinal === 0) { active += 1 maxActive = Math.max(maxActive, active) if (active === 2) release?.() @@ -101,9 +101,7 @@ describe('agent memory experiment execution', () => { expect(result.rows[1]?.scoreMean).toBeLessThan(0.3) expect(result.campaign.cells).toHaveLength(4) expect(snapshots).toHaveLength(4) - expect([...stepOrder.values()].every((steps) => steps.join(',') === 'research,delivery')).toBe( - true, - ) + expect([...stepOrder.values()].every((steps) => steps.join(',') === '0,1')).toBe(true) expect(storage.read(result.rankingJsonPath)).toContain('"candidateId": "team"') expect(storage.read(result.rankingMarkdownPath)).toContain('| 1 | team |') }) diff --git a/tests/memory/experiment-privacy.test.ts b/tests/memory/experiment-privacy.test.ts new file mode 100644 index 0000000..82a5a62 --- /dev/null +++ b/tests/memory/experiment-privacy.test.ts @@ -0,0 +1,267 @@ +import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' +import { describe, expect, it } from 'vitest' +import { createScopedTestAdapter, runAgentMemoryExperiment } from '../support/memory' + +describe('agent memory experiment privacy', () => { + it('does not expose random seeds or repetitions to candidate adapters', async () => { + const adapterInputs: unknown[] = [] + await runAgentMemoryExperiment({ + experimentId: 'paired-seeds', + sequences: [ + { + id: 'paired-history', + family: 'first-party', + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], + }, + ], + }, + ], + candidates: ['a', 'b'].map((candidateId) => ({ + id: candidateId, + ref: `${candidateId}:v1`, + createAdapter: (input) => { + const { + markExternalCall: _markExternalCall, + recordExternalCost: _recordExternalCost, + signal: _signal, + ...visible + } = input + adapterInputs.push(structuredClone(visible)) + return createScopedTestAdapter(candidateId) + }, + })), + reps: 2, + runDir: '/runs/paired-seeds', + storage: inMemoryCampaignStorage(), + maxConcurrency: 4, + }) + + expect(adapterInputs).toHaveLength(4) + for (const input of adapterInputs) { + expect(input).not.toHaveProperty('rep') + expect(input).not.toHaveProperty('seed') + } + }) + + it('never exposes evaluation labels or dataset identity to candidate callbacks', async () => { + const adapterInputs: unknown[] = [] + const stepInputs: unknown[] = [] + const executionContexts: object[] = [] + const executionCostResults: unknown[] = [] + await runAgentMemoryExperiment({ + experimentId: 'redacted-candidate-input', + sequences: [ + { + id: 'FINAL_SEQUENCE_SECRET', + family: 'first-party', + split: 'test', + steps: [ + { + id: 'FINAL_STEP_SECRET', + instruction: 'Remember the supplied launch state.', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'launch state' }], + probes: [ + { + id: 'FINAL_PROBE_SECRET', + query: 'launch state', + requiredFacts: [ + { id: 'EXPECTED_FACT_ID_SECRET', anyOf: ['EXPECTED_FACT_SECRET'] }, + ], + referenceAnswer: 'REFERENCE_ANSWER_SECRET', + }, + ], + metadata: { privateLabel: 'STEP_METADATA_SECRET' }, + }, + ], + metadata: { privateLabel: 'SEQUENCE_METADATA_SECRET' }, + }, + ], + candidates: [ + { + id: 'redacted', + ref: 'redacted:v1', + createAdapter(input) { + const { + markExternalCall: _markExternalCall, + recordExternalCost: _recordExternalCost, + signal: _signal, + ...visible + } = input + adapterInputs.push(structuredClone(visible)) + return createScopedTestAdapter('redacted') + }, + }, + ], + executeStepRef: 'redacted-worker/v1', + async executeStep(input) { + executionContexts.push(input.context) + executionCostResults.push( + await input.context.cost.runPaidCall({ + actor: 'redacted-worker', + model: 'free-local-test', + maximumCharge: { externallyEnforcedMaximumUsd: 0 }, + execute: async () => 'complete', + receipt: () => ({ + model: 'free-local-test', + inputTokens: 0, + outputTokens: 0, + actualCostUsd: 0, + }), + }), + ) + const { memory: _memory, context: _context, ...visible } = input + stepInputs.push(structuredClone(visible)) + }, + runDir: '/runs/redacted-candidate-input', + storage: inMemoryCampaignStorage(), + }) + + expect(executionContexts).toHaveLength(1) + const actualContext = executionContexts[0]! + expect(Reflect.ownKeys(actualContext).map(String).sort()).toEqual(['cost', 'signal']) + for (const privateKey of [ + 'cellId', + 'rep', + 'generation', + 'seed', + 'trace', + 'artifacts', + 'cycleId', + 'resumedFrom', + 'placement', + ]) { + expect(privateKey in actualContext).toBe(false) + } + expect(Reflect.ownKeys((actualContext as { cost: object }).cost).map(String)).toEqual([ + 'runPaidCall', + ]) + const costResult = executionCostResults[0] as { + succeeded: boolean + receipt?: Record + } + expect(costResult.succeeded).toBe(true) + expect(costResult.receipt).toBeDefined() + expect(costResult.receipt).not.toHaveProperty('tags') + expect(costResult.receipt).not.toHaveProperty('phase') + + const visible = JSON.stringify( + publicRuntimeSurface({ + adapterInputs, + stepInputs, + executionContexts, + executionCostResults, + }), + ) + for (const privateLabel of [ + 'FINAL_SEQUENCE_SECRET', + 'FINAL_STEP_SECRET', + 'FINAL_PROBE_SECRET', + 'EXPECTED_FACT_ID_SECRET', + 'EXPECTED_FACT_SECRET', + 'REFERENCE_ANSWER_SECRET', + 'STEP_METADATA_SECRET', + 'SEQUENCE_METADATA_SECRET', + ]) { + expect(visible).not.toContain(privateLabel) + } + expect(stepInputs).toEqual([ + { + candidateId: 'redacted', + step: { + ordinal: 0, + instruction: 'Remember the supplied launch state.', + scope: { agentId: 'worker' }, + }, + }, + ]) + }) + + it('redacts campaign cell identity from execution cancellation', async () => { + let abortReason: unknown + await runAgentMemoryExperiment({ + experimentId: 'redacted-execution-abort', + sequences: [ + { + id: 'FINAL_ABORT_SEQUENCE_SECRET', + family: 'first-party', + steps: [ + { + id: 'FINAL_ABORT_STEP_SECRET', + probes: [ + { + id: 'FINAL_ABORT_PROBE_SECRET', + query: 'state', + referenceAnswer: 'FINAL_ABORT_EXPECTED_SECRET', + }, + ], + }, + ], + }, + ], + candidates: [ + { + id: 'redacted', + ref: 'redacted:v1', + createAdapter: () => createScopedTestAdapter('redacted-abort'), + }, + ], + executeStepRef: 'redacted-abort-worker/v1', + async executeStep({ context }) { + if (context.signal.aborted) { + abortReason = context.signal.reason + return + } + await new Promise((resolve) => { + context.signal.addEventListener( + 'abort', + () => { + abortReason = context.signal.reason + resolve() + }, + { once: true }, + ) + }) + }, + dispatchTimeoutMs: 5, + runDir: '/runs/redacted-execution-abort', + storage: inMemoryCampaignStorage(), + }).catch(() => undefined) + + expect(abortReason).toBeInstanceOf(Error) + expect((abortReason as Error).name).toBe('AbortError') + expect((abortReason as Error).message).toBe('memory execution aborted') + expect(String(abortReason)).not.toContain('FINAL_ABORT_SEQUENCE_SECRET') + expect(String(abortReason)).not.toContain('FINAL_ABORT_STEP_SECRET') + expect(String(abortReason)).not.toContain('FINAL_ABORT_PROBE_SECRET') + expect(String(abortReason)).not.toContain('FINAL_ABORT_EXPECTED_SECRET') + }) +}) + +function publicRuntimeSurface(value: unknown, seen = new WeakSet()): unknown { + if ( + value === null || + typeof value === 'string' || + typeof value === 'number' || + typeof value === 'boolean' || + value === undefined + ) { + return value + } + if (typeof value === 'function') return '[function]' + if (typeof value !== 'object') return String(value) + if (seen.has(value)) return '[circular]' + seen.add(value) + if (value instanceof Error) return { name: value.name, message: value.message } + if (Array.isArray(value)) return value.map((item) => publicRuntimeSurface(item, seen)) + return Object.fromEntries( + Object.keys(value).map((key) => [ + key, + publicRuntimeSurface((value as Record)[key], seen), + ]), + ) +} diff --git a/tests/memory/experiment-recovery.test.ts b/tests/memory/experiment-recovery.test.ts index c3e49b7..c3522f9 100644 --- a/tests/memory/experiment-recovery.test.ts +++ b/tests/memory/experiment-recovery.test.ts @@ -36,7 +36,9 @@ describe('agent memory experiment recovery', () => { id: 'recoverable', ref: 'recoverable:v1', externalRecoveryCostUsdPerAttempt: 0.1, - createAdapter({ branchId, purpose }: { branchId: string; purpose: 'execute' | 'recovery' }) { + externalCostAccounting: 'exact' as const, + createAdapter({ branchId, purpose, recordExternalCost }) { + if (purpose === 'recovery') recordExternalCost(0.1) const executionLabel = purpose === 'recovery' ? 'recovery' : firstExecution ? 'first' : 'retry' branchIds[executionLabel] = branchId @@ -118,7 +120,6 @@ describe('agent memory experiment recovery', () => { storage.write( `${runDir}/memory-attempts.jsonl`, `${JSON.stringify({ - schema: 2, status: 'started', branchId: 'unfinished-branch', candidateId: 'memory', @@ -211,8 +212,10 @@ describe('agent memory experiment recovery', () => { id: 'crash-recoverable', ref: 'crash-recoverable:v1', externalCostUsdPerSequence: 0.1, + externalCostAccounting: 'exact' as const, externalRecoveryCostUsdPerAttempt: 0.1, - createAdapter({ purpose }: { purpose: 'execute' | 'recovery' }) { + createAdapter({ purpose, recordExternalCost }) { + recordExternalCost(0.1) const adapter = createScopedTestAdapter(`crash-recoverable:${purpose}`) const clear = adapter.clear! let clearCalls = 0 @@ -376,6 +379,7 @@ describe('agent memory experiment recovery', () => { id: 'sometimes-created', ref: 'sometimes-created:v1', externalRecoveryCostUsdPerAttempt: 0.1, + externalCostAccounting: 'exact', createAdapter({ purpose }) { purposes.push(purpose) if (purpose === 'recovery') return null @@ -427,7 +431,6 @@ describe('agent memory experiment recovery', () => { storage.write( `${runDir}/memory-attempts.jsonl`, `${JSON.stringify({ - schema: 2, status: 'started', branchId: 'retired-branch', candidateId: 'retired', @@ -462,8 +465,10 @@ describe('agent memory experiment recovery', () => { id: 'retired', ref: 'retired:v1', externalRecoveryCostUsdPerAttempt: 0.1, - createAdapter({ purpose }) { + externalCostAccounting: 'exact', + createAdapter({ purpose, recordExternalCost }) { purposes.push(`retired:${purpose}`) + recordExternalCost(0.1) const adapter = createScopedTestAdapter('retired') adapter.clear = async () => { retiredClears += 1 @@ -497,7 +502,6 @@ describe('agent memory experiment recovery', () => { storage.write( `${runDir}/memory-attempts.jsonl`, `${JSON.stringify({ - schema: 2, status: 'started', branchId: 'unfinished-branch', candidateId: 'memory', @@ -523,6 +527,7 @@ describe('agent memory experiment recovery', () => { id: 'memory', ref: 'memory:v1', externalCostUsdPerSequence: 0.1, + externalCostAccounting: 'exact', createAdapter() { adapterCreates += 1 return createScopedTestAdapter('memory') @@ -557,7 +562,6 @@ describe('agent memory experiment recovery', () => { `${runDir}/memory-attempts.jsonl`, `${[ { - schema: 2, status: 'started', branchId: 'branch-1', candidateId: 'memory', @@ -572,7 +576,6 @@ describe('agent memory experiment recovery', () => { recovery: false, }, { - schema: 2, status: 'started', branchId: 'branch-2', candidateId: 'memory', @@ -625,7 +628,6 @@ describe('agent memory experiment recovery', () => { storage.write( `${runDir}/memory-attempts.jsonl`, `${JSON.stringify({ - schema: 2, status: 'started', branchId: 'unfinished-branch', candidateId: 'memory', diff --git a/tests/memory/experiment-safety.test.ts b/tests/memory/experiment-safety.test.ts index 005bfec..1283ec1 100644 --- a/tests/memory/experiment-safety.test.ts +++ b/tests/memory/experiment-safety.test.ts @@ -231,41 +231,6 @@ describe('agent memory experiment safety', () => { ]) }) - it('uses the same random seed for every candidate on one history and repetition', async () => { - const seeds = new Map() - const sequence = { - id: 'paired-history', - family: 'first-party' as const, - steps: [ - { - id: 'probe', - scope: { agentId: 'worker' }, - probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], - }, - ], - } - await runAgentMemoryExperiment({ - experimentId: 'paired-seeds', - sequences: [sequence], - candidates: ['a', 'b'].map((candidateId) => ({ - id: candidateId, - ref: `${candidateId}:v1`, - createAdapter: ({ sequence: candidateSequence, rep, seed }) => { - seeds.set(`${candidateId}:${candidateSequence.id}:${rep}`, seed) - return createScopedTestAdapter(`${candidateId}:${rep}`) - }, - })), - reps: 2, - runDir: '/runs/paired-seeds', - storage: inMemoryCampaignStorage(), - maxConcurrency: 4, - }) - - expect(seeds.get('a:paired-history:0')).toBe(seeds.get('b:paired-history:0')) - expect(seeds.get('a:paired-history:1')).toBe(seeds.get('b:paired-history:1')) - expect(seeds.get('a:paired-history:0')).not.toBe(seeds.get('a:paired-history:1')) - }) - it('fails the experiment when accepted writes cannot be cleared after a failed step', async () => { let clears = 0 let closes = 0 diff --git a/tests/memory/graphiti.test.ts b/tests/memory/graphiti.test.ts index bb6849f..e934a89 100644 --- a/tests/memory/graphiti.test.ts +++ b/tests/memory/graphiti.test.ts @@ -1,10 +1,12 @@ import { describe, expect, it } from 'vitest' +import { buildCandidate } from '../../src/memory/improvement/candidate' import { type AgentMemoryScope, createAgentMemoryBranch, createGraphitiMemoryAdapter, type GraphitiMcpClientLike, graphitiMemoryAdapterIdentity, + type RunAgentMemoryImprovementOptions, } from '../../src/memory/index' describe('Graphiti adapter', () => { @@ -453,4 +455,26 @@ describe('Graphiti adapter', () => { }), ) }) + + it('uses its identity as a memory improvement candidate ref', async () => { + const config = { provider: 'graphiti' } as const + const ref = graphitiMemoryAdapterIdentity({ + id: 'graphiti', + backendRef: 'graphiti-cluster-a', + }) + const options = { + createCandidate: () => ({ + ref, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0, + externalCostAccounting: 'exact' as const, + createAdapter: () => null, + }), + } as RunAgentMemoryImprovementOptions + + const candidate = await buildCandidate(options, config, 'graphiti') + + expect(candidate.ref).toBe(ref) + expect(candidate.ref).toMatch(/^sha256:[a-f0-9]{64}$/) + }) }) diff --git a/tests/memory/improvement.test.ts b/tests/memory/improvement.test.ts index a8c1329..6c5b9d0 100644 --- a/tests/memory/improvement.test.ts +++ b/tests/memory/improvement.test.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto' import { canonicalJson } from '@tangle-network/agent-eval' import { campaignMeanComposite, @@ -9,6 +10,7 @@ import { } from '@tangle-network/agent-eval/campaign' import { describe, expect, it } from 'vitest' import { stableId } from '../../src/ids' +import { buildCandidate } from '../../src/memory/improvement/candidate' import { type AgentMemorySequence, type AgentMemorySequenceArtifact, @@ -20,6 +22,10 @@ import { createScopedTestAdapter, runAgentMemoryImprovement } from '../support/m type Config = { visibility: 'private' | 'team' } +function immutableRef(value: string): string { + return `sha256:${createHash('sha256').update(value).digest('hex')}` +} + describe('agent memory improvement', () => { it('runs a complete method, keeps final data private, resumes, and activates once', async () => { const storage = inMemoryCampaignStorage() @@ -41,7 +47,7 @@ describe('agent memory improvement', () => { improvementSequence('final-a', 'test'), improvementSequence('final-b', 'test'), ], - improvementRef: 'team-memory-policy/v2', + implementationRef: immutableRef('team-memory-policy'), runDir: '/runs/complete-method-memory', storage, controllerMode: 'process-local', @@ -50,13 +56,16 @@ describe('agent memory improvement', () => { createCandidate: ({ config, candidateId }) => { candidateConstructions += 1 return { - ref: `visibility:${config.visibility}:v2`, + ref: immutableRef(`visibility/${config.visibility}`), policy: { read: [config.visibility], write: config.visibility }, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0, + externalCostAccounting: 'exact', createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), } }, activation: { - ref: 'memory-policy/live:v2', + ref: immutableRef('memory-policy'), async readCurrent() { return structuredClone(activeConfig) }, @@ -88,16 +97,26 @@ describe('agent memory improvement', () => { expect(result.activation.status).toBe('activated') expect(activeConfig).toEqual({ visibility: 'team' }) expect(activationIds).toEqual([result.activation.id]) - expect( - JSON.parse( - storage.read('/runs/complete-method-memory/memory-improvement-manifest.json') ?? '{}', - ).identity?.schema, - ).toBe(7) + const identity = JSON.parse( + storage.read('/runs/complete-method-memory/memory-improvement-manifest.json') ?? '{}', + ).identity + expect(identity).toMatchObject({ + experimentId: 'complete-method-memory', + implementationRef: immutableRef('team-memory-policy'), + method: 'fixture-selection', + }) + expect(identity).not.toHaveProperty('schema') + expect(identity).not.toHaveProperty('builtInFunctionHash') + expect(identity).not.toHaveProperty('callbackFunctionHash') + expect(result.finalEvaluation).toMatchObject({ + baselineCandidateRef: immutableRef('visibility/private'), + winnerCandidateRef: immutableRef('visibility/team'), + }) expect( storage.read( `/runs/complete-method-memory/memory-config-artifacts/${result.winnerSurfaceHash}/${stableId('sequence', 'final-a')}/rep-0-${stableId('seed', '42')}.json`, ), - ).toContain('"sequenceId": "final-a"') + ).toContain(`"candidateRef": "${immutableRef('visibility/team')}"`) expect( storage.read( `/runs/complete-method-memory/memory-final-artifacts/${result.winnerSurfaceHash}/${stableId('sequence', 'final-a')}/rep-0.json`, @@ -107,13 +126,133 @@ describe('agent memory improvement', () => { const constructionsAfterFirstRun = candidateConstructions const resumed = await runAgentMemoryImprovement(options) - expect(candidateConstructions).toBe(constructionsAfterFirstRun) + expect(candidateConstructions).toBe(constructionsAfterFirstRun + 2) expect(activationIds).toEqual([result.activation.id]) expect(resumed.activation.status).toBe('already-activated') expect(resumed.winnerSurfaceHash).toBe(result.winnerSurfaceHash) expect(resumed.finalEvaluation.manifestHash).toBe(result.finalEvaluation.manifestHash) }) + it('rejects an activated journal when the live memory configuration drifted', async () => { + const storage = inMemoryCampaignStorage() + let activeConfig: Config = { visibility: 'private' } + const options = baseOptions({ + experimentId: 'activation-drift', + runDir: '/runs/activation-drift', + storage, + method: selectingMethod([{ visibility: 'private' }, { visibility: 'team' }]), + activation: { + ref: immutableRef('activation-drift-target'), + async readCurrent() { + return structuredClone(activeConfig) + }, + async compareAndSet({ expectedConfig, config }) { + expect(activeConfig).toEqual(expectedConfig) + activeConfig = structuredClone(config) + }, + }, + }) + + const activated = await runAgentMemoryImprovement(options) + expect(activated.activation.status).toBe('activated') + activeConfig = { visibility: 'private' } + + await expect(runAgentMemoryImprovement(options)).rejects.toThrow( + "memory activation target '" + + immutableRef('activation-drift-target') + + "' drifted from measured winner", + ) + }) + + it('shares one paid evaluation while identical concurrent cells await candidate construction', async () => { + let teamAdapterCreations = 0 + let candidateBuildStarted!: () => void + const buildStarted = new Promise((resolve) => { + candidateBuildStarted = resolve + }) + let releaseCandidateBuild!: () => void + const candidateBuildReleased = new Promise((resolve) => { + releaseCandidateBuild = resolve + }) + const team: Config = { visibility: 'team' } + const method: OptimizationMethod = { + name: 'concurrent-duplicate-dispatch', + async optimize(input) { + const surface = canonicalJson(team) + const dispatch = (suffix: string) => + runCampaign({ + ...input.runOptions, + scenarios: [input.trainScenarios[0]!], + dispatch: (scenario, context) => input.dispatchWithSurface(surface, scenario, context), + judges: [...input.judges], + runDir: `${input.runDir}/duplicate/${suffix}`, + seed: input.seed, + }) + const duplicateRuns = Promise.all([dispatch('a'), dispatch('b')]) + await buildStarted + await new Promise((resolve) => setTimeout(resolve, 20)) + releaseCandidateBuild() + await duplicateRuns + return { + winnerSurface: surface, + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + } + }, + } + const result = await runAgentMemoryImprovement( + baseOptions({ + experimentId: 'deduplicate-concurrent-evaluation', + runDir: '/runs/deduplicate-concurrent-evaluation', + method, + async createCandidate({ config, candidateId }) { + if (config.visibility === 'team') { + candidateBuildStarted() + await candidateBuildReleased + } + return { + ref: immutableRef(`visibility/${config.visibility}`), + policy: { read: [config.visibility], write: config.visibility }, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0, + externalCostAccounting: 'exact', + createAdapter: ({ branchId }) => { + if (config.visibility === 'team') teamAdapterCreations += 1 + return createScopedTestAdapter(`${candidateId}:${branchId}`) + }, + } + }, + }), + ) + + expect(result.winnerConfig).toEqual(team) + expect(teamAdapterCreations).toBe(3) + }) + + it('rejects a changed candidate identity before reusing a resumed result', async () => { + const storage = inMemoryCampaignStorage() + let candidateRevision = 'v1' + const options = baseOptions({ + experimentId: 'changed-candidate-identity', + runDir: '/runs/changed-candidate-identity', + storage, + createCandidate: ({ config, candidateId }) => ({ + ref: immutableRef(`visibility/${config.visibility}/${candidateRevision}`), + policy: { read: [config.visibility], write: config.visibility }, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0, + externalCostAccounting: 'exact', + createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), + }), + }) + + await runAgentMemoryImprovement(options) + candidateRevision = 'v2' + + await expect(runAgentMemoryImprovement(options)).rejects.toThrow( + 'changed candidate identity within the same improvement run', + ) + }) + it('recovers an applied activation whose final journal write was interrupted', async () => { const storage = inMemoryCampaignStorage() const append = storage.append!.bind(storage) @@ -137,7 +276,7 @@ describe('agent memory improvement', () => { storage, method: selectingMethod([{ visibility: 'private' }, { visibility: 'team' }]), activation: { - ref: 'memory-policy/live:v2', + ref: immutableRef('memory-policy'), async readCurrent() { return structuredClone(activeConfig) }, @@ -220,12 +359,32 @@ describe('agent memory improvement', () => { baseOptions({ experimentId: 'missing-evaluation-maximum', runDir: '/runs/missing-evaluation-maximum', - maxOptimizationCostUsd: 1, + maxTotalCostUsd: 1, }), ), ).rejects.toThrow('maximumEvaluationCostUsd is required when a spend limit is configured') }) + it('rejects memory candidates without explicit provider cost declarations', async () => { + const options = baseOptions({ + experimentId: 'missing-provider-costs', + runDir: '/runs/missing-provider-costs', + }) + options.createCandidate = (({ + config, + candidateId, + }: Parameters[0]) => ({ + ref: immutableRef(`visibility/${config.visibility}`), + policy: { read: [config.visibility], write: config.visibility }, + createAdapter: ({ branchId }: { branchId: string }) => + createScopedTestAdapter(`${candidateId}:${branchId}`), + })) as typeof options.createCandidate + + await expect(buildCandidate(options, options.baselineConfig, 'missing-costs')).rejects.toThrow( + 'externalCostUsdPerSequence must be a declared non-negative finite number', + ) + }) + it('holds activation when the method cannot fully account for optimization cost', async () => { let activationCalls = 0 const result = await runAgentMemoryImprovement( @@ -238,7 +397,7 @@ describe('agent memory improvement', () => { incompleteReasons: ['external optimizer usage unavailable'], }), activation: { - ref: 'memory-policy/live:v2', + ref: immutableRef('memory-policy'), async readCurrent() { return { visibility: 'private' } }, @@ -271,11 +430,14 @@ function baseOptions( improvementSequence('final-b', 'test'), ], createCandidate: ({ config, candidateId }) => ({ - ref: `visibility:${config.visibility}:v2`, + ref: immutableRef(`visibility/${config.visibility}`), policy: { read: [config.visibility], write: config.visibility }, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0, + externalCostAccounting: 'exact', createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), }), - improvementRef: 'memory-improvement:v2', + implementationRef: immutableRef('memory-improvement'), runDir: '/runs/memory-improvement', storage: inMemoryCampaignStorage(), controllerMode: 'process-local', diff --git a/tests/memory/mem0.test.ts b/tests/memory/mem0.test.ts index b65a70e..cb4b7d2 100644 --- a/tests/memory/mem0.test.ts +++ b/tests/memory/mem0.test.ts @@ -1,11 +1,13 @@ import type { MemoryClient } from 'mem0ai' import type { Memory as OssMemory } from 'mem0ai/oss' import { describe, expect, it } from 'vitest' +import { buildCandidate } from '../../src/memory/improvement/candidate' import { createAgentMemoryBranch, createMem0MemoryAdapter, type Mem0ClientLike, mem0MemoryAdapterIdentity, + type RunAgentMemoryImprovementOptions, } from '../../src/memory/index' describe('Mem0 adapter', () => { @@ -284,6 +286,29 @@ describe('Mem0 adapter', () => { ) }) + it('uses its identity as a memory improvement candidate ref', async () => { + const config = { provider: 'mem0' } as const + const ref = mem0MemoryAdapterIdentity({ + mode: 'hosted', + id: 'mem0', + backendRef: 'mem0-account-a', + }) + const options = { + createCandidate: () => ({ + ref, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0, + externalCostAccounting: 'exact' as const, + createAdapter: () => null, + }), + } as RunAgentMemoryImprovementOptions + + const candidate = await buildCandidate(options, config, 'mem0') + + expect(candidate.ref).toBe(ref) + expect(candidate.ref).toMatch(/^sha256:[a-f0-9]{64}$/) + }) + it('passes OSS entity filters on add', async () => { let addOptions: Record | undefined const adapter = createMem0MemoryAdapter({ diff --git a/tests/official-optimization.integration.test.ts b/tests/official-optimization.integration.test.ts new file mode 100644 index 0000000..6bfc3dc --- /dev/null +++ b/tests/official-optimization.integration.test.ts @@ -0,0 +1,319 @@ +import { spawnSync } from 'node:child_process' +import { mkdtemp, rm } from 'node:fs/promises' +import { createServer, type Server } from 'node:http' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import type { JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign' +import { afterEach, describe, expect, it } from 'vitest' +import type { RetrievalEvalArtifact, RetrievalEvalScenario } from '../src' +import { testExecutionRef } from './support/optimization' + +const campaign = (await import( + process.env.AGENT_EVAL_CAMPAIGN_URL ?? '@tangle-network/agent-eval/campaign' +)) as typeof import('@tangle-network/agent-eval/campaign') +const knowledge = (await import( + process.env.AGENT_KNOWLEDGE_PACKAGE_URL ?? '../src/index' +)) as typeof import('../src/index') +const { gepaOptimizationMethod, skillOptOptimizationMethod } = campaign +const { runRetrievalImprovementLoop, runSerializedKnowledgeOptimization } = knowledge + +const python = process.env.AGENT_EVAL_TEST_PYTHON +const describeWithOfficialEngines = python ? describe : describe.skip +const openServers: Server[] = [] + +afterEach(async () => { + await Promise.all( + openServers.splice(0).map( + (server) => + new Promise((resolve, reject) => { + server.closeAllConnections?.() + server.close((error) => (error ? reject(error) : resolve())) + }), + ), + ) +}) + +describeWithOfficialEngines('official optimizer integration', () => { + it('runs official GEPA through retrieval optimization and final scoring', async () => { + assertPythonModules(python!, ['agent_eval_rpc.gepa_bridge', 'gepa.optimize_anything']) + const root = await mkdtemp(join(tmpdir(), 'agent-knowledge-gepa-')) + const baseUrl = await startModelServer('```\n{"k":2}\n```') + try { + const method = gepaOptimizationMethod({ + name: 'official-gepa-retrieval', + objective: 'Return a JSON retrieval configuration that finds the expected page.', + evaluationId: 'agent-knowledge-official-gepa-retrieval', + background: 'The complete candidate is one canonical JSON object.', + recipe: { + kind: 'engine', + run: { + engine: 'gepa', + maxEvaluations: 4, + maxProposerCostUsd: 1, + maxConcurrency: 1, + stopAtScore: 1, + engineConfig: { + engine: { + capture_stdio: false, + max_workers: 1, + parallel: false, + raise_on_exception: true, + seed: 7, + }, + reflection: { + reflection_minibatch_size: 1, + skip_perfect_score: false, + }, + }, + }, + }, + optimizer: optimizerModel(baseUrl), + describeScenario: (scenario) => ({ + query: scenario.query, + expected: scenario.expected, + }), + describeArtifact: (artifact) => ({ + requestedK: artifact.requestedK, + hits: artifact.hits, + }), + runner: pythonRunner(python!, 'agent_eval_rpc.gepa_bridge'), + }) + const result = await runRetrievalImprovementLoop({ + executionRef: testExecutionRef('official-gepa-retrieval'), + baseline: { k: 1 }, + method, + trainScenarios: [retrievalScenario('gepa-train', 'train query')], + selectionScenarios: [retrievalScenario('gepa-selection', 'selection query')], + finalScenarios: [ + retrievalScenario('gepa-final-a', 'final query a'), + retrievalScenario('gepa-final-b', 'final query b'), + ], + retrieve: async ({ k }) => ({ + hits: [ + { pageId: 'distractor', path: 'knowledge/distractor.md', rank: 1 }, + ...(k >= 2 ? [{ pageId: 'gold', path: 'knowledge/gold.md', rank: 2 }] : []), + ], + }), + runDir: join(root, 'run'), + expectUsage: 'off', + maxConcurrency: 1, + costCeiling: 1, + optimizationRunOptions: { + maxConcurrency: 1, + }, + resamples: 200, + seed: 7, + }) + + expect(result.winnerConfig).toEqual({ k: 2 }) + expect(result.comparison.best).toMatchObject({ + baselineComposite: 0, + winnerComposite: 1, + lift: 1, + provenance: { + source: { package: 'gepa', evidence: 'observed' }, + bridge: { package: 'agent-eval-rpc', evidence: 'observed' }, + }, + }) + expect(result.comparison.best.provenance?.evaluationCount).toBeGreaterThan(0) + expect(result.comparison.best.provenance?.tokenUsage?.calls).toBeGreaterThan(0) + } finally { + await rm(root, { recursive: true, force: true }) + } + }, 300_000) + + it('runs official SkillOpt through serialized text optimization and final scoring', async () => { + assertPythonModules(python!, ['agent_eval_rpc.skillopt_bridge', 'skillopt.engine.trainer']) + const root = await mkdtemp(join(tmpdir(), 'agent-knowledge-skillopt-')) + const modelResponse = JSON.stringify({ + batch_size: 1, + failure_summary: [ + { + count: 1, + description: 'The required response rule is absent.', + failure_type: 'missing_rule', + }, + ], + patch: { + edits: [ + { + content: '\n\n## Required Rule\nALWAYS_RETURN_READY\n', + op: 'append', + }, + ], + reasoning: 'Add the missing response rule.', + }, + }) + const baseUrl = await startModelServer(modelResponse) + try { + const method = skillOptOptimizationMethod({ + name: 'official-skillopt-policy', + objective: 'Add the rule required for a correct answer.', + evaluationId: 'agent-knowledge-official-skillopt-policy', + trainer: { + epochs: 1, + batchSize: 1, + accumulation: 1, + editBudget: 1, + minEditBudget: 1, + analystWorkers: 1, + minibatchSize: 1, + maxAnalystRounds: 1, + evaluationWorkers: 1, + }, + optimizer: optimizerModel(baseUrl), + maxEvaluations: 3, + describeScenario: (scenario) => ({ prompt: scenario.prompt }), + describeArtifact: (artifact) => ({ candidate: artifact.candidate }), + runner: pythonRunner(python!, 'agent_eval_rpc.skillopt_bridge'), + }) + const result = await runSerializedKnowledgeOptimization({ + executionRef: testExecutionRef('official-skillopt-skill'), + baseline: '# Base Skill\nAnswer normally.\n', + method, + trainScenarios: [skillScenario('skill-train', 'Return READY for training.')], + selectionScenarios: [skillScenario('skill-selection', 'Return READY for selection.')], + finalScenarios: [ + skillScenario('skill-final-a', 'Return READY for final case A.'), + skillScenario('skill-final-b', 'Return READY for final case B.'), + ], + codec: { + serialize: (candidate) => candidate, + parse: (surface) => surface, + }, + dispatchCandidate: async ({ candidate }) => { + return { candidate } + }, + judges: [skillJudge], + runDir: join(root, 'run'), + expectUsage: 'off', + maxConcurrency: 1, + costCeiling: 1, + optimizationRunOptions: { + maxConcurrency: 1, + }, + resamples: 200, + seed: 11, + }) + + expect(result.winner.value).toContain('ALWAYS_RETURN_READY') + expect(result.comparison.best).toMatchObject({ + baselineComposite: 0, + winnerComposite: 1, + lift: 1, + provenance: { + source: { package: 'skillopt', evidence: 'observed' }, + bridge: { package: 'agent-eval-rpc', evidence: 'observed' }, + }, + }) + expect(result.comparison.best.provenance?.evaluationCount).toBe(3) + expect(result.comparison.best.provenance?.tokenUsage?.calls).toBe(1) + } finally { + await rm(root, { recursive: true, force: true }) + } + }, 300_000) +}) + +interface SkillScenario extends Scenario { + kind: 'skill-policy' + prompt: string +} + +interface SkillArtifact { + candidate: string +} + +const skillJudge: JudgeConfig = { + name: 'required-rule', + dimensions: [{ key: 'correctness', description: 'The candidate includes the required rule.' }], + score: ({ artifact }) => { + const score = artifact.candidate.includes('ALWAYS_RETURN_READY') ? 1 : 0 + return { + dimensions: { correctness: score }, + composite: score, + notes: score ? '' : 'The required response rule is absent.', + } + }, +} + +function retrievalScenario(id: string, query: string): RetrievalEvalScenario { + return { + id, + kind: 'retrieval-eval', + query, + expected: { kind: 'page', pageId: 'gold' }, + } +} + +function skillScenario(id: string, prompt: string): SkillScenario { + return { id, kind: 'skill-policy', prompt } +} + +function pythonRunner(command: string, module: string) { + return { command, args: ['-m', module] } +} + +function optimizerModel(baseUrl: string) { + return { + model: 'local-optimizer', + baseUrl, + apiKey: 'local-test-key', + budget: { + maxCostUsd: 1, + maxRequests: 10, + maxRequestBytes: 100_000, + maxResponseBytes: 100_000, + maxOutputTokensPerRequest: 2_000, + pricing: { + inputUsdPerMillion: 1, + outputUsdPerMillion: 2, + }, + }, + } +} + +function assertPythonModules(command: string, modules: readonly string[]): void { + const imports = modules.map((module) => `import ${module}`).join('; ') + const result = spawnSync(command, ['-c', imports], { encoding: 'utf8' }) + if (result.status !== 0) { + throw new Error(`official optimizer Python environment is unavailable: ${result.stderr.trim()}`) + } +} + +async function startModelServer(content: string): Promise { + const server = createServer(async (request, response) => { + for await (const _chunk of request) { + // Drain the request before replying. + } + response.writeHead(200, { 'content-type': 'application/json' }) + response.end( + JSON.stringify({ + id: 'local-completion', + choices: [ + { + finish_reason: 'stop', + index: 0, + message: { role: 'assistant', content }, + }, + ], + model: 'local-optimizer', + usage: { + prompt_tokens: 20, + completion_tokens: 20, + total_tokens: 40, + }, + }), + ) + }) + openServers.push(server) + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', () => { + server.off('error', reject) + resolve() + }) + }) + const address = server.address() + if (!address || typeof address === 'string') throw new Error('model server did not bind') + return `http://127.0.0.1:${address.port}/v1` +} diff --git a/tests/official-optimizer-resume.contract.test.ts b/tests/official-optimizer-resume.contract.test.ts new file mode 100644 index 0000000..eba3c51 --- /dev/null +++ b/tests/official-optimizer-resume.contract.test.ts @@ -0,0 +1,191 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + gepaOptimizationMethod, + type OptimizationMethod, + type Scenario, + skillOptOptimizationMethod, +} from '@tangle-network/agent-eval/campaign' +import { afterEach, describe, expect, it } from 'vitest' +import { runSerializedKnowledgeOptimization } from '../src/index' +import { testExecutionRef } from './support/optimization' + +interface ResumeScenario extends Scenario { + kind: 'resume-contract' + prompt: string +} + +interface ResumeArtifact { + score: number +} + +type ResumeCandidate = { policy: string } + +const roots: string[] = [] +const modelBudget = { + maxCostUsd: 0.1, + maxRequests: 1, + maxRequestBytes: 10_000, + maxResponseBytes: 10_000, + maxOutputTokensPerRequest: 100, + pricing: { inputUsdPerMillion: 1, outputUsdPerMillion: 1 }, +} + +afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))) +}) + +describe('official optimizer resume identity', () => { + it.each([ + [ + 'GEPA', + () => + gepaOptimizationMethod({ + recipe: { + kind: 'engine', + run: { + engine: 'gepa', + maxEvaluations: 1, + maxProposerCostUsd: 0.1, + }, + }, + objective: 'Improve the policy.', + evaluationId: 'knowledge-resume-contract', + resume: 'if-compatible', + trustResumeState: true, + runner: fakeOfficialOptimizerRunner('gepa'), + }), + ], + [ + 'SkillOpt', + () => + skillOptOptimizationMethod({ + objective: 'Improve the policy.', + evaluationId: 'knowledge-resume-contract', + trainer: { epochs: 1, batchSize: 1 }, + optimizer: { + model: 'unused-test-model', + baseUrl: 'http://127.0.0.1:1/v1', + apiKey: 'unused-test-key', + budget: modelBudget, + }, + maxEvaluations: 1, + resume: 'if-compatible', + runner: fakeOfficialOptimizerRunner('skillopt'), + }), + ], + ] as const)( + 'changes %s compatible resume identity when Knowledge executionRef changes', + async (_label, createMethod) => { + const root = await mkdtemp(join(tmpdir(), 'agent-knowledge-official-resume-')) + roots.push(root) + const method = createMethod() + const first = await runOfficialMethod(root, method, testExecutionRef('implementation-a')) + const second = await runOfficialMethod(root, method, testExecutionRef('implementation-b')) + + expect(first).not.toBe(second) + }, + ) +}) + +async function runOfficialMethod( + runDir: string, + method: OptimizationMethod, + executionRef: string, +): Promise { + const result = await runSerializedKnowledgeOptimization< + ResumeCandidate, + ResumeScenario, + ResumeArtifact + >({ + executionRef, + baseline: { policy: 'baseline' }, + method, + trainScenarios: [scenario('train', 'training policy')], + selectionScenarios: [scenario('selection', 'selection policy')], + finalScenarios: [ + scenario('final-a', 'first final policy'), + scenario('final-b', 'second final policy'), + ], + dispatchCandidate: async ({ candidate }) => ({ + score: candidate.policy === 'better' ? 1 : 0, + }), + judges: [ + { + name: 'resume-contract-quality', + dimensions: [{ key: 'quality', description: 'candidate policy quality' }], + score: ({ artifact }) => ({ + composite: artifact.score, + dimensions: { quality: artifact.score }, + }), + }, + ], + runDir, + expectUsage: 'off', + resamples: 40, + }) + return result.comparison.best.provenance?.compatibleRunId +} + +function scenario(id: string, prompt: string): ResumeScenario { + return { id, kind: 'resume-contract', prompt } +} + +function fakeOfficialOptimizerRunner(optimizer: 'gepa' | 'skillopt') { + const runtime = { + python: { implementation: 'CPython', version: '3.12.0' }, + bridge: { + package: 'agent-eval-rpc', + version: 'test', + sourceUrl: 'https://github.com/tangle-network/agent-eval', + revision: 'test', + sourceSha256: 'a'.repeat(64), + }, + optimizer: { + package: optimizer, + version: 'test', + sourceUrl: + optimizer === 'gepa' + ? 'https://github.com/gepa-ai/gepa' + : 'https://github.com/microsoft/SkillOpt', + revision: 'test', + sourceSha256: 'b'.repeat(64), + }, + engineModules: [], + } + const optimizeResult = + optimizer === 'gepa' + ? [ + 'bestCandidate: input.seedCandidate,', + 'bestScore: 0,', + 'totalEvaluations: 0,', + 'recipeKind: input.recipe.kind,', + 'proposerCostAccounting: "unavailable",', + ] + : [ + 'bestCandidate: input.seedCandidate,', + 'bestScore: 0,', + 'totalEvaluations: 0,', + 'totalSteps: 0,', + 'tokenUsage: { inputTokens: 0, outputTokens: 0, totalTokens: 0, calls: 0, requestAttempts: 0 },', + ] + const source = [ + "const fs = require('node:fs')", + "const inputPath = process.argv[process.argv.indexOf('--input') + 1]", + "const outputPath = process.argv[process.argv.indexOf('--output') + 1]", + 'const input = JSON.parse(fs.readFileSync(inputPath, "utf8"))', + `const runtime = ${JSON.stringify(runtime)}`, + 'if (input.operation === "inspect") {', + ' fs.writeFileSync(outputPath, JSON.stringify({ runtime }))', + ' process.exit(0)', + '}', + 'fs.writeFileSync(outputPath, JSON.stringify({', + ...optimizeResult, + 'upstream: runtime.optimizer,', + 'runId: input.runId,', + 'resumed: false,', + '}))', + ].join('\n') + return { command: process.execPath, args: ['-e', source, '--'] } +} diff --git a/tests/rag-eval.test.ts b/tests/rag-eval.test.ts index f6a220f..814abf0 100644 --- a/tests/rag-eval.test.ts +++ b/tests/rag-eval.test.ts @@ -25,6 +25,12 @@ const scenario: RagAnswerEvalScenario = { requireCitations: true, } +const secondScenario: RagAnswerEvalScenario = { + ...scenario, + id: 'refund-window-paraphrase', + query: 'What is the refund request deadline?', +} + const strongArtifact: RagAnswerEvalArtifact = { query: scenario.query, answer: 'Customers can request refunds within 30 days.', @@ -123,8 +129,10 @@ describe('RAG answer evaluation', () => { it('builds a lifecycle answer-quality hook over real answer cases', async () => { const hook = createRagAnswerQualityHook({ - scenarios: [scenario], - run: () => strongArtifact, + scenarios: [scenario, secondScenario], + evaluatorRef: `sha256:${'a'.repeat(64)}`, + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + run: (item) => ({ ...strongArtifact, query: item.query }), externalEvaluator: () => ({ provider: 'trulens', scores: { groundedness: 1, answer_relevance: 1, context_relevance: 1 }, @@ -134,7 +142,9 @@ describe('RAG answer evaluation', () => { const result = await hook() expect(result.passed).toBe(true) expect(result.metrics.composite).toBe(1) - expect(result.metadata?.scenarioCount).toBe(1) + expect(result.finalScenarioIds).toEqual(['refund-window', 'refund-window-paraphrase']) + expect(result.datasetRef).toMatch(/^sha256:[a-f0-9]{64}$/) + expect(result.metadata?.scenarioCount).toBe(2) }) it('returns an agent-eval judge for direct campaign wiring', async () => { diff --git a/tests/rag-improvement-loop.test.ts b/tests/rag-improvement-loop.test.ts index 9102af2..e99d408 100644 --- a/tests/rag-improvement-loop.test.ts +++ b/tests/rag-improvement-loop.test.ts @@ -7,12 +7,12 @@ import { } from '@tangle-network/agent-eval/campaign' import { afterEach, describe, expect, it } from 'vitest' import { - boundedRetrievalConfigMethod, type RagAnswerEvalArtifact, type RagAnswerEvalScenario, type RetrievalEvalScenario, runRagKnowledgeImprovementLoop, } from '../src/index' +import { fixedOptimizationMethod, testExecutionRef } from './support/optimization' const tempRoots: string[] = [] @@ -24,6 +24,7 @@ afterEach(async () => { describe('RAG knowledge improvement loop', () => { it('runs a complete method over retrieval and answer configuration without exposing final data', async () => { const methodInputs: string[][] = [] + let finalDispatchStarted = false const method: OptimizationMethod = { name: 'fixture-rag-method', async optimize(input) { @@ -52,12 +53,14 @@ describe('RAG knowledge improvement loop', () => { enabledPhases: ['rag-optimization', 'gap-diagnosis'], requiredPhases: ['rag-optimization'], optimization: { + executionRef: testExecutionRef('rag-complete-method'), baseline: { answerMode: 'unsupported', k: 1 }, method, trainScenarios: [scenario('rag-train')], selectionScenarios: [scenario('rag-selection')], finalScenarios: [scenario('rag-final-a'), scenario('rag-final-b')], async run({ config, scenario: item }) { + if (item.id.startsWith('rag-final-')) finalDispatchStarted = true const claim = `${item.id} refunds are allowed within 30 days` if (config.answerMode !== 'grounded') { return { @@ -87,7 +90,8 @@ describe('RAG knowledge improvement loop', () => { resamples: 200, }, diagnose({ optimization }) { - expect(optimization?.winnerConfig).toEqual({ answerMode: 'grounded', k: 2 }) + expect(optimization).toBeUndefined() + expect(finalDispatchStarted).toBe(false) return [] }, }) @@ -97,13 +101,14 @@ describe('RAG knowledge improvement loop', () => { expect(result.optimization?.comparison.testScenarioIds).toEqual(['rag-final-a', 'rag-final-b']) expect(result.optimization?.comparison.best.lift).toBeGreaterThan(0) expect(result.phases.map((phase) => `${phase.phase}:${phase.status}`)).toEqual([ - 'rag-optimization:completed', 'gap-diagnosis:completed', + 'rag-optimization:completed', ]) }) it('exposes retrieval, diagnosis, acquisition, update, answer eval, and promotion phases', async () => { const calls: string[] = [] + let finalRetrievalCalls = 0 const trainScenario: RetrievalEvalScenario = { id: 'q-train', kind: 'retrieval-eval', @@ -120,6 +125,7 @@ describe('RAG knowledge improvement loop', () => { const result = await runRagKnowledgeImprovementLoop({ goal: 'Improve support RAG', retrieval: { + executionRef: testExecutionRef('rag-retrieval-lifecycle'), baseline: { k: 1 }, trainScenarios: [trainScenario], selectionScenarios: [ @@ -128,17 +134,16 @@ describe('RAG knowledge improvement loop', () => { makeScenario('q-selection-c'), ], finalScenarios: [makeScenario('q-final-a'), makeScenario('q-final-b')], - method: boundedRetrievalConfigMethod({ - searchSpace: { k: [1, 2] }, - targetRecall: 1, - configurationConcurrency: 1, - }), - retrieve: async ({ k }) => ({ - hits: [ - { pageId: 'distractor', path: 'knowledge/distractor.md', rank: 1 }, - ...(k >= 2 ? [{ pageId: 'gold', path: 'knowledge/gold.md', rank: 2 }] : []), - ], - }), + method: fixedOptimizationMethod('{"k":2}'), + retrieve: async ({ k, scenario }) => { + if (scenario.id.startsWith('q-final-')) finalRetrievalCalls += 1 + return { + hits: [ + { pageId: 'distractor', path: 'knowledge/distractor.md', rank: 1 }, + ...(k >= 2 ? [{ pageId: 'gold', path: 'knowledge/gold.md', rank: 2 }] : []), + ], + } + }, runDir: 'memory://rag-lifecycle-retrieval-test', storage: inMemoryCampaignStorage(), expectUsage: 'off', @@ -146,7 +151,8 @@ describe('RAG knowledge improvement loop', () => { }, diagnose({ retrieval }) { calls.push('diagnose') - expect(retrieval?.winnerConfig).toMatchObject({ k: 2 }) + expect(retrieval).toBeUndefined() + expect(finalRetrievalCalls).toBe(0) return [ { id: 'missing-refund-policy', @@ -156,9 +162,12 @@ describe('RAG knowledge improvement loop', () => { }, ] }, - acquireKnowledge({ findings }) { + acquireKnowledge({ findings, retrieval, phases }) { calls.push('acquire') expect(findings).toHaveLength(1) + expect(retrieval).toBeUndefined() + expect(finalRetrievalCalls).toBe(0) + expect(phases.some((phase) => phase.summary.includes('final_lift'))).toBe(false) return { sourceTexts: [ { @@ -171,19 +180,33 @@ describe('RAG knowledge improvement loop', () => { done: true, } }, - updateKnowledge({ acquisition }) { + updateKnowledge({ acquisition, retrieval }) { calls.push('update') expect(acquisition?.sourceTexts).toHaveLength(1) + expect(retrieval).toBeUndefined() + expect(finalRetrievalCalls).toBe(0) return { applied: true, summary: 'external vector DB updated' } }, - evaluateAnswers({ knowledgeUpdate }) { + evaluateAnswers({ knowledgeUpdate, retrieval }) { calls.push('answer') expect(knowledgeUpdate?.applied).toBe(true) - return { passed: true, metrics: { faithfulness: 1, answer_relevance: 0.95 } } + expect(retrieval).not.toHaveProperty('comparison') + expect(finalRetrievalCalls).toBeGreaterThan(0) + return { + passed: true, + metrics: { faithfulness: 1, answer_relevance: 0.95 }, + finalScenarioIds: ['answer-final-a', 'answer-final-b'], + datasetRef: testExecutionRef('answer-final-dataset'), + evaluatorRef: testExecutionRef('answer-final-evaluator'), + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + } }, - promote({ answerQuality }) { + answerQualityCostCeiling: 0, + decidePromotion({ answerQuality, retrieval, retrievalComparison }) { calls.push('promote') expect(answerQuality?.passed).toBe(true) + expect(retrieval).not.toHaveProperty('comparison') + expect(retrievalComparison?.best.liftCi.low).toBeGreaterThanOrEqual(0) return { promoted: true, reason: 'retrieval and answer checks passed' } }, }) @@ -195,15 +218,136 @@ describe('RAG knowledge improvement loop', () => { expect(result.answerQuality?.passed).toBe(true) expect(result.promotion?.promoted).toBe(true) expect(result.phases.map((phase) => `${phase.phase}:${phase.status}`)).toEqual([ - 'retrieval-tuning:completed', 'gap-diagnosis:completed', 'knowledge-acquisition:completed', 'knowledge-update:completed', + 'retrieval-tuning:completed', 'answer-quality:completed', 'promotion:completed', ]) }) + it('rejects weak answer-only evidence before a promotion decision can run', async () => { + let promotionCalls = 0 + await expect( + runRagKnowledgeImprovementLoop({ + goal: 'Reject self-attested answer evidence', + enabledPhases: ['answer-quality', 'promotion'], + evaluateAnswers: () => + ({ + passed: true, + metrics: {}, + finalScenarioIds: ['final-a', 'final-b'], + datasetRef: testExecutionRef('weak-answer-dataset'), + evaluatorRef: testExecutionRef('weak-answer-evaluator'), + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + }) as never, + answerQualityCostCeiling: 0, + decidePromotion() { + promotionCalls += 1 + return { promoted: true, reason: 'must not run' } + }, + }), + ).rejects.toThrow('answer-quality evidence requires non-empty finite metrics') + expect(promotionCalls).toBe(0) + }) + + it('holds answer-only promotion when observed cost is incomplete', async () => { + let promotionCalls = 0 + const result = await runRagKnowledgeImprovementLoop({ + goal: 'Require observed answer-evaluation cost', + enabledPhases: ['answer-quality', 'promotion'], + evaluateAnswers: () => ({ + passed: true, + metrics: { faithfulness: 1 }, + finalScenarioIds: ['final-a', 'final-b'], + datasetRef: testExecutionRef('incomplete-answer-dataset'), + evaluatorRef: testExecutionRef('incomplete-answer-evaluator'), + cost: { + totalCostUsd: 0, + accountingComplete: false, + incompleteReasons: ['provider receipt unavailable'], + }, + }), + answerQualityCostCeiling: 1, + decidePromotion() { + promotionCalls += 1 + return { promoted: true, reason: 'must not run' } + }, + }) + + expect(result.promotion).toMatchObject({ + promoted: false, + reason: expect.stringContaining('incomplete cost accounting'), + }) + expect(promotionCalls).toBe(0) + }) + + it('refuses promotion when the selected retrieval candidate regresses on final cases', async () => { + let decisionCalls = 0 + const scenario = (id: string): RetrievalEvalScenario => ({ + id, + kind: 'retrieval-eval', + query: id, + expected: [{ kind: 'page', pageId: 'gold' }], + }) + const result = await runRagKnowledgeImprovementLoop({ + goal: 'Reject a final regression', + enabledPhases: ['retrieval-tuning', 'promotion'], + retrieval: { + executionRef: testExecutionRef('rag-final-regression'), + baseline: { k: 1 }, + trainScenarios: [scenario('train')], + selectionScenarios: [scenario('selection')], + finalScenarios: [scenario('final-a'), scenario('final-b')], + method: fixedOptimizationMethod('{"k":2}'), + retrieve: async ({ k }) => ({ + hits: + k === 1 + ? [{ pageId: 'gold', path: 'knowledge/gold.md', rank: 1 }] + : [{ pageId: 'distractor', path: 'knowledge/distractor.md', rank: 1 }], + }), + runDir: 'memory://rag-final-regression-test', + storage: inMemoryCampaignStorage(), + expectUsage: 'off', + resamples: 200, + }, + decidePromotion() { + decisionCalls += 1 + return { promoted: true, reason: 'caller requested promotion' } + }, + }) + + expect(result.retrieval?.comparison.best).toMatchObject({ + baselineComposite: 1, + winnerComposite: 0, + lift: -1, + }) + expect(result.promotion).toMatchObject({ + promoted: false, + reason: expect.stringContaining('does not rule out a regression'), + }) + expect(decisionCalls).toBe(0) + }) + + it('does not call the promotion decision without final evidence', async () => { + let decisionCalls = 0 + const result = await runRagKnowledgeImprovementLoop({ + goal: 'Reject an unevaluated update', + enabledPhases: ['promotion'], + decidePromotion() { + decisionCalls += 1 + return { promoted: true, reason: 'caller requested promotion' } + }, + }) + + expect(decisionCalls).toBe(0) + expect(result.promotion).toEqual({ + promoted: false, + reason: 'promotion requires final RAG, retrieval, or answer-quality evidence', + }) + }) + it('does not run full RAG optimization when the phase is disabled', async () => { let methodCalled = false const method: OptimizationMethod = { @@ -226,6 +370,7 @@ describe('RAG knowledge improvement loop', () => { goal: 'Run diagnosis only', enabledPhases: ['gap-diagnosis'], optimization: { + executionRef: testExecutionRef('rag-disabled-method'), baseline: { k: 1 }, method, trainScenarios: [scenario('disabled-train')], diff --git a/tests/retrieval-eval.test.ts b/tests/retrieval-eval.test.ts index 8c7f188..9b80880 100644 --- a/tests/retrieval-eval.test.ts +++ b/tests/retrieval-eval.test.ts @@ -5,8 +5,6 @@ import { } from '@tangle-network/agent-eval/campaign' import { describe, expect, it } from 'vitest' import { - boundedRetrievalConfigMethod, - buildBoundedRetrievalConfigs, buildRetrievalEvalDispatch, type KnowledgeIndex, type RetrievalEvalArtifact, @@ -17,8 +15,10 @@ import { runRetrievalImprovementLoop, scoreRetrievalArtifact, } from '../src/index' +import { fixedOptimizationMethod, testExecutionRef } from './support/optimization' const signal = new AbortController().signal +const executionRef = testExecutionRef('retrieval-eval-fixture') function testContext() { return { @@ -187,35 +187,6 @@ describe('retrieval eval', () => { expect(campaign.aggregates.cost.totalCalls).toBe(1) }) - it('enumerates a bounded retrieval grid and rejects spaces above its explicit limit', () => { - const baseline = { k: 5, hybrid: false, reranker: null, chunk: { overlap: 100 } } - const configurations = buildBoundedRetrievalConfigs( - { - 'chunk.overlap': [100, 200], - hybrid: [false, true], - k: [5, 10], - }, - { baseline, maxConfigurations: 8 }, - ) - - expect(configurations).toHaveLength(7) - expect(configurations).toContainEqual({ - k: 10, - hybrid: true, - reranker: null, - chunk: { overlap: 200 }, - }) - expect(() => - buildBoundedRetrievalConfigs( - { - k: [1, 2, 3], - hybrid: [false, true], - }, - { baseline, maxConfigurations: 5 }, - ), - ).toThrow(/more than 5 configurations/) - }) - it('runs a complete OptimizationMethod without exposing final cases to it', async () => { const seen: string[][] = [] const method: OptimizationMethod = { @@ -232,6 +203,7 @@ describe('retrieval eval', () => { }, } const result = await runRetrievalImprovementLoop({ + executionRef, baseline: { k: 1 }, method, trainScenarios: [retrievalScenario('train', 'train query')], @@ -277,6 +249,41 @@ describe('retrieval eval', () => { ) }) + it('requires an immutable execution identity before starting the method', async () => { + let methodCalled = false + const method: OptimizationMethod = { + name: 'must-not-run-without-identity', + async optimize(input) { + methodCalled = true + return { + winnerSurface: input.baselineSurface, + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + } + }, + } + + await expect( + runRetrievalImprovementLoop({ + executionRef: 'git:ABCDEF', + baseline: { k: 1 }, + method, + trainScenarios: [retrievalScenario('identity-train', 'train query')], + selectionScenarios: [retrievalScenario('identity-selection', 'selection query')], + finalScenarios: [ + retrievalScenario('identity-final-a', 'final query a'), + retrievalScenario('identity-final-b', 'final query b'), + ], + retrieve: retrievalFixture, + runDir: '/runs/retrieval-invalid-identity-test', + storage: inMemoryCampaignStorage(), + expectUsage: 'off', + }), + ).rejects.toThrow( + 'knowledge optimization executionRef must be lowercase sha256:<64 hex> or git:<40 hex>', + ) + expect(methodCalled).toBe(false) + }) + it('rejects renamed duplicate scenarios before starting the method', async () => { let methodCalled = false const method: OptimizationMethod = { @@ -292,6 +299,7 @@ describe('retrieval eval', () => { await expect( runRetrievalImprovementLoop({ + executionRef, baseline: { k: 1 }, method, trainScenarios: [ @@ -328,6 +336,7 @@ describe('retrieval eval', () => { await expect( runRetrievalImprovementLoop({ + executionRef, baseline: { k: 1 }, method, trainScenarios: [retrievalScenario('invalid-train', 'train query')], @@ -348,9 +357,11 @@ describe('retrieval eval', () => { expect(retrievalCalls).toBe(0) }) - it('uses the neutral bounded method for a small finite retrieval space', async () => { + it('runs and resumes a supplied complete retrieval method', async () => { const storage = inMemoryCampaignStorage() const retrievedK: number[] = [] + let activeExecutionRef = testExecutionRef('retrieval-resume-v1') + let candidateImproves = true const trainScenarios = [retrievalScenario('train', 'train query')] const selectionScenarios = [ retrievalScenario('selection-a', 'selection query a'), @@ -361,13 +372,13 @@ describe('retrieval eval', () => { retrievalScenario('final-a', 'final query a'), retrievalScenario('final-b', 'final query b'), ] - const method = boundedRetrievalConfigMethod({ - configurations: [{ k: 2 }, { k: 3 }], - configurationConcurrency: 1, - targetRecall: 1, - }) + const method = fixedOptimizationMethod( + retrievalConfigSurface({ k: 2 }), + 'fixture-retrieval', + ) const run = () => runRetrievalImprovementLoop({ + executionRef: activeExecutionRef, baseline: { k: 1 }, method, trainScenarios, @@ -375,9 +386,14 @@ describe('retrieval eval', () => { finalScenarios, retrieve: async (input) => { retrievedK.push(input.k) - return retrievalFixture(input) + const findsGold = candidateImproves ? input.k >= 2 : input.k === 1 + return { + hits: findsGold + ? [{ pageId: 'gold', path: 'knowledge/gold.md', rank: 1 }] + : [{ pageId: 'distractor', path: 'knowledge/distractor.md', rank: 1 }], + } }, - runDir: '/runs/retrieval-bounded-test', + runDir: '/runs/retrieval-method-test', storage, expectUsage: 'off', resamples: 200, @@ -385,8 +401,7 @@ describe('retrieval eval', () => { const result = await run() expect(result.winnerConfig).toMatchObject({ k: 2 }) - expect(result.methodName).toBe('bounded-retrieval-config') - expect(retrievedK).not.toContain(3) + expect(result.methodName).toBe('fixture-retrieval') expect(result.trainScenarios).toHaveLength(1) expect(result.selectionScenarios).toHaveLength(3) expect(result.finalScenarios).toHaveLength(2) @@ -395,6 +410,12 @@ describe('retrieval eval', () => { const resumed = await run() expect(retrievedK).toHaveLength(callsAfterFirstRun) expect(resumed.winner.surfaceHash).toBe(result.winner.surfaceHash) + + candidateImproves = false + activeExecutionRef = testExecutionRef('retrieval-resume-v2') + const changed = await run() + expect(retrievedK.length).toBeGreaterThan(callsAfterFirstRun) + expect(changed.comparison.best.lift).toBe(-1) }) it('fails loudly on invalid config surfaces and empty expected labels', () => { diff --git a/tests/support/kb-improvement.ts b/tests/support/kb-improvement.ts index 52f1e6e..a37fa94 100644 --- a/tests/support/kb-improvement.ts +++ b/tests/support/kb-improvement.ts @@ -9,10 +9,11 @@ import { } from '@tangle-network/agent-interface' import { defineReadinessSpec, - improveKnowledgeBase, + improveKnowledgeBase as improveKnowledgeBaseRaw, initKnowledgeBase, type KnowledgeImprovementCandidateRef, type KnowledgeImprovementMutationReceipt, + type KnowledgeImprovementOptions, knowledgeImprovementCandidateRef, knowledgeImprovementRunDir, promoteKnowledgeCandidate, @@ -20,6 +21,20 @@ import { stableId, } from '../../src/index' +export const TEST_KNOWLEDGE_IMPLEMENTATION_REF = + 'sha256:4b6f6866d7f2c1fbb0df2ab91d0f2f8a2da124f3e95640d42c416d2675f9d6ce' + +export function improveTestKnowledgeBase( + options: Omit & { + implementationRef?: string + }, +) { + return improveKnowledgeBaseRaw({ + implementationRef: TEST_KNOWLEDGE_IMPLEMENTATION_REF, + ...options, + }) +} + export async function withKb(fn: (root: string) => Promise): Promise { const root = await mkdtemp(join(tmpdir(), 'agent-knowledge-improve-')) try { @@ -191,8 +206,8 @@ export function knowledgeActivationResult( }) } -export async function improveAndPromote(options: Parameters[0]) { - const staged = await improveKnowledgeBase(options) +export async function improveAndPromote(options: Parameters[0]) { + const staged = await improveTestKnowledgeBase(options) const promoted = await promoteKnowledgeCandidate({ root: options.root, candidate: knowledgeImprovementCandidateRef(staged), diff --git a/tests/support/optimization.ts b/tests/support/optimization.ts new file mode 100644 index 0000000..c35f170 --- /dev/null +++ b/tests/support/optimization.ts @@ -0,0 +1,21 @@ +import { createHash } from 'node:crypto' +import type { OptimizationMethod, Scenario } from '@tangle-network/agent-eval/campaign' + +export function testExecutionRef(value: string): string { + return `sha256:${createHash('sha256').update(value).digest('hex')}` +} + +export function fixedOptimizationMethod( + winnerSurface: string, + name = 'fixture-fixed', +): OptimizationMethod { + return { + name, + async optimize() { + return { + winnerSurface, + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + } + }, + } +}