From ad7ed7f57eabfdb6eb19cd70d3b47a2e47877e2f Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 24 Jul 2026 01:24:22 -0600 Subject: [PATCH 01/14] fix(optimization): configure official optimizer models --- bench/src/hev-improve.mts | 178 +++++--- bench/src/official-optimizer-config.mts | 87 ++++ bench/src/swe-improve.mts | 212 +++++---- bench/src/trata-gepa.mts | 419 ++++++++---------- examples/ablation-suite/gepa-driver-prompt.ts | 178 +++++--- 5 files changed, 607 insertions(+), 467 deletions(-) create mode 100644 bench/src/official-optimizer-config.mts diff --git a/bench/src/hev-improve.mts b/bench/src/hev-improve.mts index 6e370159..cf88ebfa 100644 --- a/bench/src/hev-improve.mts +++ b/bench/src/hev-improve.mts @@ -1,8 +1,6 @@ /** - * SELF-IMPROVEMENT on HumanEval — the prompt-sensitive, VISIBLE-ORACLE counterpart - * to the SWE-bench run. Same machinery (improve(surface:'prompt') + gepaProposer + - * held-out gate), but the worker is a single chat completion and the judge is the - * deterministic Docker checker (run the function against its own hidden unit tests). + * Official GEPA prompt optimization on HumanEval. The worker is a single chat + * completion and the judge is the deterministic Docker checker. * * WHY this exists: on SWE-bench the same GEPA loop was NULL because the grading test * is withheld — the worker cannot verify, so prompt wording cannot move resolve. @@ -13,11 +11,15 @@ * Worker + reflect models call the zai coding endpoint directly (no tangle router, * no WAF, no 503): TANGLE_API_KEY=$ZAI_API_KEY ROUTER_BASE=https://api.z.ai/api/coding/paas/v4 */ -import { improve } from '@tangle-network/agent-runtime' +import { improve, officialGepa } from '@tangle-network/agent-runtime' import type { AgentProfile } from '@tangle-network/agent-interface' import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract' -import { gepaProposer } from '@tangle-network/agent-eval/campaign' import { extractCode, loadHumanEval, runChecker, type HumanEvalTask } from './benchmarks/humaneval' +import { + assertCompleteCost, + officialOptimizerModel, + requiredTokenPricing, +} from './official-optimizer-config.mjs' // The SEED instruction GEPA evolves. Byte-identical to humaneval.ts basePrompt's // solveInstruction so the baseline arm reproduces the plain-prompt denominator. @@ -26,7 +28,6 @@ const SEED_INSTRUCTION = interface Completion { text: string - usd: number tokIn: number tokOut: number } @@ -45,10 +46,7 @@ async function complete(base: string, key: string, model: string, prompt: string const text = d.choices?.[0]?.message?.content ?? '' const tokIn = d.usage?.prompt_tokens ?? 0 const tokOut = d.usage?.completion_tokens ?? 0 - // zai glm pricing is ~ $0.6/M in, $2.2/M out (coding plan); a rough cost tag so the - // stub-guard sees a real backend. Exact cost is not the metric (pass-rate is). - const usd = (tokIn * 0.6 + tokOut * 2.2) / 1_000_000 - return { text, usd, tokIn, tokOut } + return { text, tokIn, tokOut } } async function main(): Promise { @@ -63,29 +61,53 @@ async function main(): Promise { const reflectBase = process.env.REFLECT_BASE ?? base const reflectKey = process.env.REFLECT_KEY ?? key const trainN = Number(process.env.TRAIN_N ?? 12) - const holdoutN = Number(process.env.HOLDOUT_N ?? 12) + const selectionN = Number(process.env.SELECTION_N ?? 12) + const testN = Number(process.env.TEST_N ?? 12) const offset = Number(process.env.OFFSET ?? 80) - // generations=1 never exercises the GEPA Pareto/combine path (the frontier - // needs >=1 completed generation before combine can fire) — default to a - // multi-generation budget so the default run measures the full loop. - const generations = Number(process.env.GENERATIONS ?? 6) - const population = Number(process.env.POPULATION ?? 4) + const maxEvaluations = Number(process.env.MAX_EVALUATIONS ?? 24) + const maxProposerCostUsd = Number(process.env.MAX_PROPOSER_COST_USD ?? 5) const workerMaxTokens = Number(process.env.MAX_TOKENS ?? 6000) const reflectMaxTokens = Number(process.env.REFLECT_MAX_TOKENS ?? 8000) const maxConcurrency = Number(process.env.MAX_CONCURRENCY ?? 4) + const runDir = process.env.RUN_DIR ?? '.runs/humaneval-official-gepa' + const evaluationId = [ + 'humaneval-prompt-eval', + `worker=${workerModel}`, + `endpoint=${new URL(base).origin}`, + `tokens=${workerMaxTokens}`, + 'checker=local-python', + ].join('|') + if (process.env.DRYRUN) { + console.log( + `DRYRUN: imports OK (improve=${typeof improve}, officialGepa=${typeof officialGepa})`, + ) + return + } + const workerPricing = requiredTokenPricing(process.env, 'WORKER') + const optimizer = officialOptimizerModel({ + env: process.env, + model: reflectModel, + baseUrl: reflectBase, + apiKey: reflectKey, + maxCostUsd: maxProposerCostUsd, + maxOutputTokensPerRequest: reflectMaxTokens, + }) - // TRAIN and HOLDOUT are DISJOINT slices of the harder middle band (offset). + // All three partitions are disjoint slices of the harder middle band. const train = await loadHumanEval(trainN, offset) - const holdout = await loadHumanEval(holdoutN, offset + trainN) - const byId = new Map([...train, ...holdout].map((t) => [t.taskId, t])) - const allIds = [...byId.keys()] + const selection = await loadHumanEval(selectionN, offset + trainN) + const testCases = await loadHumanEval(testN, offset + trainN + selectionN) + const byId = new Map( + [...train, ...selection, ...testCases].map((t) => [t.taskId, t]), + ) - console.log('═══ HumanEval self-improvement — VISIBLE oracle (deterministic Docker checker) ═══') - console.log(`worker=${workerModel} reflect=${reflectModel} base=${base}`) + console.log('=== HumanEval prompt optimization with official GEPA ===') + console.log(`worker=${workerModel} reflect=${reflectModel} base=${base}`) console.log(`train=[${train.map((t) => t.taskId).join(', ')}]`) - console.log(`holdout=[${holdout.map((t) => t.taskId).join(', ')}]`) - console.log(`generations=${generations} population=${population} offset=${offset} maxTokens=${workerMaxTokens}`) - console.log(`≈ ${trainN * (1 + generations * population) + 2 * holdoutN} cells (each = 1 completion + 1 Docker check)\n`) + console.log(`selection=[${selection.map((t) => t.taskId).join(', ')}]`) + console.log(`test=[${testCases.map((t) => t.taskId).join(', ')}]`) + console.log(`maxEvaluations=${maxEvaluations} maxProposerCostUsd=${maxProposerCostUsd} offset=${offset} maxTokens=${workerMaxTokens}`) + console.log(`runDir=${runDir}\n`) const stats = { n: 0 } const agent = async (surface: unknown, scenario: Scenario, ctx: DispatchContext): Promise => { @@ -94,20 +116,32 @@ async function main(): Promise { if (!t) throw new Error(`agent: unknown scenario ${scenario.id}`) const prompt = `${instr}\n\n\`\`\`python\n${t.prompt}\`\`\`` const t0 = Date.now() - const r = await complete(base, key, workerModel, prompt, workerMaxTokens) - const zeroUsage = r.tokIn === 0 && r.tokOut === 0 + const paid = await ctx.cost.runPaidCall({ + channel: 'agent', + actor: 'humaneval-worker', + model: workerModel, + execute: () => complete(base, key, workerModel, prompt, workerMaxTokens), + receipt: (result) => { + const usageUnknown = result.tokIn === 0 && result.tokOut === 0 + return { + model: workerModel, + inputTokens: result.tokIn, + outputTokens: result.tokOut, + customTokenPricing: workerPricing, + ...(usageUnknown ? { usageUnknown: true } : {}), + } + }, + }) + if (!paid.succeeded) throw paid.error + const r = paid.value const hasText = r.text.trim().length > 0 - ctx.cost.observe(zeroUsage && hasText ? Math.max(r.usd, 0.0001) : r.usd, workerModel) - ctx.cost.observeTokens( - zeroUsage && hasText ? { input: Math.max(r.tokIn, 1), output: Math.max(r.tokOut, 1) } : { input: r.tokIn, output: r.tokOut }, - ) stats.n += 1 const codeLen = extractCode(r.text).length console.log(` [agent] ${scenario.id} instr=${instr.length}c code=${codeLen}b tok=in:${r.tokIn}/out:${r.tokOut} ${Math.round((Date.now() - t0) / 1000)}s`) return hasText ? r.text : null } - const judge: JudgeConfig = { + const judge: JudgeConfig = { name: 'humaneval-docker', dimensions: [{ key: 'pass', description: 'the completed function passes its hidden unit tests (deterministic Docker checker)' }], async score({ artifact, scenario }) { @@ -137,49 +171,53 @@ async function main(): Promise { } const profile: AgentProfile = { name: 'hev-solver', prompt: { systemPrompt: SEED_INSTRUCTION } } - const proposer = gepaProposer({ - llm: { baseUrl: reflectBase, apiKey: reflectKey }, - model: reflectModel, - target: - 'the instruction/system prompt strategy for a SMALL model completing Python functions to pass hidden unit tests. ' + - 'Propose SUBSTANTIALLY different strategies, not wording tweaks: e.g. require the model to first reason step-by-step ' + - 'about the algorithm and edge cases (empty inputs, off-by-one, boundary values, types) in a brief plan or comments ' + - 'BEFORE writing the code; provide a short worked example; or add an explicit self-check against the docstring. ' + - 'Bold rewrites that change model BEHAVIOR beat cosmetic edits.', - maxTokens: reflectMaxTokens, - temperature: 0.7, - }) - - const scenarios: Scenario[] = allIds.map((id) => ({ id, kind: 'humaneval' })) - const holdoutScenarios: Scenario[] = holdout.map((t) => ({ id: t.taskId, kind: 'humaneval' })) + const scenario = (task: HumanEvalTask): Scenario => ({ id: task.taskId, kind: 'humaneval' }) - const out = await improve(profile, [], { + const out = await improve(profile, { surface: 'prompt', - gate: 'holdout', - generator: proposer, - scenarios, - judge, + method: officialGepa({ + objective: + 'Improve the complete instruction for a small model that writes Python functions which pass hidden unit tests.', + evaluationId, + background: + 'Prefer behavioral strategies over wording changes. Address algorithm choice, edge cases, boundary values, type behavior, and self-checking. Return only the complete instruction.', + recipe: { + kind: 'engine', + run: { + engine: 'gepa', + maxEvaluations, + maxProposerCostUsd, + }, + }, + optimizer, + resume: 'if-compatible', + trustResumeState: true, + describeScenario: (item) => ({ prompt: byId.get(item.id)?.prompt ?? item.id }), + }), + trainScenarios: train.map(scenario), + selectionScenarios: selection.map(scenario), + testScenarios: testCases.map(scenario), + judges: [judge], agent, expectUsage: 'warn', - // rawTraceContext stays OFF deliberately: it swaps the distilled findings for - // filesystem paths + grep/cat instructions (rawTraceDistiller), which only a - // coding harness can execute. This run's proposer is prompt-tier (gepaProposer - // — a single LLM call that cannot run grep), so the trace evidence arrives via - // the judge's traceback notes + the breakdown's `emitted` excerpt instead. - budget: { generations, populationSize: population, holdoutScenarios, maxConcurrency, reps: 1 }, - llm: { baseUrl: reflectBase, apiKey: reflectKey, model: reflectModel }, + maxConcurrency, + reps: 1, + runDir, + optimizationRunOptions: { + expectUsage: 'warn', + maxConcurrency, + reps: 1, + }, }) - console.log('\n═══ RESULT ═══') - console.log(`gateDecision=${out.gateDecision} shipped=${out.shipped} lift=${out.lift}`) - console.log(`baseline holdout pass-rate = ${out.raw.baseline.compositeMean}`) - console.log(`winner holdout pass-rate = ${out.raw.winner.compositeMean}`) - console.log(`baseline per-scenario: ${JSON.stringify(out.raw.baseline.perScenario)}`) - console.log(`winner per-scenario: ${JSON.stringify(out.raw.winner.perScenario)}`) - if (out.raw.winner.label) console.log(`winner label : ${out.raw.winner.label}`) - if ((out.raw.winner as { surface?: unknown }).surface) { - console.log(`winner instruction:\n${String((out.raw.winner as { surface?: unknown }).surface).slice(0, 1200)}`) - } + assertCompleteCost('humaneval official GEPA run', out.cost) + console.log('\n=== RESULT ===') + console.log(`decision=${out.decision} lift=${out.lift} interval=[${out.liftInterval.low}, ${out.liftInterval.high}]`) + console.log(`baseline test pass-rate=${out.raw.best.baselineComposite}`) + console.log(`winner test pass-rate=${out.raw.best.winnerComposite}`) + console.log(`test scenarios=${JSON.stringify(out.raw.best.scenarioScores)}`) + console.log(`cost=${JSON.stringify(out.cost)}`) + console.log(`winner instruction:\n${String(out.candidate.value).slice(0, 2000)}`) } main().catch((e) => { diff --git a/bench/src/official-optimizer-config.mts b/bench/src/official-optimizer-config.mts new file mode 100644 index 00000000..da7a4d7f --- /dev/null +++ b/bench/src/official-optimizer-config.mts @@ -0,0 +1,87 @@ +function requiredNonNegativeNumber( + env: NodeJS.ProcessEnv, + name: string, +): number { + const raw = env[name] + if (raw === undefined || raw.trim() === '') { + throw new Error(`env ${name} is required`) + } + const value = Number(raw) + if (!Number.isFinite(value) || value < 0) { + throw new Error(`env ${name} must be a finite non-negative number`) + } + return value +} + +function positiveInteger( + env: NodeJS.ProcessEnv, + name: string, + fallback: number, +): number { + const value = Number(env[name] ?? fallback) + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`env ${name} must be a positive integer`) + } + return value +} + +export function requiredTokenPricing( + env: NodeJS.ProcessEnv, + prefix: 'WORKER' | 'REFLECT', +) { + return { + inputUsdPerMillion: requiredNonNegativeNumber( + env, + `${prefix}_INPUT_USD_PER_MILLION`, + ), + cachedInputUsdPerMillion: requiredNonNegativeNumber( + env, + `${prefix}_CACHED_INPUT_USD_PER_MILLION`, + ), + cacheWriteUsdPerMillion: requiredNonNegativeNumber( + env, + `${prefix}_CACHE_WRITE_USD_PER_MILLION`, + ), + outputUsdPerMillion: requiredNonNegativeNumber( + env, + `${prefix}_OUTPUT_USD_PER_MILLION`, + ), + } +} + +export function officialOptimizerModel(options: { + env: NodeJS.ProcessEnv + model: string + baseUrl: string + apiKey: string + maxCostUsd: number + maxOutputTokensPerRequest: number +}) { + const { env } = options + return { + model: options.model, + baseUrl: options.baseUrl, + apiKey: options.apiKey, + budget: { + maxCostUsd: options.maxCostUsd, + maxRequests: positiveInteger(env, 'REFLECT_MAX_REQUESTS', 100), + maxRequestBytes: positiveInteger(env, 'REFLECT_MAX_REQUEST_BYTES', 2_000_000), + maxResponseBytes: positiveInteger(env, 'REFLECT_MAX_RESPONSE_BYTES', 2_000_000), + maxOutputTokensPerRequest: options.maxOutputTokensPerRequest, + requestTimeoutMs: positiveInteger(env, 'REFLECT_REQUEST_TIMEOUT_MS', 300_000), + pricing: requiredTokenPricing(env, 'REFLECT'), + }, + } +} + +export function assertCompleteCost( + label: string, + cost: { accountingComplete: boolean; incompleteReasons: readonly string[] }, +): void { + if (cost.accountingComplete) return + const reasons = + cost.incompleteReasons.length > 0 + ? cost.incompleteReasons.join('; ') + : 'no incomplete reason was recorded' + throw new Error(`${label}: cost accounting is incomplete: ${reasons}`) +} diff --git a/bench/src/swe-improve.mts b/bench/src/swe-improve.mts index a7e7695a..0fbca03a 100644 --- a/bench/src/swe-improve.mts +++ b/bench/src/swe-improve.mts @@ -1,10 +1,9 @@ /** - * SELF-IMPROVEMENT on the SEE-able LOCAL SWE-bench path — NO tangle sandbox. + * Official GEPA prompt optimization on the local SWE-bench path. * - * Composes the three proven pieces into ONE held-out-gated improvement generation: - * 1. `improve({ surface: 'prompt' })` (agent-runtime) drives the loop: it asks - * `gepaProposer` to EVOLVE the SWE agent's system prompt, then measures each - * candidate prompt on real instances and gates the winner on a held-out split. + * Composes three pieces: + * 1. `improve({ method: officialGepa(...) })` runs GEPA's upstream + * Optimize Anything engine on explicit train and selection partitions. * 2. Per candidate + scenario, the `agent` fn runs the LOCAL SWE env * (`createSweBenchEnvironment` + `runAgentic`): clone the instance repo to a * host tmpdir, run the jailed list/read/edit tool loop with the CANDIDATE @@ -16,23 +15,26 @@ * IN-LOOP score is a cheap patch-exists proxy (NOT the Docker judge) so the ONLY * Docker run per cell is the improve judge — one deterministic verdict per cell. * - * Cost per run = T·(1 + G·P) + 2·H cells, each = 1 clone + 1 runAgentic + 1 judge. - * - * TANGLE_API_KEY=… dotenvx run -f …/agent-state.env -- \ - * TRAIN_IDS=psf__requests-2931 HOLDOUT_IDS=psf__requests-1142 \ - * GENERATIONS=1 POPULATION=1 WORKER_MODEL=glm-4.6 REFLECT_MODEL=glm-4.6 \ + * TANGLE_API_KEY=... dotenvx run -f .../agent-state.env -- \ + * TRAIN_IDS=psf__requests-2931 SELECTION_IDS=pallets__flask-5014 \ + * TEST_IDS=psf__requests-1142,psf__requests-1921 \ + * MAX_EVALUATIONS=4 MAX_PROPOSER_COST_USD=2 \ * node_modules/.bin/tsx bench/src/swe-improve.mts */ import { execFile } from 'node:child_process' import { promisify } from 'node:util' -import { improve } from '@tangle-network/agent-runtime' +import { improve, officialGepa } from '@tangle-network/agent-runtime' import type { AgentProfile } from '@tangle-network/agent-interface' import type { AgenticSurface, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/loops' import { refine, runAgentic } from '@tangle-network/agent-runtime/loops' import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract' -import { gepaProposer } from '@tangle-network/agent-eval/campaign' import { createSweBenchAdapter } from './benchmarks/swe-bench' import type { BenchTask } from './benchmarks/types' +import { + assertCompleteCost, + officialOptimizerModel, + requiredTokenPricing, +} from './official-optimizer-config.mjs' import { createSweBenchEnvironment, SWE_SEED_PROMPT, SWE_SEED_PROMPT_WITH_RUN } from './swe-bench-env' const exec = promisify(execFile) @@ -42,36 +44,56 @@ async function main(): Promise { if (!routerKey) throw new Error('TANGLE_API_KEY required (the worker + reflection call the router)') const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1' const workerModel = process.env.WORKER_MODEL ?? 'glm-4.6' + const reflectBase = process.env.REFLECT_BASE ?? routerBaseUrl + const reflectKey = process.env.REFLECT_KEY ?? routerKey const reflectModel = process.env.REFLECT_MODEL ?? 'glm-4.6' const trainIds = (process.env.TRAIN_IDS ?? 'psf__requests-2931').split(',').map((s) => s.trim()).filter(Boolean) - const holdoutIds = (process.env.HOLDOUT_IDS ?? 'psf__requests-1142').split(',').map((s) => s.trim()).filter(Boolean) - const generations = Number(process.env.GENERATIONS ?? 1) - const population = Number(process.env.POPULATION ?? 1) + const selectionIds = (process.env.SELECTION_IDS ?? 'pallets__flask-5014').split(',').map((s) => s.trim()).filter(Boolean) + const testIds = (process.env.TEST_IDS ?? 'psf__requests-1142,psf__requests-1921').split(',').map((s) => s.trim()).filter(Boolean) + const maxEvaluations = Number(process.env.MAX_EVALUATIONS ?? 4) + const maxProposerCostUsd = Number(process.env.MAX_PROPOSER_COST_USD ?? 2) const innerTurns = Number(process.env.INNER_TURNS ?? 40) const workerMaxTokens = Number(process.env.MAX_TOKENS ?? 8000) const reflectMaxTokens = Number(process.env.REFLECT_MAX_TOKENS ?? 12000) const maxConcurrency = Number(process.env.MAX_CONCURRENCY ?? 1) const budgetShots = Number(process.env.BUDGET ?? 1) + const runDir = process.env.RUN_DIR ?? '.runs/swe-official-gepa' // WITH-TOOLS arm: RUN_TOOL=1 exposes the jailed `run` tool AND swaps the seed to the run-aware prompt. // Default OFF ⇒ reproduces the read/edit-only baseline denominator unchanged. const enableRun = ['1', 'true', 'yes'].includes((process.env.RUN_TOOL ?? '').toLowerCase()) const SEED_PROMPT = enableRun ? SWE_SEED_PROMPT_WITH_RUN : SWE_SEED_PROMPT + const evaluationId = [ + 'swe-prompt-eval', + `worker=${workerModel}`, + `router=${new URL(routerBaseUrl).origin}`, + `turns=${innerTurns}`, + `tokens=${workerMaxTokens}`, + `shots=${budgetShots}`, + `runTool=${String(enableRun)}`, + ].join('|') + const allIds = [...new Set([...trainIds, ...selectionIds, ...testIds])] - const allIds = [...new Set([...trainIds, ...holdoutIds])] - const cellsMax = trainIds.length * (1 + generations * population) + 2 * holdoutIds.length - - console.log('═══ SWE-bench self-improvement — SEE-able LOCAL (no tangle sandbox) ═══') - console.log(`worker=${workerModel} reflect=${reflectModel} router=${routerBaseUrl}`) - console.log(`train=[${trainIds.join(', ')}] holdout=[${holdoutIds.join(', ')}]`) - console.log(`generations=${generations} population=${population} innerTurns=${innerTurns} workerMaxTokens=${workerMaxTokens} reflectMaxTokens=${reflectMaxTokens} runTool=${enableRun}`) - console.log(`≈ ${cellsMax} cells max (each = 1 clone + 1 runAgentic + 1 Docker judge)\n`) + console.log('=== SWE-bench prompt optimization with official GEPA ===') + console.log(`worker=${workerModel} reflect=${reflectModel} router=${routerBaseUrl}`) + console.log(`train=[${trainIds.join(', ')}] selection=[${selectionIds.join(', ')}] test=[${testIds.join(', ')}]`) + console.log(`maxEvaluations=${maxEvaluations} maxProposerCostUsd=${maxProposerCostUsd} innerTurns=${innerTurns} workerMaxTokens=${workerMaxTokens} runTool=${enableRun}`) + console.log(`runDir=${runDir}\n`) if (process.env.DRYRUN) { // Import + wiring smoke: prove every module resolves and the plan is well-formed // WITHOUT paying for a clone / model call / Docker judge. - console.log(`DRYRUN: imports OK (improve=${typeof improve}, gepaProposer=${typeof gepaProposer}, runAgentic=${typeof runAgentic}, refine=${typeof refine})`) + console.log(`DRYRUN: imports OK (improve=${typeof improve}, officialGepa=${typeof officialGepa}, runAgentic=${typeof runAgentic}, refine=${typeof refine})`) return } + const workerPricing = requiredTokenPricing(process.env, 'WORKER') + const optimizer = officialOptimizerModel({ + env: process.env, + model: reflectModel, + baseUrl: reflectBase, + apiKey: reflectKey, + maxCostUsd: maxProposerCostUsd, + maxOutputTokensPerRequest: reflectMaxTokens, + }) const { environment, adapter } = await createSweBenchEnvironment(allIds.length, { ids: allIds, enableRun }) const pool = await adapter.loadTasks({ ids: allIds, split: 'test' }) @@ -111,40 +133,44 @@ async function main(): Promise { }, } const t0 = Date.now() - const r = await runAgentic({ - surface: proxy, - task, - strategy: refine, - routerBaseUrl, - routerKey, + const paid = await ctx.cost.runPaidCall({ + channel: 'agent', + actor: 'swe-worker', model: workerModel, - maxTokens: workerMaxTokens, - innerTurns, - budget: budgetShots, + execute: () => + runAgentic({ + surface: proxy, + task, + strategy: refine, + routerBaseUrl, + routerKey, + model: workerModel, + maxTokens: workerMaxTokens, + innerTurns, + budget: budgetShots, + }), + receipt: (result) => { + const inputTokens = result.tokens.input ?? 0 + const outputTokens = result.tokens.output ?? 0 + const usageUnknown = inputTokens === 0 && outputTokens === 0 + return { + model: workerModel, + inputTokens, + outputTokens, + customTokenPricing: workerPricing, + ...(usageUnknown ? { usageUnknown: true } : {}), + } + }, }) - // Report REAL cost/tokens so the backend-integrity guard sees a real backend - // rather than a silent-zero stub. A glm-5.2 turn occasionally returns a real - // patch with an UNPOPULATED usage block (a router telemetry gap on some - // reasoning-model responses — NOT a stub: the cell made real tool calls and - // produced a patch). In that gap case report a nominal floor so the stub-guard - // (artifact + zero usage) cannot abort the whole campaign on a telemetry gap. - // The lift metric is judge-derived, so a floored count does not distort it; only - // cost accounting undercounts those few cells (disclosed). No-patch cells return - // null below and are skipped by the guard's own contract, so this floor only - // ever applies to a cell that genuinely produced a patch. + if (!paid.succeeded) throw paid.error + const r = paid.value const zeroUsage = (r.tokens.input ?? 0) === 0 && (r.tokens.output ?? 0) === 0 const hasPatch = capturedPatch.trim().length > 0 - ctx.cost.observe(zeroUsage && hasPatch ? Math.max(r.usd ?? 0, 0.0001) : r.usd ?? 0, workerModel) - ctx.cost.observeTokens( - zeroUsage && hasPatch - ? { input: Math.max(r.tokens.input ?? 0, 1), output: Math.max(r.tokens.output ?? 0, 1) } - : { input: r.tokens.input, output: r.tokens.output }, - ) const files = capturedPatch ? [...capturedPatch.matchAll(/^diff --git a\/(\S+)/gm)].map((m) => m[1]) : [] console.log( ` [agent] ${scenario.id} prompt=${promptText.length}c tools(l/r/e+/e-/run/run!)=${stats.list}/${stats.read}/${stats.edit_ok}/${stats.edit_fail}/${stats.run}/${stats.run_err} ` + - `patch=${capturedPatch.length}b files=[${files.join(', ') || 'none'}] tok=in:${r.tokens.input}/out:${r.tokens.output} usd=${r.usd} ${Math.round((Date.now() - t0) / 1000)}s` + - `${zeroUsage ? (hasPatch ? ' [zero-usage telemetry gap: patch kept, usage floored]' : ' [zero-usage cell: empty completion — scored as no-patch]') : ''}`, + `patch=${capturedPatch.length}b files=[${files.join(', ') || 'none'}] tok=in:${r.tokens.input}/out:${r.tokens.output} ${Math.round((Date.now() - t0) / 1000)}s` + + `${zeroUsage ? ' [provider usage unavailable]' : ''}`, ) // A cell with no patch produced NO artifact. Return null (not '') so the // backend-integrity guard's own contract (`artifact == null → skip`) applies: @@ -156,7 +182,7 @@ async function main(): Promise { // The judge: the OFFICIAL swebench Docker harness. Deterministic FAIL_TO_PASS + // PASS_TO_PASS → resolved 0/1. This is the held-out gate's scoring axis. - const judge: JudgeConfig = { + const judge: JudgeConfig = { name: 'swebench-docker', dimensions: [{ key: 'resolved', description: 'FAIL_TO_PASS + PASS_TO_PASS resolved by the official swebench Docker harness' }], async score({ artifact, scenario }) { @@ -177,53 +203,53 @@ async function main(): Promise { } const profile: AgentProfile = { name: 'swe-agent-glm46', prompt: { systemPrompt: SEED_PROMPT } } - const proposer = gepaProposer({ - llm: { baseUrl: routerBaseUrl, apiKey: routerKey }, - model: reflectModel, - target: 'the system prompt of a coding agent that fixes real GitHub bugs via list_files/read_file/edit_file tools', - maxTokens: reflectMaxTokens, - temperature: 0.7, - }) - - const scenarios: Scenario[] = allIds.map((id) => ({ id, kind: 'swe-bench-verified' })) - const holdoutScenarios: Scenario[] = holdoutIds.map((id) => ({ id, kind: 'swe-bench-verified' })) + const scenario = (id: string): Scenario => ({ id, kind: 'swe-bench-verified' }) - const out = await improve(profile, [], { + const out = await improve(profile, { surface: 'prompt', - gate: 'holdout', - generator: proposer, - scenarios, - judge, + method: officialGepa({ + objective: + 'Improve the system prompt of a coding agent that fixes real GitHub bugs with list_files, read_file, edit_file, and optional run tools.', + evaluationId, + background: + 'Return the complete system prompt. Preserve tool names and require evidence from repository files and tests.', + recipe: { + kind: 'engine', + run: { + engine: 'gepa', + maxEvaluations, + maxProposerCostUsd, + }, + }, + optimizer, + resume: 'if-compatible', + trustResumeState: true, + describeScenario: (item) => ({ prompt: byId.get(item.id)?.prompt ?? item.id }), + }), + trainScenarios: trainIds.map(scenario), + selectionScenarios: selectionIds.map(scenario), + testScenarios: testIds.map(scenario), + judges: [judge], agent, - // glm-5.2 occasionally returns a real patch with an unpopulated usage block - // (a router telemetry gap on some reasoning-model responses — NOT a stub: the - // cell made real tool calls and produced a patch). 'assert' would abort the - // whole campaign on such a cell; 'warn' logs it and continues. The lift metric - // (resolved) is judge-derived, so a missing token count does not distort it — - // only the cost accounting undercounts those cells, which is disclosed. expectUsage: 'warn', - budget: { generations, populationSize: population, holdoutScenarios, maxConcurrency, reps: 1 }, - llm: { baseUrl: routerBaseUrl, apiKey: routerKey, model: reflectModel }, + maxConcurrency, + reps: 1, + runDir, + optimizationRunOptions: { + expectUsage: 'warn', + maxConcurrency, + reps: 1, + }, }) - console.log('\n═══ RESULT ═══') - console.log(`gateDecision=${out.gateDecision} shipped=${out.shipped} lift=${out.lift}`) - console.log(`baseline holdout composite = ${out.raw.baseline.compositeMean}`) - console.log(`winner holdout composite = ${out.raw.winner.compositeMean}`) - console.log(`baseline per-scenario: ${JSON.stringify(out.raw.baseline.perScenario)}`) - console.log(`winner per-scenario: ${JSON.stringify(out.raw.winner.perScenario)}`) - if (out.raw.winner.label) console.log(`winner label : ${out.raw.winner.label}`) - if (out.raw.winner.rationale) console.log(`winner rationale: ${out.raw.winner.rationale}`) - - // Per-candidate verdicts on the train set (the "real swebench verdict per candidate"). - for (const gen of out.raw.generations ?? []) { - console.log(`\n── generation ${gen.record.generationIndex} candidates ──`) - for (const c of gen.record.candidates) { - const perScenario = (c as { scenarios?: Array<{ scenarioId: string; composite: number }> }).scenarios ?? [] - const detail = perScenario.map((s) => `${s.scenarioId}=${s.composite}`).join(' ') - console.log(` candidate ${c.surfaceHash.slice(0, 8)} composite=${c.composite}${c.label ? ` "${c.label}"` : ''} [${detail}]`) - } - } + assertCompleteCost('SWE-bench official GEPA run', out.cost) + console.log('\n=== RESULT ===') + console.log(`decision=${out.decision} lift=${out.lift} interval=[${out.liftInterval.low}, ${out.liftInterval.high}]`) + console.log(`baseline test composite=${out.raw.best.baselineComposite}`) + console.log(`winner test composite=${out.raw.best.winnerComposite}`) + console.log(`test scenarios=${JSON.stringify(out.raw.best.scenarioScores)}`) + console.log(`cost=${JSON.stringify(out.cost)}`) + console.log(`candidate prompt:\n${String(out.candidate.value).slice(0, 2000)}`) } main().catch((e) => { diff --git a/bench/src/trata-gepa.mts b/bench/src/trata-gepa.mts index d760408d..3946e810 100644 --- a/bench/src/trata-gepa.mts +++ b/bench/src/trata-gepa.mts @@ -1,22 +1,22 @@ /** - * trata-gepa — selfImprove (GEPA) outer loop for Trata hedge-bench. + * Official GEPA prompt optimization for Trata hedge-bench. * * The optimization surface is the system prompt given to the financial analyst * worker. GEPA reflects on which rubric themes were missed across training tasks - * and proposes improved system prompts — learning to instruct the model to + * and proposes improved system prompts, learning to instruct the model to * extract specific quantitative claims, cover multiple analytical themes, and - * cite named evidence. Gated on a frozen holdout. + * cite named evidence. Candidate selection and final testing use disjoint data. * * The surface evolves beyond a bare system instruction: GEPA naturally discovers * that it can add few-shot analytical patterns, calculation templates, and - * structured coverage checklists — effectively skill-creating without + * structured coverage checklists, effectively skill-creating without * hand-engineering. Set K_ROUNDS=2 to add a self-critique refine pass. * * Usage: * TRATA_BENCH_ROOT=/tmp/trata-hedge-bench \ * JUDGE_MODEL=gemini-2.5-flash WORKER_MODEL=deepseek-v4-flash \ * REFLECT_MODEL=gemini-2.5-pro \ - * TRAIN_N=70 HOLDOUT_N=32 GENS=2 POP=3 CONCURRENCY=8 \ + * TRAIN_N=70 SELECTION_N=16 TEST_N=16 MAX_EVALUATIONS=12 CONCURRENCY=8 \ * dotenvx run -f ~/company/devops/secrets/agent-state.env -- \ * pnpm exec tsx bench/src/trata-gepa.mts * @@ -26,9 +26,10 @@ * JUDGE_MODEL judge model in trata adapter (default gemini-2.5-flash) * REFLECT_MODEL GEPA reflection model (default gemini-2.5-pro) * TRAIN_N training tasks (default 70) - * HOLDOUT_N frozen holdout tasks (default 32) - * GENS optimization generations (default 2) - * POP candidates per generation (default 3) + * SELECTION_N optimizer selection tasks (default 16) + * TEST_N untouched final comparison tasks (default 16) + * MAX_EVALUATIONS GEPA callback evaluations (default 12) + * MAX_PROPOSER_COST_USD GEPA proposal budget (default 5) * REPS reps per scenario (default 1) * CONCURRENCY parallel worker slots (default 8) * K_ROUNDS 1=single-shot, 2=analysis+self-critique (default 1) @@ -36,30 +37,50 @@ * CORPUS path to write JSONL run records (optional) */ -import { selfImprove } from '@tangle-network/agent-eval/contract' -import type { CampaignResult, JudgeConfig, JudgeScore, Scenario } from '@tangle-network/agent-eval/campaign' -import { heldoutSignificance, inMemoryCampaignStorage, pairHoldout } from '@tangle-network/agent-eval/campaign' +import type { AgentProfile } from '@tangle-network/agent-interface' +import type { + DispatchContext, + JudgeConfig, + JudgeScore, + MutableSurface, + Scenario, +} from '@tangle-network/agent-eval/campaign' +import { improve, officialGepa } from '@tangle-network/agent-runtime' import { appendFileSync, writeFileSync } from 'node:fs' import { createTrataHedgeAdapter } from './benchmarks/trata-hedge' import type { BenchTask } from './benchmarks/types' +import { + assertCompleteCost, + officialOptimizerModel, + requiredTokenPricing, +} from './official-optimizer-config.mjs' interface TrataScenario extends Scenario { task: BenchTask } -interface DiagnosedFinding { - claim: string - severity: 'critical' | 'high' | 'medium' | 'low' | 'info' - area?: string - recommended_action?: string -} - function must(name: string): string { const v = process.env[name] if (!v) throw new Error(`env ${name} is required`) return v } +function positiveInteger(name: string, fallback: number): number { + const value = Number(process.env[name] ?? fallback) + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`env ${name} must be a positive integer`) + } + return value +} + +function positiveNumber(name: string, fallback: number): number { + const value = Number(process.env[name] ?? fallback) + if (!Number.isFinite(value) || value <= 0) { + throw new Error(`env ${name} must be a positive number`) + } + return value +} + // GEPA-optimised baseline — the best surface found across 9 runs (+8.6pp on holdout, 2 independent // confirmations). Future GEPA runs start from here; BASELINE_DIRECTIVE overrides if you want to // experiment from a different starting point. @@ -91,13 +112,15 @@ async function chatComplete( baseUrl: string, key: string, model: string, + maxTokens: number, messages: Array<{ role: string; content: string }>, + signal: AbortSignal, ): Promise<{ content: string; usage?: { input: number; output: number } }> { const res = await fetch(`${baseUrl}/chat/completions`, { method: 'POST', - signal: AbortSignal.timeout(180_000), + signal: AbortSignal.any([signal, AbortSignal.timeout(180_000)]), headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` }, - body: JSON.stringify({ model, temperature: 0, max_tokens: 4096, messages }), + body: JSON.stringify({ model, temperature: 0, max_tokens: maxTokens, messages }), }) if (!res.ok) throw new Error(`router ${res.status}: ${(await res.text()).slice(0, 300)}`) const j = (await res.json()) as { @@ -112,50 +135,14 @@ async function chatComplete( return { content, usage } } -function parseFindings(content: string): DiagnosedFinding[] { - // Balanced-bracket scan so `]` inside string values doesn't terminate early. - const startIdx = content.indexOf('[') - if (startIdx < 0) return [] - let depth = 0, inString = false, endIdx = -1 - for (let i = startIdx; i < content.length; i++) { - const ch = content[i] - if (inString) { - if (ch === '\\') { i++; continue } - if (ch === '"') inString = false - } else { - if (ch === '"') inString = true - else if (ch === '[' || ch === '{') depth++ - else if (ch === ']' || ch === '}') { - depth-- - if (depth === 0 && ch === ']') { endIdx = i; break } - } - } - } - if (endIdx < 0) return [] - const candidate = content.slice(startIdx, endIdx + 1) - let arr: unknown - try { - arr = JSON.parse(candidate) - } catch (e1) { - try { arr = JSON.parse(candidate.replace(/,(\s*[}\]])/g, '$1')) } - catch { console.error(`[trata-gepa] parseFindings failed: ${(e1 as Error).message} | head: ${candidate.slice(0, 120)}`); return [] } - } - if (!Array.isArray(arr)) return [] - const sev = new Set(['critical', 'high', 'medium', 'low', 'info']) - return arr - .filter( - (x): x is Record => - typeof x === 'object' && x !== null && typeof (x as { claim?: unknown }).claim === 'string', +async function main(): Promise { + if (process.env.DRYRUN) { + console.log( + `DRYRUN: imports OK (improve=${typeof improve}, officialGepa=${typeof officialGepa})`, ) - .map((x) => ({ - claim: String(x.claim), - severity: (sev.has(String(x.severity)) ? String(x.severity) : 'medium') as DiagnosedFinding['severity'], - area: x.area !== undefined ? String(x.area) : 'failure-mode', - recommended_action: x.recommended_action !== undefined ? String(x.recommended_action) : undefined, - })) -} + return + } -async function main(): Promise { const adapter = createTrataHedgeAdapter() await adapter.preflight() @@ -163,15 +150,47 @@ async function main(): Promise { const reflectModel = process.env.REFLECT_MODEL ?? 'deepseek-v4-flash' const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1' const routerKey = must('TANGLE_API_KEY') - const trainN = Number(process.env.TRAIN_N ?? 70) - const holdoutN = Number(process.env.HOLDOUT_N ?? 32) - const kRounds = Number(process.env.K_ROUNDS ?? 1) + const reflectBaseUrl = process.env.REFLECT_BASE ?? routerBaseUrl + const reflectKey = process.env.REFLECT_KEY ?? routerKey + const trainN = positiveInteger('TRAIN_N', 70) + const selectionN = positiveInteger('SELECTION_N', 16) + const testN = positiveInteger('TEST_N', 16) + const kRounds = positiveInteger('K_ROUNDS', 1) + const maxEvaluations = positiveInteger('MAX_EVALUATIONS', 12) + const maxProposerCostUsd = positiveNumber('MAX_PROPOSER_COST_USD', 5) + const maxConcurrency = positiveInteger('CONCURRENCY', 8) + const reps = positiveInteger('REPS', 1) + const workerMaxTokens = positiveInteger('MAX_TOKENS', 4096) + const reflectMaxTokens = positiveInteger('REFLECT_MAX_TOKENS', 8192) const corpusPath = process.env.CORPUS const baselineSurface = process.env.BASELINE_DIRECTIVE ?? DEFAULT_TRATA_SYSTEM + const runDir = process.env.RUN_DIR ?? '.runs/trata-official-gepa' + const evaluationId = [ + 'trata-hedge-prompt', + `worker=${model}`, + `workerEndpoint=${new URL(routerBaseUrl).origin}`, + `judge=${process.env.JUDGE_MODEL ?? 'adapter-default'}`, + `rounds=${kRounds}`, + `tokens=${workerMaxTokens}`, + ].join('|') + const workerPricing = requiredTokenPricing(process.env, 'WORKER') + const optimizer = officialOptimizerModel({ + env: process.env, + model: reflectModel, + baseUrl: reflectBaseUrl, + apiKey: reflectKey, + maxCostUsd: maxProposerCostUsd, + maxOutputTokensPerRequest: reflectMaxTokens, + }) - // Load all tasks and split deterministically. - // Hash-shuffle by task id so both splits carry the same difficulty mix. - const tasks = await adapter.loadTasks({ limit: trainN + holdoutN }) + // Hash-shuffle by task id so all three partitions carry the same difficulty mix. + const requestedTasks = trainN + selectionN + testN + const tasks = await adapter.loadTasks({ limit: requestedTasks }) + if (tasks.length !== requestedTasks) { + throw new Error( + `Trata returned ${tasks.length} tasks; ${requestedTasks} are required for exact train/selection/test partitions`, + ) + } const idHash = (s: string): number => { let h = 2166136261 for (let i = 0; i < s.length; i += 1) { @@ -180,45 +199,64 @@ async function main(): Promise { } return h >>> 0 } - tasks.sort((a, b) => idHash(a.id) - idHash(b.id)) - const train = tasks.slice(0, Math.min(trainN, tasks.length)) - const holdout = tasks.slice(train.length, train.length + Math.min(holdoutN, tasks.length - train.length)) + tasks.sort((a, b) => idHash(a.id) - idHash(b.id) || a.id.localeCompare(b.id)) + const train = tasks.slice(0, trainN) + const selection = tasks.slice(trainN, trainN + selectionN) + const test = tasks.slice(trainN + selectionN) const toScenario = (t: BenchTask): TrataScenario => ({ id: t.id, kind: 'trata-hedge', task: t }) console.log( - `[trata-gepa] worker=${model} reflect=${reflectModel} rounds=${kRounds} train=${train.length} holdout=${holdout.length}`, + `[trata-gepa] worker=${model} reflect=${reflectModel} rounds=${kRounds} train=${train.length} selection=${selection.length} test=${test.length}`, ) - // Domain seam: run the financial analyst worker under the candidate surface. + // Run the financial analyst worker under the candidate surface. // For K_ROUNDS=2, a second round asks the model to review its own coverage. - // Reports real token usage to ctx.cost (never fabricated). + // Every provider call reports its returned usage through the campaign ledger. + const runPaidCompletion = async ( + ctx: DispatchContext, + actor: string, + messages: Array<{ role: string; content: string }>, + ): Promise<{ content: string; usage?: { input: number; output: number } }> => { + const paid = await ctx.cost.runPaidCall({ + actor, + model, + execute: (signal) => + chatComplete(routerBaseUrl, routerKey, model, workerMaxTokens, messages, signal), + receipt: (result) => ({ + model, + inputTokens: result.usage?.input ?? 0, + outputTokens: result.usage?.output ?? 0, + customTokenPricing: workerPricing, + ...(result.usage ? {} : { usageUnknown: true }), + }), + }) + if (!paid.succeeded) throw paid.error + return paid.value + } + const runWithSurface = async ( - surface: string, + surface: MutableSurface, scenario: TrataScenario, - ctx: { - cost: { - observe(usd: number, source: string): void - observeTokens(u: { input: number; output: number }): void - } - }, + ctx: DispatchContext, ): Promise => { + if (typeof surface !== 'string') { + throw new Error('Trata prompt optimization requires a text surface') + } // Round 1: initial analysis under the candidate system prompt. - const r1 = await chatComplete(routerBaseUrl, routerKey, model, [ + const r1 = await runPaidCompletion(ctx, 'trata-worker-round-1', [ { role: 'system', content: surface }, { role: 'user', content: scenario.task.prompt }, ]) - if (r1.usage) ctx.cost.observeTokens(r1.usage) let answer = r1.content // Round 2 (optional): self-critique for rubric coverage. if (kRounds >= 2 && answer.trim()) { - const r2 = await chatComplete(routerBaseUrl, routerKey, model, [ + const r2 = await runPaidCompletion(ctx, 'trata-worker-round-2', [ { role: 'system', content: surface }, { role: 'user', content: scenario.task.prompt }, { role: 'assistant', content: answer }, { role: 'user', content: REFINE_INSTRUCTION }, ]) - if (r2.usage) ctx.cost.observeTokens(r2.usage) if (r2.content.trim()) answer = r2.content } @@ -253,173 +291,68 @@ async function main(): Promise { }, } - // EYES→HANDS: diagnose FAILED runs using themesMissed from judge detail. - // Trata's judge returns structured per-theme failure data — richer than the - // generic "wrong answer" signal that other benches feed into this hook. - const taskById = new Map(tasks.map((t) => [t.id, t])) - const analyzeGeneration = async (input: { - generation: number - runDir: string - candidates: Array<{ - surfaceHash: string - campaign: CampaignResult - composite: number - }> - history: unknown[] - }): Promise => { - interface FailureItem { - question: string - themesMissed: string[] - themesHit: string[] - answer: string - note: string - } - const failures = new Map() - for (const cand of input.candidates) { - for (const cell of cand.campaign.cells) { - const js = cell.judgeScores?.[judge.name] - if ((js?.composite ?? 0) >= 1) continue - if (failures.has(cell.scenarioId)) continue - const task = taskById.get(cell.scenarioId) - if (!task) continue - let themesMissed: string[] = [] - let themesHit: string[] = [] - try { - const d = JSON.parse(js?.notes ?? '{}') as { - themesMissed?: string[] - themesHit?: string[] - } - themesMissed = d.themesMissed ?? [] - themesHit = d.themesHit ?? [] - } catch { - // no structured detail available - } - failures.set(cell.scenarioId, { - question: task.prompt.slice(0, 800), - themesMissed, - themesHit, - answer: (typeof cell.artifact === 'string' ? cell.artifact : '').slice(-1200), - note: (js?.notes ?? '').slice(0, 200), - }) - } - } - const items = [...failures.values()].slice(0, 8) - if (items.length === 0) { - console.log(`[trata-gepa] gen ${input.generation}: 0 failures to diagnose`) - return [] - } - const user = items - .map( - (f, i) => - `### Failure ${i + 1}\nTASK (excerpt): ${f.question}\n` + - (f.themesMissed.length > 0 ? `MISSED THEMES: ${f.themesMissed.join(', ')}\n` : '') + - (f.themesHit.length > 0 ? `HIT THEMES: ${f.themesHit.join(', ')}\n` : '') + - `AGENT ANSWER (tail): ${f.answer}`, - ) - .join('\n\n') - const system = - 'You are a failure analyst for a financial analyst agent. The agent produces investment memos ' + - 'scored by a rubric with 4-6 analytical themes, each requiring specific quantitative claims. ' + - 'Below are FAILED runs showing which themes were missed and the agent\'s answer. ' + - 'Identify the COMMON failure patterns — e.g., generic statements without specific figures, ' + - 'missing peer comparisons, no explicit calculations, ignoring certain data file types. ' + - 'For each finding, recommend a CONCRETE change to the system instruction that would fix it. ' + - 'Return ONLY a JSON array (no prose): [{"claim","severity":"high"|"medium"|"low","area","recommended_action"}]. Max 6.' - let content: string | undefined - for (let attempt = 1; attempt <= 4; attempt += 1) { - try { - const r = await chatComplete(routerBaseUrl, routerKey, reflectModel, [ - { role: 'system', content: system }, - { role: 'user', content: user }, - ]) - content = r.content - break - } catch (err) { - const msg = (err as Error).message - if (attempt === 4) { - console.error(`[trata-gepa] analyzeGeneration failed gen ${input.generation}: ${msg}`) - return [] - } - await new Promise((r) => setTimeout(r, 1000 * 2 ** (attempt - 1))) - } - } - if (!content) return [] - const findings = parseFindings(content) - console.log(`[trata-gepa] gen ${input.generation}: ${items.length} failures → ${findings.length} findings`) - return findings + const profile: AgentProfile = { + name: 'trata-financial-analyst', + prompt: { systemPrompt: baselineSurface }, } - - const result = await selfImprove({ - agent: (surface, scenario, ctx) => runWithSurface(surface as string, scenario, ctx), - scenarios: train.map(toScenario), - judge, - baselineSurface, - budget: { - generations: Number(process.env.GENS ?? 2), - populationSize: Number(process.env.POP ?? 3), - maxConcurrency: Number(process.env.CONCURRENCY ?? 8), - reps: Number(process.env.REPS ?? 1), - promoteTopK: Number(process.env.TOPK ?? 1), - holdoutScenarios: holdout.map(toScenario), - }, - llm: { - baseUrl: routerBaseUrl, - apiKey: routerKey, - model: reflectModel, + const result = await improve(profile, { + surface: 'prompt', + method: officialGepa({ + objective: + 'Improve the complete system instruction for a financial analyst that writes evidence-backed investment memos.', + evaluationId, + background: + 'The judge awards partial credit for covering every requested analytical theme with specific quantitative claims, named peer comparisons, explicit calculations, source citations, and a decisive synthesis. Preserve the required ANALYSIS: prefix.', + recipe: { + kind: 'engine', + run: { + engine: 'gepa', + maxEvaluations, + maxProposerCostUsd, + }, + }, + optimizer, + resume: 'if-compatible', + trustResumeState: true, + describeScenario: (scenario) => ({ + id: scenario.id, + prompt: scenario.task.prompt, + }), + describeArtifact: (artifact) => ({ answer: artifact.slice(-4000) }), + }), + trainScenarios: train.map(toScenario), + selectionScenarios: selection.map(toScenario), + testScenarios: test.map(toScenario), + judges: [judge], + agent: runWithSurface, + expectUsage: 'warn', + maxConcurrency, + reps, + runDir, + optimizationRunOptions: { + expectUsage: 'warn', + maxConcurrency, + reps, }, - proposerTarget: - 'a FINANCIAL ANALYST SYSTEM INSTRUCTION: the directive given to an agent that produces an investment memo from embedded earnings call transcripts, SEC filings, financial statements, and investor presentations. ' + - 'The memo is scored by a rubric with 4-6 analytical themes, each requiring 2-4 specific analytical moves (quantitative claims, strategic conclusions, peer comparisons, or explicit calculations). ' + - 'A theme is "hit" only when the agent makes the SPECIFIC move — not just gestures at the theme. ' + - 'The directive must make the agent: (1) extract and cite specific numerical targets from management guidance, ' + - '(2) compute implied returns/IRRs when comparing capital allocation options, ' + - '(3) cover every distinct analytical theme with a dedicated paragraph, ' + - '(4) benchmark against named peers with specific metrics. The "ANALYSIS:" sentinel must start the response.', - mutationPrimitives: [ - 'instruct the agent to identify and verbatim-cite specific numerical targets in management guidance (earnings per share targets, margin percentages, growth rates, AUM figures) rather than paraphrasing in approximate terms', - 'instruct the agent to explicitly compute implied returns or IRRs when evaluating capital allocation trade-offs — show the arithmetic using the price levels and targets from the source data', - 'instruct the agent to structure the analysis with a clearly-labeled section for each distinct analytical theme (valuation, capital allocation, competitive dynamics, risk factors, etc.) so no major investment consideration is merged or omitted', - 'instruct the agent to compare the company against its NAMED sector peers with specific metrics (EV/EBITDA, P/E, margin differential, growth premium) cited from the peer financials files in the data', - ], - runDir: 'improve-prompt-trata-hedge', - storage: inMemoryCampaignStorage(), - autoOnPromote: 'none', - analyzeGeneration, }) + assertCompleteCost('Trata official GEPA run', result.cost) console.log('\n=== trata-gepa RESULT ===') - const improved = result.gateDecision === 'ship' - console.log(` baseline held-out mean: ${(result.baseline.compositeMean * 100).toFixed(1)}%`) - console.log(` winner held-out mean: ${(result.winner.compositeMean * 100).toFixed(1)}%`) - console.log(` ► held-out delta: ${(result.lift * 100).toFixed(1)} pp`) - console.log(` gate decision: ${result.gateDecision}`) - - try { - const cellsToMap = (cells: ReadonlyArray<{ scenarioId: string; judgeScores: Record }>) => { - const m = new Map>() - for (const c of cells) m.set(c.scenarioId, c.judgeScores) - return m - } - const baseMap = cellsToMap(result.raw.baselineOnHoldout.cells) - const winMap = cellsToMap(result.raw.winnerOnHoldout.cells) - const ids = new Set([...baseMap.keys()].filter((id) => winMap.has(id))) - const paired = pairHoldout(winMap, baseMap, ids, (s) => s.composite) - const sig = heldoutSignificance(paired) - console.log( - ` ► 95% CI (n=${sig.n}): [${(sig.bootstrap.low * 100).toFixed(1)}, ${(sig.bootstrap.high * 100).toFixed(1)}] pp · median ${(sig.bootstrap.median * 100).toFixed(1)}pp · significant=${sig.significant}`, - ) - if (!sig.significant) - console.log(' (CI spans 0 — scale n or generations before promoting)') - } catch (err) { - console.log(` (significance unavailable: ${(err instanceof Error ? err.message : String(err)).slice(0, 80)})`) - } + const improved = result.decision === 'ship' + console.log(` baseline test mean: ${(result.raw.best.baselineComposite * 100).toFixed(1)}%`) + console.log(` winner test mean: ${(result.raw.best.winnerComposite * 100).toFixed(1)}%`) + console.log(` test delta: ${(result.lift * 100).toFixed(1)} pp`) + console.log( + ` 95% interval: [${(result.liftInterval.low * 100).toFixed(1)}, ${(result.liftInterval.high * 100).toFixed(1)}] pp`, + ) + console.log(` decision: ${result.decision}`) + console.log(` cost: ${JSON.stringify(result.cost)}`) - const winnerSurface = result.winner.surface as string + const winnerSurface = String(result.candidate.value) if (improved) { console.log(`\n PROMOTED SYSTEM PROMPT:\n${winnerSurface}`) - if (result.winner.rationale) console.log(`\n rationale: ${result.winner.rationale}`) } else { - console.log(' kept baseline (gate did not promote)') + console.log(' kept baseline (the final-test interval did not clear zero)') console.log(`\n BEST CANDIDATE SURFACE (set as BASELINE_DIRECTIVE to seed next run):\n${winnerSurface}`) } try { diff --git a/examples/ablation-suite/gepa-driver-prompt.ts b/examples/ablation-suite/gepa-driver-prompt.ts index 82d8c878..d8ca3359 100644 --- a/examples/ablation-suite/gepa-driver-prompt.ts +++ b/examples/ablation-suite/gepa-driver-prompt.ts @@ -1,13 +1,9 @@ /** - * gepa-driver-prompt — GEPA-optimize the REAL supervisor driver prompt (the proposer) on TRAIN, - * executable-graded by the ACTUAL supervised resolve, frozen, held-out-certified, and return the winner. + * Optimize the supervisor driver prompt with GEPA's official engine. * - * This is the `optimize: 'gepa'` knob from the ablation board (ablation.ts), wired over the real - * substrate: agent-eval's `selfImprove` (the held-out-gated closed loop) driven by `gepaProposer` - * (the reflective prompt mutator). The surface under improvement is the driver/proposer standing - * prompt that `supervise()` runs as its brain — each candidate string becomes the driver profile's - * `systemPrompt`. So each candidate IS a candidate driver prompt, and its fitness IS the resolve it - * earns when it actually drives a `superviseSurface` run over the surface. + * The surface under improvement is the standing prompt that `supervise()` uses + * to steer workers. Agent-eval's complete method owns candidate search and + * selection. Runtime scores the selected candidate on a separate final set. * * The grading is EXECUTABLE, never an LLM judge: each candidate driver prompt runs a real supervised * rollout via `superviseSurface` (its harness-verified `resolved`/`score` come from the @@ -16,20 +12,24 @@ * check — there is no model in the scoring loop to flatter it. */ -import { - type DispatchContext, - gepaProposer, - type JudgeConfig, - type MutableSurface, - type Scenario, - selfImprove, +import type { + DispatchContext, + JudgeConfig, + MutableSurface, + Scenario, } from '@tangle-network/agent-eval/contract' +import type { AgentProfile } from '@tangle-network/agent-interface' +import { improve, officialGepa } from '@tangle-network/agent-runtime' import { type AgenticSurface, type AgenticTask, failuresAnalyst, superviseSurface, } from '@tangle-network/agent-runtime/loops' +import { + assertCompleteCost, + officialOptimizerModel, +} from '../../bench/src/official-optimizer-config.mjs' /** One TRAIN scenario: the coding task carried as the scenario's domain payload. The agent reads * `scenario.task` to run the supervised rollout; the judge reads the artifact the rollout produced. */ @@ -47,25 +47,15 @@ interface SupervisedOutcome { tokensOut: number } -/** The default reflection model — a model the Tangle router actually serves. The substrate default - * (`anthropic/claude-sonnet-4.6`) is NOT served by the router, so `gepaProposer` would fail every - * reflection call; callers should pass their own, but this keeps the zero-config path live. */ const defaultReflectionModel = 'gemini-2.5-pro' -/** The mutation levers offered to the reflective proposer — what a driver-prompt rewrite may change. - * These orient the model toward the kinds of edits that move a supervisor brain's effectiveness. */ -const driverMutationPrimitives = [ - 'sharpen what the driver must verify on a settled worker before it stops or re-spawns', - 'make the next-spawn instruction the driver composes more concrete and tool-grounded', - 'add an explicit verify-it-took step the driver must confirm before declaring done', - 'tighten the stop / re-spawn decision so it settles only when every required change is verified', -] - export async function optimizeDriverPrompt(opts: { surface: AgenticSurface tasks: (offset: number, n: number) => Promise trainOffset: number trainN: number + selectionN?: number + testN?: number baselinePrompt: string worker: { routerBaseUrl: string @@ -81,6 +71,12 @@ export async function optimizeDriverPrompt(opts: { * inference). Defaults to the worker's router + model when omitted. */ supervisorRouter?: { baseUrl: string; apiKey: string; model: string } reflectionModel?: string + /** Change when execution or scoring behavior changes outside the listed runtime knobs. */ + evaluationId?: string + maxEvaluations?: number + maxProposerCostUsd?: number + maxConcurrency?: number + runDir?: string }): Promise<{ systemPrompt: string; lift: number | undefined; shipped: boolean; usd: number }> { const { surface, worker } = opts @@ -93,14 +89,34 @@ export async function optimizeDriverPrompt(opts: { model: worker.model, } - // TRAIN scenarios — the disjoint training slice. `selfImprove` splits a held-out fraction off these - // for the gate, so the winner is certified on tasks the proposer never optimized against. - const trainTasks = await opts.tasks(opts.trainOffset, opts.trainN) - const scenarios: DriverPromptScenario[] = trainTasks.map((task) => ({ - id: task.id, - kind: 'coding', - task, - })) + const selectionN = opts.selectionN ?? opts.trainN + const testN = opts.testN ?? opts.trainN + const tasks = await opts.tasks(opts.trainOffset, opts.trainN + selectionN + testN) + const trainTasks = tasks.slice(0, opts.trainN) + const selectionTasks = tasks.slice(opts.trainN, opts.trainN + selectionN) + const testTasks = tasks.slice(opts.trainN + selectionN) + if ( + trainTasks.length !== opts.trainN || + selectionTasks.length !== selectionN || + testTasks.length !== testN + ) { + throw new Error('optimizeDriverPrompt: task source returned too few tasks for all partitions') + } + const scenarios = new Map( + tasks.map((task) => [ + task.id, + { + id: task.id, + kind: 'coding', + task, + } satisfies DriverPromptScenario, + ]), + ) + const mapScenarios = (items: AgenticTask[]): DriverPromptScenario[] => + items.map((task) => scenarios.get(task.id)!) + const trainScenarios = mapScenarios(trainTasks) + const selectionScenarios = mapScenarios(selectionTasks) + const testScenarios = mapScenarios(testTasks) // The agent under improvement: it receives the CURRENT candidate driver prompt (the surface string) // and runs a REAL supervised rollout with that prompt as the supervisor brain's standing instruction. @@ -179,36 +195,76 @@ export async function optimizeDriverPrompt(opts: { }), } - const reflectionModel = opts.reflectionModel ?? defaultReflectionModel - - const result = await selfImprove({ - agent, - scenarios, - judge, - baselineSurface: opts.baselinePrompt, - proposer: gepaProposer({ - llm: { baseUrl: worker.routerBaseUrl, apiKey: worker.routerKey }, - model: reflectionModel, - target: - 'the supervisor driver prompt (the standing brain instruction that spawns + steers workers)', - mutationPrimitives: driverMutationPrimitives, + const profile: AgentProfile = { + name: 'ablation-driver', + prompt: { systemPrompt: opts.baselinePrompt }, + } + const maxProposerCostUsd = opts.maxProposerCostUsd ?? 3 + const optimizer = officialOptimizerModel({ + env: process.env, + model: opts.reflectionModel ?? defaultReflectionModel, + baseUrl: supervisorRouter.baseUrl, + apiKey: supervisorRouter.apiKey, + maxCostUsd: maxProposerCostUsd, + maxOutputTokensPerRequest: Number(process.env.REFLECT_MAX_TOKENS ?? 8192), + }) + const result = await improve(profile, { + surface: 'prompt', + method: officialGepa({ + objective: + 'Improve the complete standing prompt used by a supervisor that spawns, steers, and verifies coding workers.', + evaluationId: + opts.evaluationId ?? + [ + 'ablation-driver-prompt', + `worker=${worker.model}`, + `supervisor=${supervisorRouter.model}`, + `tokens=${worker.maxTokens ?? 'default'}`, + `turns=${worker.innerTurns ?? 'default'}`, + `shots=${worker.budget ?? 'default'}`, + ].join('|'), + background: [ + 'Return only the complete supervisor prompt.', + 'The supervisor must verify settled workers, target subsequent workers at observed failures, and stop only after the task check passes.', + 'Do not change worker tools, models, or budgets.', + ].join(' '), + recipe: { + kind: 'engine', + run: { + engine: 'gepa', + maxEvaluations: opts.maxEvaluations ?? 8, + maxProposerCostUsd, + }, + }, + optimizer, + resume: 'if-compatible', + trustResumeState: true, + describeScenario: (scenario) => ({ + id: scenario.id, + systemPrompt: scenario.task.systemPrompt, + userPrompt: scenario.task.userPrompt, + }), }), - // One generation, two candidates, a third of TRAIN held out for the gate — the cheap proof shape; - // raise generations/populationSize for a deeper search once the cheap run is green. - budget: { generations: 1, populationSize: 2, holdoutFraction: 0.34 }, + trainScenarios, + selectionScenarios, + testScenarios, + judges: [judge], + agent, + runDir: opts.runDir ?? `.runs/ablation-driver-gepa-${opts.trainOffset}`, + maxConcurrency: opts.maxConcurrency ?? 1, + reps: 1, + optimizationRunOptions: { + maxConcurrency: opts.maxConcurrency ?? 1, + reps: 1, + }, }) - // The winner surface is the promoted driver prompt. A `CodeSurface` winner is impossible here (the - // baseline + every mutation is a string), but guard the type so the return stays a clean string. - const winner = result.winner.surface - const systemPrompt = typeof winner === 'string' ? winner : opts.baselinePrompt - + assertCompleteCost('optimizeDriverPrompt', result.cost) return { - systemPrompt, + systemPrompt: + typeof result.candidate.value === 'string' ? result.candidate.value : opts.baselinePrompt, lift: result.lift, - shipped: result.gateDecision === 'ship', - // The TRAIN-side optimization cost (baseline + every generation) — counted into the arm's $ so the - // cost-aware ablation never hides the price of GEPA behind the held-out run alone. - usd: result.totalCostUsd, + shipped: result.decision === 'ship', + usd: result.cost.totalCostUsd, } } From dfb42bc1b85c5f1cc767efc48b5fc169bf1d6e54 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 24 Jul 2026 01:24:52 -0600 Subject: [PATCH 02/14] fix(bench): harden GEPA seat provenance --- bench/src/swe-arena/gepa-seat.mts | 428 ++++++++++++++++++------- bench/src/swe-arena/gepa-seat.test.mts | 377 ++++++++++++++++------ 2 files changed, 604 insertions(+), 201 deletions(-) diff --git a/bench/src/swe-arena/gepa-seat.mts b/bench/src/swe-arena/gepa-seat.mts index 9a203126..330a07f3 100644 --- a/bench/src/swe-arena/gepa-seat.mts +++ b/bench/src/swe-arena/gepa-seat.mts @@ -1,7 +1,6 @@ /** - * GEN-6 GEPA proposer seat — agent-eval's external-GEPA adapter - * (`gepaOptimizationMethod`, tangle-network/agent-eval PRs #408/#409, - * main@58a28aa) wired as ONE seat in the swe-arena proposer fan-out. + * GEN-6 GEPA proposer seat using agent-eval's official + * `gepaOptimizationMethod` as one author in the swe-arena fan-out. * * Two-tier evaluator, the critical shape: * @@ -31,26 +30,29 @@ * (`gepa_bridge.py` `_validate_input`: `if "testSet" in value ... raise`). * This module never mentions holdout instances to begin with. * - * RUNTIME SEAMS (both fail LOUD at provenance time, t=0, mirroring the codex - * seat's auth check — a dead seat cannot be silently skipped mid-run): - * - Node: the installed @tangle-network/agent-eval must export - * `gepaOptimizationMethod` (0.123.x predates it) — `loadGepaMethodFactory` - * throws with the exact upgrade instruction otherwise. + * RUNTIME SEAM (fails at provenance time, before a candidate slot is used): * - Python: `agent_eval_rpc.gepa_bridge` + a GEPA build with * `optimize_anything`/`OptimizeAnythingConfig` must import — * `probeGepaRuntime` throws with the pip install instruction otherwise. + * The TypeScript adapter is a pinned package dependency and imported directly. */ -import { createHash } from 'node:crypto' -import { mkdir, readFile, writeFile } from 'node:fs/promises' +import { createHash, randomUUID } from 'node:crypto' +import { mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises' import { dirname, join } from 'node:path' -import type { - DispatchContext, - JudgeConfig, - MutableSurface, - OptimizationMethod, - OptimizationMethodInput, - Scenario, +import { + type DispatchContext, + createRunCostLedger, + fsCampaignStorage, + type GepaOptimizationMethodConfig, + type GepaOptimizationRecipe, + gepaOptimizationMethod, + type JudgeConfig, + type MutableSurface, + type OptimizationMethod, + type OptimizationMethodInput, + type OptimizationMethodProvenance, + type Scenario, } from '@tangle-network/agent-eval/campaign' import { ACTIVATION_PREDICATE_RELPATH, type ActivationPredicate } from './activation.mts' import { changeSpaceViolations, type OuterLoopConfig } from './outer-loop.mts' @@ -110,25 +112,18 @@ export function validateGepaSeat(spec: ProposerSpec): asserts spec is GepaSeatSp } // --------------------------------------------------------------------------- -// Recipe — the adapter's own shape, mirrored structurally (the installed -// agent-eval may predate the export; see loadGepaMethodFactory). +// Recipe. // --------------------------------------------------------------------------- -export interface GepaEngineRun { - engine: string - maxEvaluations: number - maxProposerCostUsd: number - engineConfig?: Record -} - -export type GepaOptimizationRecipe = - | { kind: 'engine'; run: GepaEngineRun } - | { kind: 'best-of-then-continue'; explore: readonly GepaEngineRun[]; continueWith: GepaEngineRun } +export type GepaSeatRecipe = Extract< + GepaOptimizationRecipe, + { kind: 'engine' | 'omni' } +> /** Build the bounded recipe for a seat. The TOTAL inner-evaluation budget is - * exactly `maxMetricCalls` — the adapter's local callback enforces the sum + * exactly `maxMetricCalls`. The adapter's local callback enforces the sum * of per-run limits, and the seat's own dispatch wrapper re-enforces it. */ -export function recipeForSeat(spec: GepaSeatSpec): GepaOptimizationRecipe { +export function recipeForSeat(spec: GepaSeatSpec): GepaSeatRecipe { const calls = spec.maxMetricCalls ?? DEFAULT_MAX_METRIC_CALLS const cost = spec.maxProposerCostUsd ?? DEFAULT_MAX_PROPOSER_COST_USD if (spec.engine === 'gepa') { @@ -145,13 +140,13 @@ export function recipeForSeat(spec: GepaSeatSpec): GepaOptimizationRecipe { maxProposerCostUsd: perRunCost, })) return { - kind: 'best-of-then-continue', + kind: 'omni', explore, continueWith: { engine: 'gepa', maxEvaluations: continueCalls, maxProposerCostUsd: perRunCost }, } } -export function recipeEvaluationBudget(recipe: GepaOptimizationRecipe): number { +export function recipeEvaluationBudget(recipe: GepaSeatRecipe): number { const runs = recipe.kind === 'engine' ? [recipe.run] : [...recipe.explore, recipe.continueWith] return runs.reduce((sum, run) => sum + run.maxEvaluations, 0) } @@ -228,55 +223,17 @@ export function innerSmokeJudge(): JudgeConfig { } // --------------------------------------------------------------------------- -// Runtime seams — Node adapter export + Python bridge, both loud. +// Optional Python runtime. // --------------------------------------------------------------------------- -export const GEPA_ADAPTER_UPGRADE_HINT = - "the installed @tangle-network/agent-eval does not export gepaOptimizationMethod — " + - 'upgrade to a release containing tangle-network/agent-eval PRs #408/#409 (merged at main@58a28aa; ' + - 'first release after 0.123.5), then reinstall bench deps' - export const GEPA_PYTHON_INSTALL_HINT = - "install the optional Python bridge: pip install 'agent-eval-rpc[gepa]' " + - '(the extra pins the GEPA source commit providing optimize_anything/OptimizeAnythingConfig; ' + - 'published gepa<=0.1.4 does not contain the multi-engine API — see agent-eval docs/campaign-proposers.md)' - -/** Adapter config, mirrored structurally from agent-eval's - * `GepaOptimizationMethodConfig` (src/campaign/gepa-optimization-method.ts). */ -export interface GepaMethodConfig { - name?: string - recipe: GepaOptimizationRecipe - objective: string - background?: string - maxCandidateChars?: number - timeoutMs?: number - describeScenario?: (scenario: GepaSeatScenario) => unknown - runner?: { command?: string; args?: readonly string[]; cwd?: string; env?: NodeJS.ProcessEnv } -} + 'install the optional Python bridge with `python -m pip install agent-eval-rpc`, ' + + "then install the GEPA source revision pinned in agent-eval's Python client README" export type GepaMethodFactory = ( - config: GepaMethodConfig, + config: GepaOptimizationMethodConfig, ) => OptimizationMethod -export type CampaignModuleImport = () => Promise> - -const defaultImportCampaign: CampaignModuleImport = () => - import('@tangle-network/agent-eval/campaign') as Promise> - -/** Resolve the adapter factory from the installed agent-eval, or throw the - * exact upgrade instruction. Checked at provenance time (t=0) AND at author - * time, so a stale install can never silently skip the seat. */ -export async function loadGepaMethodFactory( - importCampaign: CampaignModuleImport = defaultImportCampaign, -): Promise { - const mod = await importCampaign() - const factory = mod['gepaOptimizationMethod'] - if (typeof factory !== 'function') { - throw new Error(`gepa seat: ${GEPA_ADAPTER_UPGRADE_HINT}`) - } - return factory as GepaMethodFactory -} - export type ProbeExec = ( command: string, args: string[], @@ -338,6 +295,11 @@ export interface GepaInnerCall { wallS: number } +type OptimizationPackageSource = OptimizationMethodProvenance['source'] +type OptimizationModuleSource = NonNullable[number] +type OptimizationPythonRuntime = NonNullable +type OptimizationTokenUsage = NonNullable + export interface GepaSeatInnerRun { seat: string engine: GepaEngineName @@ -347,28 +309,246 @@ export interface GepaSeatInnerRun { innerCallCount: number innerScores: GepaInnerCall[] bestComposite: number | null - adapterReportedCostUsd: number | null - adapterCostAccountingComplete: boolean + source: OptimizationPackageSource & { revision: string; sourceSha256: string } + bridge: OptimizationPackageSource & { sourceSha256: string } + modules: OptimizationModuleSource[] + python: OptimizationPythonRuntime + runId: string + compatibleRunId: string + resumed: boolean + evaluationCount: number + tokenUsage: OptimizationTokenUsage + artifactDir: string + totalCostUsd: number + accountingComplete: boolean + incompleteReasons: string[] durationMs: number } export const PROPOSER_PROVENANCE_FILENAME = 'proposer-provenance.json' +export const GEPA_INNER_RUNS_DIRNAME = 'gepa-inner-runs' -/** Merge one seat run's inner-call record into `proposer-provenance.json` - * under `gepaInnerRuns` (additive; the t=0 capture record is preserved). */ -export async function recordGepaSeatInnerRun(outDir: string, run: GepaSeatInnerRun): Promise { - const path = join(outDir, PROPOSER_PROVENANCE_FILENAME) - let record: Record = {} +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function errorCode(error: unknown): string | undefined { + return isRecord(error) && typeof error.code === 'string' ? error.code : undefined +} + +async function readJsonObject(path: string): Promise> { + const raw = await readFile(path, 'utf8') + let value: unknown try { - record = JSON.parse(await readFile(path, 'utf8')) as Record - } catch { - // No capture record yet (unit-test or crash-before-write): still persist. - } - const runs = Array.isArray(record.gepaInnerRuns) ? (record.gepaInnerRuns as unknown[]) : [] - runs.push(run) - record.gepaInnerRuns = runs - await mkdir(dirname(path), { recursive: true }) - await writeFile(path, JSON.stringify(record, null, 2)) + value = JSON.parse(raw) + } catch (error) { + throw new Error(`gepa seat: malformed JSON in existing provenance file ${path}`, { cause: error }) + } + if (!isRecord(value)) { + throw new Error(`gepa seat: existing provenance file ${path} must contain a JSON object`) + } + return value +} + +async function validateExistingRunRecords(outDir: string): Promise { + const launchRecordPath = join(outDir, PROPOSER_PROVENANCE_FILENAME) + try { + await readJsonObject(launchRecordPath) + } catch (error) { + if (errorCode(error) !== 'ENOENT') throw error + } + + const recordsDir = join(outDir, GEPA_INNER_RUNS_DIRNAME) + let entries + try { + entries = await readdir(recordsDir, { withFileTypes: true }) + } catch (error) { + if (errorCode(error) === 'ENOENT') return + throw error + } + for (const entry of entries) { + if (entry.isFile() && entry.name.endsWith('.json')) { + await readJsonObject(join(recordsDir, entry.name)) + } + } +} + +/** Persist one immutable record without mutating the shared launch record. */ +export async function recordGepaSeatInnerRun(outDir: string, run: GepaSeatInnerRun): Promise { + await validateExistingRunRecords(outDir) + const recordsDir = join(outDir, GEPA_INNER_RUNS_DIRNAME) + await mkdir(recordsDir, { recursive: true }) + const identity = createHash('sha256') + .update(JSON.stringify({ seat: run.seat, generation: run.generation, runId: run.runId })) + .digest('hex') + .slice(0, 16) + const nonce = randomUUID() + const filename = `${identity}-${nonce}.json` + const finalPath = join(recordsDir, filename) + const temporaryPath = join(recordsDir, `.${filename}.tmp`) + try { + await writeFile(temporaryPath, JSON.stringify(run, null, 2), { flag: 'wx' }) + await rename(temporaryPath, finalPath) + } finally { + await rm(temporaryPath, { force: true }) + } + return finalPath +} + +function requiredText(value: unknown, label: string): string { + if (typeof value !== 'string' || value.length === 0 || value !== value.trim()) { + throw new Error(`gepa seat: optimizer result omitted ${label}`) + } + return value +} + +function requiredSha256(value: unknown, label: string): string { + const hash = requiredText(value, label) + if (!/^[0-9a-f]{64}$/.test(hash)) { + throw new Error(`gepa seat: optimizer result returned invalid ${label}`) + } + return hash +} + +function requiredCount(value: unknown, label: string): number { + if (!Number.isSafeInteger(value) || (value as number) < 0) { + throw new Error(`gepa seat: optimizer result returned invalid ${label}`) + } + return value as number +} + +function completeProvenance( + provenance: OptimizationMethodProvenance | undefined, + seatName: string, +): Pick< + GepaSeatInnerRun, + | 'source' + | 'bridge' + | 'modules' + | 'python' + | 'runId' + | 'compatibleRunId' + | 'resumed' + | 'evaluationCount' + | 'tokenUsage' + | 'artifactDir' +> { + const label = `gepa seat '${seatName}'` + if (provenance === undefined) { + throw new Error(`${label}: optimizer result omitted provenance`) + } + if ( + provenance.source.kind !== 'package' || + provenance.source.evidence !== 'observed' || + requiredText(provenance.source.package, 'source.package') !== 'gepa' + ) { + throw new Error(`${label}: optimizer result returned invalid source package`) + } + requiredText(provenance.source.version, 'source.version') + const sourceRevision = requiredText(provenance.source.revision, 'source.revision') + const sourceSha256 = requiredSha256(provenance.source.sourceSha256, 'source.sourceSha256') + if (provenance.bridge === undefined) { + throw new Error(`${label}: optimizer result omitted bridge provenance`) + } + if ( + provenance.bridge.kind !== 'package' || + provenance.bridge.evidence !== 'observed' || + requiredText(provenance.bridge.package, 'bridge.package') !== 'agent-eval-rpc' + ) { + throw new Error(`${label}: optimizer result returned invalid bridge package`) + } + requiredText(provenance.bridge.version, 'bridge.version') + const bridgeSha256 = requiredSha256(provenance.bridge.sourceSha256, 'bridge.sourceSha256') + if (provenance.modules === undefined) { + throw new Error(`${label}: optimizer result omitted module provenance`) + } + const modules = provenance.modules.map((module, index) => ({ + module: requiredText(module.module, `modules[${index}].module`), + sourceSha256: requiredSha256(module.sourceSha256, `modules[${index}].sourceSha256`), + })) + if (provenance.python === undefined) { + throw new Error(`${label}: optimizer result omitted Python provenance`) + } + const python = { + implementation: requiredText(provenance.python.implementation, 'python.implementation'), + version: requiredText(provenance.python.version, 'python.version'), + } + if (provenance.compatibleRunId === undefined) { + throw new Error(`${label}: optimizer result omitted compatibleRunId`) + } + if (provenance.tokenUsage === undefined) { + throw new Error(`${label}: optimizer result omitted token usage`) + } + const tokenUsage = { + inputTokens: requiredCount(provenance.tokenUsage.inputTokens, 'tokenUsage.inputTokens'), + ...(provenance.tokenUsage.cachedInputTokens === undefined + ? {} + : { + cachedInputTokens: requiredCount( + provenance.tokenUsage.cachedInputTokens, + 'tokenUsage.cachedInputTokens', + ), + }), + ...(provenance.tokenUsage.cacheWriteInputTokens === undefined + ? {} + : { + cacheWriteInputTokens: requiredCount( + provenance.tokenUsage.cacheWriteInputTokens, + 'tokenUsage.cacheWriteInputTokens', + ), + }), + outputTokens: requiredCount(provenance.tokenUsage.outputTokens, 'tokenUsage.outputTokens'), + ...(provenance.tokenUsage.reasoningTokens === undefined + ? {} + : { + reasoningTokens: requiredCount( + provenance.tokenUsage.reasoningTokens, + 'tokenUsage.reasoningTokens', + ), + }), + totalTokens: requiredCount(provenance.tokenUsage.totalTokens, 'tokenUsage.totalTokens'), + calls: requiredCount(provenance.tokenUsage.calls, 'tokenUsage.calls'), + } + if (tokenUsage.totalTokens !== tokenUsage.inputTokens + tokenUsage.outputTokens) { + throw new Error(`${label}: optimizer result returned inconsistent token usage`) + } + return { + source: { ...provenance.source, revision: sourceRevision, sourceSha256 }, + bridge: { ...provenance.bridge, sourceSha256: bridgeSha256 }, + modules, + python, + runId: requiredText(provenance.runId, 'runId'), + compatibleRunId: requiredText(provenance.compatibleRunId, 'compatibleRunId'), + resumed: provenance.resumed, + evaluationCount: requiredCount(provenance.evaluationCount, 'evaluationCount'), + tokenUsage, + artifactDir: requiredText(provenance.artifactDir, 'artifactDir'), + } +} + +function assertCompleteCost( + cost: { totalCostUsd: number; accountingComplete: boolean; incompleteReasons: string[] }, + seatName: string, +): void { + if (!Number.isFinite(cost.totalCostUsd) || cost.totalCostUsd < 0) { + throw new Error(`gepa seat '${seatName}': optimizer returned invalid total cost`) + } + if ( + !Array.isArray(cost.incompleteReasons) || + cost.incompleteReasons.some( + (reason) => typeof reason !== 'string' || reason.length === 0 || reason !== reason.trim(), + ) + ) { + throw new Error(`gepa seat '${seatName}': optimizer returned invalid incomplete reasons`) + } + if (cost.accountingComplete !== (cost.incompleteReasons.length === 0)) { + throw new Error(`gepa seat '${seatName}': optimizer returned inconsistent cost accounting`) + } + if (!cost.accountingComplete) { + throw new Error( + `gepa seat '${seatName}': cost accounting is incomplete: ${cost.incompleteReasons.join('; ') || 'no reason provided'}`, + ) + } } // --------------------------------------------------------------------------- @@ -414,7 +594,7 @@ export interface GepaSeatDeps { * split's public set; re-asserted here fail-closed). */ smokeInstanceId: string scoreSplit: Pick | null - /** Test seam. Default: checked dynamic import of the installed adapter. */ + /** Test seam. Default: agent-eval's official GEPA method. */ methodFactory?: GepaMethodFactory log?: (msg: string) => void } @@ -451,6 +631,13 @@ export function gepaSeatAuthor(config: OuterLoopConfig, deps: GepaSeatDeps): Aut assertNoPrivateLeak(objective + background + JSON.stringify([...scenarios.train, ...scenarios.selection]), deps.scoreSplit, 'bridge payload') + const storage = fsCampaignStorage() + const costLedger = + args.costLedger ?? + createRunCostLedger({ + storage, + runDir: `${runDir}/cost`, + }) const innerScores: GepaInnerCall[] = [] const dispatchWithSurface = async ( surface: MutableSurface, @@ -469,7 +656,7 @@ export function gepaSeatAuthor(config: OuterLoopConfig, deps: GepaSeatDeps): Aut scratchPath: args.worktreePath, generation, proposer: spec, - ...(args.costLedger ? { costLedger: args.costLedger } : {}), + costLedger, }) if (deps.scoreSplit !== null && deps.scoreSplit.privateInstances.includes(verdict.iid)) { throw new Error( @@ -494,15 +681,24 @@ export function gepaSeatAuthor(config: OuterLoopConfig, deps: GepaSeatDeps): Aut return verdict } - const factory = deps.methodFactory ?? (await loadGepaMethodFactory()) + const factory: GepaMethodFactory = + deps.methodFactory ?? gepaOptimizationMethod const method = factory({ name: `gepa-seat:${spec.name}`, recipe, objective, + evaluationId: [ + 'swe-arena-gepa-seat', + `smoke=${deps.smokeInstanceId}`, + 'judge=gepa-inner-smoke', + `dispatchTimeoutMs=${config.dispatchTimeoutMs}`, + ].join('|'), background, describeScenario: (scenario) => ({ id: scenario.id }), // Ceiling, not expectation: every inner call is a real arm cell. timeoutMs: budget * config.dispatchTimeoutMs, + resume: 'if-compatible', + trustResumeState: true, runner: { command: spec.python ?? DEFAULT_GEPA_PYTHON }, }) @@ -515,6 +711,7 @@ export function gepaSeatAuthor(config: OuterLoopConfig, deps: GepaSeatDeps): Aut runDir, seed: config.round * 1000 + generation, runOptions: { + storage, maxConcurrency: 1, dispatchTimeoutMs: config.dispatchTimeoutMs, labeledStore: 'off', @@ -522,31 +719,42 @@ export function gepaSeatAuthor(config: OuterLoopConfig, deps: GepaSeatDeps): Aut expectUsage: 'off', resumable: false, }, + costLedger, } const started = Date.now() const result = await method.optimize(input) + let innerRun: GepaSeatInnerRun + try { + innerRun = { + seat: spec.name, + engine: spec.engine, + surface: spec.surface, + generation, + budget, + innerCallCount: innerScores.length, + innerScores, + bestComposite: innerScores.length > 0 ? Math.max(...innerScores.map((s) => s.composite)) : null, + ...completeProvenance(result.provenance, spec.name), + totalCostUsd: result.cost.totalCostUsd, + accountingComplete: result.cost.accountingComplete, + incompleteReasons: [...result.cost.incompleteReasons], + durationMs: Date.now() - started, + } + await writeFile(join(runDir, 'inner-provenance.json'), JSON.stringify(innerRun, null, 2)) + await recordGepaSeatInnerRun(config.outDir, innerRun) + assertCompleteCost(result.cost, spec.name) + } catch (error) { + await writeFile(surfacePath, seed) + throw error + } + const winner = result.winnerSurface if (typeof winner !== 'string' || winner.trim().length === 0) { + await writeFile(surfacePath, seed) throw new Error(`gepa seat '${spec.name}': adapter returned a non-string winner surface`) } - const innerRun: GepaSeatInnerRun = { - seat: spec.name, - engine: spec.engine, - surface: spec.surface, - generation, - budget, - innerCallCount: innerScores.length, - innerScores, - bestComposite: innerScores.length > 0 ? Math.max(...innerScores.map((s) => s.composite)) : null, - adapterReportedCostUsd: result.cost.totalCostUsd, - adapterCostAccountingComplete: result.cost.accountingComplete, - durationMs: Date.now() - started, - } - await writeFile(join(runDir, 'inner-provenance.json'), JSON.stringify(innerRun, null, 2)) - await recordGepaSeatInnerRun(config.outDir, innerRun) - if (winner === seed) { // Restore the seed (the last inner call may have left another candidate) // and decline the slot — an unchanged surface has no candidate diff. diff --git a/bench/src/swe-arena/gepa-seat.test.mts b/bench/src/swe-arena/gepa-seat.test.mts index 15c68d46..c3e351ef 100644 --- a/bench/src/swe-arena/gepa-seat.test.mts +++ b/bench/src/swe-arena/gepa-seat.test.mts @@ -1,20 +1,22 @@ import { spawnSync } from 'node:child_process' import { existsSync } from 'node:fs' -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' -import type { DispatchContext } from '@tangle-network/agent-eval/campaign' +import type { + DispatchContext, + OptimizationMethodProvenance, +} from '@tangle-network/agent-eval/campaign' import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { ACTIVATION_PREDICATE_RELPATH, parseActivationPredicate } from './activation.mts' import { DEFAULT_GEPA_PYTHON, DEFAULT_MAX_METRIC_CALLS, - GEPA_ADAPTER_UPGRADE_HINT, + GEPA_INNER_RUNS_DIRNAME, gepaBridgeScenarios, innerSmokeComposite, innerSmokeJudge, isGepaSeat, - loadGepaMethodFactory, mechanicalActivationPredicate, probeGepaRuntime, recipeEvaluationBudget, @@ -85,10 +87,10 @@ describe('recipeForSeat', () => { expect(recipeEvaluationBudget(recipe)).toBe(DEFAULT_MAX_METRIC_CALLS) }) - it("'omni' is GEPA's best-of-then-continue shape and the four bounded runs sum EXACTLY to the budget", () => { + it("'omni' uses the official recipe and its four bounded runs preserve the budget", () => { const recipe = recipeForSeat(seat({ engine: 'omni', maxMetricCalls: 10 }) as GepaSeatSpec) - expect(recipe.kind).toBe('best-of-then-continue') - if (recipe.kind !== 'best-of-then-continue') throw new Error('unreachable') + expect(recipe.kind).toBe('omni') + if (recipe.kind !== 'omni') throw new Error('unreachable') expect(recipe.explore.map((r) => r.engine)).toEqual(['gepa', 'autoresearch', 'meta_harness']) expect(recipe.continueWith.engine).toBe('gepa') expect(recipeEvaluationBudget(recipe)).toBe(10) @@ -169,21 +171,9 @@ describe('inner smoke score', () => { }) // --------------------------------------------------------------------------- -// Runtime seams: loud fails with exact instructions. +// Optional Python runtime: loud failures with exact instructions. // --------------------------------------------------------------------------- -describe('loadGepaMethodFactory', () => { - it('throws the upgrade instruction when the installed agent-eval predates the adapter', async () => { - await expect(loadGepaMethodFactory(async () => ({}))).rejects.toThrow(/#408/) - await expect(loadGepaMethodFactory(async () => ({}))).rejects.toThrow(/gepaOptimizationMethod/) - }) - - it('returns the export when present', async () => { - const factory = (() => ({})) as unknown as GepaMethodFactory - await expect(loadGepaMethodFactory(async () => ({ gepaOptimizationMethod: factory }))).resolves.toBe(factory) - }) -}) - describe('probeGepaRuntime', () => { const execFailingOn = (failFragment: string, stderr: string): ProbeExec => @@ -196,7 +186,9 @@ describe('probeGepaRuntime', () => { it('fails loud with pip install instructions when the bridge module is missing', async () => { const exec = execFailingOn('agent_eval_rpc.gepa_bridge', "ModuleNotFoundError: No module named 'agent_eval_rpc'") - await expect(probeGepaRuntime('python3', exec, 'gepa-author')).rejects.toThrow(/pip install 'agent-eval-rpc\[gepa\]'/) + await expect(probeGepaRuntime('python3', exec, 'gepa-author')).rejects.toThrow( + /pip install agent-eval-rpc/, + ) await expect(probeGepaRuntime('python3', exec, 'gepa-author')).rejects.toThrow(/not installed/) }) @@ -234,15 +226,10 @@ describe('captureProposerProvenance with a gepa seat', () => { } return { code: 0, stdout: 'source', stderr: '' } } - const withAdapter = async (): Promise> => ({ - gepaOptimizationMethod: () => ({}), - }) - it('records engine, surface, gepa version, bridge module, and the python runtime as harnessVersion', async () => { const record = await captureProposerProvenance([{ name: 'claude-author', harness: 'claude' }, seat()], { exec: okExec, readSettingsModel: () => 'settings-model', - importCampaign: withAdapter, }) const gepa = record.proposers.find((p) => p.name === 'gepa-author')! expect(gepa).toMatchObject({ @@ -267,15 +254,9 @@ describe('captureProposerProvenance with a gepa seat', () => { args.join(' ').includes('gepa_bridge') ? { code: 1, stdout: '', stderr: 'ModuleNotFoundError' } : { code: 0, stdout: 'Python 3.12.3', stderr: '' } - await expect( - captureProposerProvenance([seat()], { exec, importCampaign: withAdapter }), - ).rejects.toThrow(/pip install 'agent-eval-rpc\[gepa\]'/) - }) - - it('fails LOUD at t=0 when the installed agent-eval predates the adapter export', async () => { - await expect( - captureProposerProvenance([seat()], { exec: okExec, importCampaign: async () => ({}) }), - ).rejects.toThrow(GEPA_ADAPTER_UPGRADE_HINT.slice(0, 40)) + await expect(captureProposerProvenance([seat()], { exec })).rejects.toThrow( + /pip install agent-eval-rpc/, + ) }) }) @@ -309,34 +290,128 @@ describe('mechanicalActivationPredicate', () => { }) describe('recordGepaSeatInnerRun', () => { - it('appends to gepaInnerRuns while preserving the t=0 capture record', async () => { + const sourceHash = 'a'.repeat(64) + const bridgeHash = 'b'.repeat(64) + const moduleHash = 'c'.repeat(64) + const provenance = (runId: string): OptimizationMethodProvenance => ({ + source: { + kind: 'package', + evidence: 'observed', + package: 'gepa', + version: '0.1.4', + sourceUrl: 'https://github.com/gepa-ai/gepa.git', + revision: 'f919db0a622e2e9f9204779b81fe00cc1b2d808f', + sourceSha256: sourceHash, + }, + bridge: { + kind: 'package', + evidence: 'observed', + package: 'agent-eval-rpc', + version: '0.126.0', + sourceSha256: bridgeHash, + }, + modules: [{ module: 'example.engine', sourceSha256: moduleHash }], + python: { implementation: 'CPython', version: '3.12.3' }, + runId, + compatibleRunId: 'compatible-run', + resumed: false, + evaluationCount: 3, + tokenUsage: { + inputTokens: 120, + cachedInputTokens: 20, + outputTokens: 30, + reasoningTokens: 10, + totalTokens: 150, + calls: 2, + }, + artifactDir: `/tmp/${runId}`, + }) + const run = (runId: string, over: Partial = {}): GepaSeatInnerRun => { + const result = provenance(runId) + if ( + result.source.revision === undefined || + result.source.sourceSha256 === undefined || + result.bridge?.sourceSha256 === undefined || + result.modules === undefined || + result.python === undefined || + result.compatibleRunId === undefined || + result.tokenUsage === undefined + ) { + throw new Error('invalid test provenance') + } + return { + seat: 'gepa-author', + engine: 'gepa', + surface: SURFACE, + generation: 0, + budget: 10, + innerCallCount: 2, + innerScores: [], + bestComposite: 1, + source: { + ...result.source, + revision: result.source.revision, + sourceSha256: result.source.sourceSha256, + }, + bridge: { ...result.bridge, sourceSha256: result.bridge.sourceSha256 }, + modules: result.modules, + python: result.python, + runId: result.runId, + compatibleRunId: result.compatibleRunId, + resumed: result.resumed, + evaluationCount: result.evaluationCount, + tokenUsage: result.tokenUsage, + artifactDir: result.artifactDir, + totalCostUsd: 0.25, + accountingComplete: true, + incompleteReasons: [], + durationMs: 5, + ...over, + } + } + + it('writes collision-free immutable records in parallel and preserves the launch record', async () => { const dir = await mkdtemp(join(tmpdir(), 'gepa-prov-')) try { - await writeFile(join(dir, 'proposer-provenance.json'), JSON.stringify({ schema: 'swe-arena.proposer-provenance.v1', proposers: [] })) - const run: GepaSeatInnerRun = { - seat: 'gepa-author', - engine: 'gepa', - surface: SURFACE, - generation: 0, - budget: 10, - innerCallCount: 2, - innerScores: [], - bestComposite: 1, - adapterReportedCostUsd: 0, - adapterCostAccountingComplete: false, - durationMs: 5, - } - await recordGepaSeatInnerRun(dir, run) - await recordGepaSeatInnerRun(dir, { ...run, generation: 1 }) - const record = JSON.parse(await readFile(join(dir, 'proposer-provenance.json'), 'utf8')) - expect(record.schema).toBe('swe-arena.proposer-provenance.v1') - expect(record.gepaInnerRuns).toHaveLength(2) - expect(record.gepaInnerRuns[0]).toMatchObject({ seat: 'gepa-author', innerCallCount: 2 }) - expect(record.gepaInnerRuns[1]).toMatchObject({ generation: 1 }) + const launchRecord = JSON.stringify({ capturedAt: '2026-07-24T00:00:00.000Z', proposers: [] }, null, 2) + await writeFile(join(dir, 'proposer-provenance.json'), launchRecord) + const paths = await Promise.all( + Array.from({ length: 24 }, (_, index) => + recordGepaSeatInnerRun(dir, run(`run-${index}`, { generation: index })), + ), + ) + expect(new Set(paths).size).toBe(24) + expect(await readFile(join(dir, 'proposer-provenance.json'), 'utf8')).toBe(launchRecord) + + const names = (await readdir(join(dir, GEPA_INNER_RUNS_DIRNAME))).filter((name) => name.endsWith('.json')) + expect(names).toHaveLength(24) + const records = await Promise.all( + names.map(async (name) => JSON.parse(await readFile(join(dir, GEPA_INNER_RUNS_DIRNAME, name), 'utf8'))), + ) + expect(new Set(records.map((record) => record.runId))).toEqual( + new Set(Array.from({ length: 24 }, (_, index) => `run-${index}`)), + ) + expect(records.every((record) => record.source.sourceSha256 === sourceHash)).toBe(true) } finally { await rm(dir, { recursive: true, force: true }) } }) + + it('fails on malformed existing launch or run data', async () => { + const launchDir = await mkdtemp(join(tmpdir(), 'gepa-prov-bad-launch-')) + const runDir = await mkdtemp(join(tmpdir(), 'gepa-prov-bad-run-')) + try { + await writeFile(join(launchDir, 'proposer-provenance.json'), '{not json') + await expect(recordGepaSeatInnerRun(launchDir, run('new-run'))).rejects.toThrow(/malformed JSON/) + + await mkdir(join(runDir, GEPA_INNER_RUNS_DIRNAME), { recursive: true }) + await writeFile(join(runDir, GEPA_INNER_RUNS_DIRNAME, 'broken.json'), '[]') + await expect(recordGepaSeatInnerRun(runDir, run('new-run'))).rejects.toThrow(/must contain a JSON object/) + } finally { + await rm(launchDir, { recursive: true, force: true }) + await rm(runDir, { recursive: true, force: true }) + } + }) }) // --------------------------------------------------------------------------- @@ -345,6 +420,45 @@ describe('recordGepaSeatInnerRun', () => { const fakeCtx = {} as unknown as DispatchContext +const fullProvenance = ( + runId = 'gepa-run', + over: Partial = {}, +): OptimizationMethodProvenance => ({ + source: { + kind: 'package', + evidence: 'observed', + package: 'gepa', + version: '0.1.4', + sourceUrl: 'https://github.com/gepa-ai/gepa.git', + revision: 'f919db0a622e2e9f9204779b81fe00cc1b2d808f', + sourceSha256: '1'.repeat(64), + }, + bridge: { + kind: 'package', + evidence: 'observed', + package: 'agent-eval-rpc', + version: '0.126.0', + sourceSha256: '2'.repeat(64), + }, + modules: [{ module: 'custom_gepa_engines', sourceSha256: '3'.repeat(64) }], + python: { implementation: 'CPython', version: '3.12.3' }, + runId, + compatibleRunId: 'compatible-gepa-run', + resumed: false, + evaluationCount: 3, + tokenUsage: { + inputTokens: 100, + cachedInputTokens: 10, + cacheWriteInputTokens: 5, + outputTokens: 25, + reasoningTokens: 8, + totalTokens: 125, + calls: 2, + }, + artifactDir: `/tmp/${runId}`, + ...over, +}) + /** Mimics the adapter's loop: score the seed and each provided candidate via * the seat's dispatch + judge, return the best-scoring candidate — exactly * the contract gepaOptimizationMethod fulfills through the Python bridge. */ @@ -365,8 +479,9 @@ const fakeGepaFactory = } return { winnerSurface: best.surface, - cost: { totalCostUsd: 0, accountingComplete: false, incompleteReasons: ['fake factory'] }, + cost: { totalCostUsd: 0.125, accountingComplete: true, incompleteReasons: [] }, durationMs: 1, + provenance: fullProvenance(), } }, } @@ -440,10 +555,10 @@ describe('fanOutLoopsGenerator with the gepa seat', () => { it('materializes each candidate into the scratch surface, applies the winner through the normal prefilter path, and records provenance', async () => { const WINNER = `${SEED}Always run the neighboring test file before finalizing.\n` const LOSER = `${SEED}delete all tests\n` - const seen: Array<{ content: string; scratch: string }> = [] - const smokeRunner: SmokeRunner = async ({ scratchPath }) => { + const seen: Array<{ content: string; scratch: string; hasCostLedger: boolean }> = [] + const smokeRunner: SmokeRunner = async ({ scratchPath, costLedger }) => { const content = await readFile(join(scratchPath, SURFACE), 'utf8') - seen.push({ content, scratch: scratchPath }) + seen.push({ content, scratch: scratchPath, hasCostLedger: costLedger !== undefined }) // The winner candidate resolves; the seed gets verify-pass only; the // loser gets nothing — exercising resolve-dominates + tiebreak. if (content === WINNER) return smokeVerdict({ resolved: true, verifyPass: true }) @@ -468,6 +583,7 @@ describe('fanOutLoopsGenerator with the gepa seat', () => { expect(seen).toHaveLength(4) for (const call of seen) expect(call.scratch).not.toBe(driverWt) expect(seen.map((c) => c.content)).toEqual([SEED, LOSER, WINNER, WINNER]) + expect(seen.slice(0, 3).every((call) => call.hasCostLedger)).toBe(true) // The winner landed on the driver worktree with the mechanical predicate. expect(await readFile(join(driverWt, SURFACE), 'utf8')).toBe(WINNER) const predicate = parseActivationPredicate(await readFile(join(driverWt, ACTIVATION_PREDICATE_RELPATH), 'utf8')) @@ -479,16 +595,99 @@ describe('fanOutLoopsGenerator with the gepa seat', () => { .filter(Boolean) expect(changed.sort()).toEqual([ACTIVATION_PREDICATE_RELPATH, SURFACE].sort()) // Budget threaded into the adapter recipe. - expect(observed.config).toMatchObject({ recipe: { kind: 'engine', run: { engine: 'gepa', maxEvaluations: 5 } } }) - // Inner-run provenance: per-seat file + the merged proposer-provenance.json. + expect(observed.config).toMatchObject({ + evaluationId: + `swe-arena-gepa-seat|smoke=astropy__astropy-13033|judge=gepa-inner-smoke|dispatchTimeoutMs=${config.dispatchTimeoutMs}`, + recipe: { kind: 'engine', run: { engine: 'gepa', maxEvaluations: 5 } }, + resume: 'if-compatible', + trustResumeState: true, + }) + // Inner-run provenance: the seat-local file plus one immutable shared record. const inner = JSON.parse(await readFile(join(outDir, 'gepa-seat', 'gen0-gepa-author', 'inner-provenance.json'), 'utf8')) - expect(inner).toMatchObject({ seat: 'gepa-author', engine: 'gepa', budget: 5, innerCallCount: 3, bestComposite: 1.25 }) + expect(inner).toMatchObject({ + seat: 'gepa-author', + engine: 'gepa', + budget: 5, + innerCallCount: 3, + bestComposite: 1.25, + source: { + package: 'gepa', + version: '0.1.4', + revision: 'f919db0a622e2e9f9204779b81fe00cc1b2d808f', + sourceSha256: '1'.repeat(64), + }, + bridge: { + package: 'agent-eval-rpc', + version: '0.126.0', + sourceSha256: '2'.repeat(64), + }, + modules: [{ module: 'custom_gepa_engines', sourceSha256: '3'.repeat(64) }], + python: { implementation: 'CPython', version: '3.12.3' }, + runId: 'gepa-run', + compatibleRunId: 'compatible-gepa-run', + resumed: false, + evaluationCount: 3, + tokenUsage: { + inputTokens: 100, + cachedInputTokens: 10, + cacheWriteInputTokens: 5, + outputTokens: 25, + reasoningTokens: 8, + totalTokens: 125, + calls: 2, + }, + artifactDir: '/tmp/gepa-run', + totalCostUsd: 0.125, + accountingComplete: true, + incompleteReasons: [], + }) expect(inner.innerScores.map((s: { composite: number }) => s.composite)).toEqual([0.25, 0, 1.25]) - const merged = JSON.parse(await readFile(join(outDir, 'proposer-provenance.json'), 'utf8')) - expect(merged.gepaInnerRuns).toHaveLength(1) + const records = await readdir(join(outDir, GEPA_INNER_RUNS_DIRNAME)) + expect(records.filter((name) => name.endsWith('.json'))).toHaveLength(1) + expect(JSON.parse(await readFile(join(outDir, GEPA_INNER_RUNS_DIRNAME, records[0]!), 'utf8'))).toEqual(inner) expect(gen.drainPrefilterKills()).toEqual([]) }) + it('records but rejects a winner when cost accounting is incomplete', async () => { + const WINNER = `${SEED}Always run the neighboring test file before finalizing.\n` + const incomplete: GepaMethodFactory = () => ({ + name: 'incomplete-cost', + async optimize(input) { + await input.dispatchWithSurface(WINNER, input.trainScenarios[0]!, fakeCtx) + return { + winnerSurface: WINNER, + cost: { + totalCostUsd: 0.25, + accountingComplete: false, + incompleteReasons: ['optimizer model receipt missing'], + }, + durationMs: 1, + provenance: fullProvenance('incomplete-run', { evaluationCount: 1 }), + } + }, + }) + const gen = fanOutLoopsGenerator(baseConfig([seat()]), { + smokeRunner: async () => smokeVerdict({ resolved: true }), + smokeInstanceId: 'astropy__astropy-13033', + scoreSplit: null, + gepaMethodFactory: incomplete, + }) + + await expect(gen.generate(generatorArgs(0))).rejects.toThrow( + /cost accounting is incomplete: optimizer model receipt missing/, + ) + expect(await readFile(join(driverWt, SURFACE), 'utf8')).toBe(SEED) + const inner = JSON.parse( + await readFile(join(outDir, 'gepa-seat', 'gen0-gepa-author', 'inner-provenance.json'), 'utf8'), + ) + expect(inner).toMatchObject({ + runId: 'incomplete-run', + totalCostUsd: 0.25, + accountingComplete: false, + incompleteReasons: ['optimizer model receipt missing'], + }) + }) + it('enforces the inner-call budget cap fail-closed', async () => { const runaway: GepaMethodFactory = () => ({ name: 'runaway', @@ -500,6 +699,7 @@ describe('fanOutLoopsGenerator with the gepa seat', () => { winnerSurface: SEED, cost: { totalCostUsd: 0, accountingComplete: false, incompleteReasons: [] }, durationMs: 1, + provenance: fullProvenance('runaway'), } }, }) @@ -538,9 +738,8 @@ describe('fanOutLoopsGenerator with the gepa seat', () => { }) // --------------------------------------------------------------------------- -// Integration: ONE real Node→Python→score roundtrip through the installed -// adapter + bridge. Skips with an exact reason when either runtime is absent -// (this is the same condition the t=0 provenance capture enforces loud). +// Integration: one real Node-to-Python-to-score roundtrip through the installed +// bridge. The TypeScript adapter is a compile-time package dependency. // --------------------------------------------------------------------------- const pythonBridgeReady = (): { ok: boolean; reason: string } => { @@ -555,18 +754,10 @@ const pythonBridgeReady = (): { ok: boolean; reason: string } => { } describe('integration: real adapter roundtrip', () => { - it('runs one inner smoke roundtrip through gepaOptimizationMethod when the full runtime is installed', async (ctx) => { - const campaign = (await import('@tangle-network/agent-eval/campaign')) as Record - const missing: string[] = [] - if (typeof campaign.gepaOptimizationMethod !== 'function') { - missing.push( - 'installed @tangle-network/agent-eval lacks gepaOptimizationMethod (needs a release after 0.123.5 containing PRs #408/#409)', - ) - } + it('runs the bridge but rejects its unmetered winner when the full runtime is installed', async (ctx) => { const python = pythonBridgeReady() - if (!python.ok) missing.push(python.reason) - if (missing.length > 0) { - ctx.skip(`skip-with-reason: ${missing.join('; ')}`) + if (!python.ok) { + ctx.skip(`skip-with-reason: ${python.reason}`) return } @@ -612,21 +803,25 @@ describe('integration: real adapter roundtrip', () => { scoreSplit: null, }, ) - const result = await gen.generate({ - worktreePath: driverWt, - report: undefined, - findings: [], - maxShots: 1, - signal: new AbortController().signal, - generation: 0, - candidateIndex: 0, - }) - // The bridge ran: inner provenance must show >= 1 scored callback. + await expect( + gen.generate({ + worktreePath: driverWt, + report: undefined, + findings: [], + maxShots: 1, + signal: new AbortController().signal, + generation: 0, + candidateIndex: 0, + }), + ).rejects.toThrow(/cost accounting is incomplete/) + // The bridge ran and its result was retained even though activation was refused. const inner = JSON.parse( await readFile(join(outDir, 'gepa-seat', 'gen0-gepa-author', 'inner-provenance.json'), 'utf8'), ) expect(inner.innerCallCount).toBeGreaterThanOrEqual(1) - expect(typeof result.applied).toBe('boolean') + expect(inner.accountingComplete).toBe(false) + expect(inner.incompleteReasons.length).toBeGreaterThan(0) + expect((await runOk('git', ['-C', driverWt, 'status', '--porcelain'])).stdout.trim()).toBe('') } finally { await rm(outDir, { recursive: true, force: true }) await rm(loopsRepo, { recursive: true, force: true }) From 8b3905980dec16bfd3307a306d996e3db90e7e94 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 24 Jul 2026 01:42:28 -0600 Subject: [PATCH 03/14] feat(optimization): adopt official optimizer methods --- README.md | 104 +- bench/README.md | 7 + bench/gen6-config.json | 146 -- bench/package.json | 4 +- bench/scripts/trata-hedge/README.md | 11 +- bench/src/benchmarks/humaneval.test.mts | 4 +- bench/src/live-improve-campaign-mbpp.mts | 641 ------- bench/src/live-improve-campaign.mts | 500 ------ bench/src/official-optimizer-config.mts | 14 +- bench/src/official-optimizer-config.test.mts | 88 + bench/src/profiles.ts | 4 +- bench/src/swe-arena/activation.mts | 5 +- bench/src/swe-arena/activation.test.mts | 23 +- bench/src/swe-arena/gepa-seat.mts | 37 +- bench/src/swe-arena/gepa-seat.test.mts | 131 +- bench/src/swe-arena/lineage-record.mts | 164 -- bench/src/swe-arena/lineage-record.test.mts | 115 -- bench/src/swe-arena/outer-loop.mts | 78 +- bench/src/swe-arena/proposer-fanout.mts | 41 +- bench/src/swe-arena/proposer-fanout.test.mts | 1 - bench/src/swe-arena/proposer-provenance.mts | 27 +- bench/src/swe-code-improve.mts | 47 +- docs/agent-managed-compute/architecture.md | 9 +- docs/api/index.md | 1566 +++++++++++------ docs/api/intelligence.md | 14 +- docs/api/mcp.md | 58 +- docs/api/primitive-catalog.md | 134 +- docs/architecture-interpretations.md | 4 +- docs/architecture.md | 7 +- docs/canonical-api.md | 13 +- docs/design.md | 2 +- docs/design/structural-rollout-integration.md | 2 +- docs/intelligence-sdk.md | 4 +- docs/learning-flywheel.md | 9 +- docs/research/simplification-plan.md | 17 +- examples/README.md | 4 +- examples/ablation-suite/ablation.ts | 7 +- examples/improve/README.md | 53 +- examples/improve/improve.ts | 66 +- examples/intelligence-recommend/README.md | 29 +- .../intelligence-recommend.ts | 29 +- examples/self-improving-coder/README.md | 2 +- .../self-improving-loop.ts | 16 +- package.json | 6 +- pnpm-lock.yaml | 46 +- skills/build-with-agent-runtime/SKILL.md | 22 +- src/improvement/agentic-generator.ts | 5 +- src/improvement/campaign-otlp.test.ts | 231 --- src/improvement/campaign-otlp.ts | 265 --- src/improvement/improve.test.ts | 1321 +++++--------- src/improvement/improve.ts | 696 +++++--- src/improvement/improvement-driver.ts | 23 +- src/improvement/index.ts | 55 +- src/improvement/official-optimizers.test.ts | 403 +++++ src/improvement/official-optimizers.ts | 204 +++ src/improvement/profile-diff-proposer.test.ts | 132 -- src/improvement/profile-diff-proposer.ts | 98 -- src/improvement/raw-trace-distiller.ts | 5 +- src/improvement/reflective-generator.ts | 5 +- src/improvement/rollout-policy.test.ts | 287 +-- src/improvement/rollout-policy.ts | 138 +- src/index.ts | 7 +- src/intelligence/improvement-cycle.ts | 17 +- src/loop-runner.ts | 19 +- src/mcp/memory-server.ts | 3 +- src/runtime/structural-rollout.ts | 2 +- src/runtime/supervise-surface.ts | 2 +- tests/improve.test.ts | 205 --- tests/improvement-cycle.test.ts | 82 +- tests/improvement-driver.test.ts | 3 +- tests/loop-runner.test.ts | 5 + tests/profile-improvement-stack.test.ts | 64 +- 72 files changed, 3382 insertions(+), 5206 deletions(-) delete mode 100644 bench/gen6-config.json delete mode 100644 bench/src/live-improve-campaign-mbpp.mts delete mode 100644 bench/src/live-improve-campaign.mts create mode 100644 bench/src/official-optimizer-config.test.mts delete mode 100644 bench/src/swe-arena/lineage-record.mts delete mode 100644 bench/src/swe-arena/lineage-record.test.mts delete mode 100644 src/improvement/campaign-otlp.test.ts delete mode 100644 src/improvement/campaign-otlp.ts create mode 100644 src/improvement/official-optimizers.test.ts create mode 100644 src/improvement/official-optimizers.ts delete mode 100644 src/improvement/profile-diff-proposer.test.ts delete mode 100644 src/improvement/profile-diff-proposer.ts delete mode 100644 tests/improve.test.ts diff --git a/README.md b/README.md index cac72415..1571c735 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ The fully annotated version of this loop, with every seam explained, is [`exampl |---|---| | Run a **chat turn** for a production product agent | `handleChatTurn(...)` | | Have one agent **supervise a team of agents** toward a goal | `supervise(profile, task, opts)` | -| **Improve** an agent and prove the gain on fresh tasks | `improve(profile, findings, opts)` | +| **Improve** an agent and prove the gain on fresh tasks | `improve(profile, opts)` | | Produce a measured knowledge-base candidate with agents and checks | `runKnowledgeImprovementJob(...)` | | Evaluate or train the same agent on **PrimeIntellect** | `createPrimeIntellectPackage(...)` | @@ -107,34 +107,92 @@ const result = await supervise( ### Improve an agent -`improve` optimizes one part of an agent and returns a detached winner plus a decision. The decision is `ship` only when the candidate beats the current agent on tasks it never practiced on. -It accepts prompt, skill document, curated memory, tool, MCP, hook, subagent, whole-profile, and code surfaces through one call. -Prompt, skill-document, and memory optimization have built-in generators; structured profile surfaces take an explicit generator, and code runs from isolated incumbent and candidate checkouts. -Workflow and rollout-policy files use the code surface so the measured winner is an exact patch that can be sealed and executed; JSON parameter sweeps use agent-eval's `parameterSweepProposer` instead of a runtime-specific optimizer. +`improve` runs one complete `OptimizationMethod` against one profile field. +The method owns candidate generation and selection. +Runtime keeps the final test set out of the method, scores the baseline and selected candidate on it, and returns `ship` only when the paired confidence interval clears `minimumLift`. +The profile is never changed. ```ts -import { improve } from '@tangle-network/agent-runtime' +import { improve, officialGepa } from '@tangle-network/agent-runtime' -const { candidate, decision, lift } = await improve(baseProfile, findings, { +const result = await improve(baseProfile, { surface: 'prompt', - gate: 'holdout', - scenarios, - judge, + method: officialGepa({ + objective: 'Improve the complete support-agent prompt.', + evaluationId: 'support-prompt', + recipe: { + kind: 'engine', + run: { + engine: 'gepa', + maxEvaluations: 40, + maxProposerCostUsd: 10, + }, + }, + resume: 'if-compatible', + describeScenario: ({ input }) => ({ input }), + }), + findings, + trainScenarios, + selectionScenarios, + testScenarios, + judges: [judge], agent, + runDir: '.runs/support-prompt', }) -if (decision === 'ship') console.log({ candidate, lift }) +if (result.decision === 'ship') { + console.log(result.candidate.profile, result.liftInterval) +} ``` -Skill and curated-memory candidates are exact profile changes, not free-floating text. -Name one inline skill through `skills.resourceName`; curated memory uses `profile.resources.instructions`. -Both require `profile.resources.failOnError: true` so an unsupported resource cannot silently disappear. +`officialGepa(...)` delegates the complete search to GEPA's upstream Optimize Anything API through agent-eval. +Pass one explicit `engine`, `sequential`, `adaptive-sequential`, `best-of`, `vote`, or `omni` recipe. +`evaluationId` identifies the dispatch, judges, models, and scoring logic. +Change it whenever any of those behaviors change. +With `resume: 'if-compatible'`, agent-eval resumes only when the saved run identity matches the candidate, recipe, data, optimizer settings, runner, and evaluation ID. +Use `resume: 'required'` to fail when no matching run exists. +`result.provenance` reports the upstream package, run ID, resume status, evaluation count, and artifact directory. +There is no local fallback. +Install its optional Python process before using it: + +```bash +python -m pip install agent-eval-rpc +python -m pip install "gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f" +``` + +Use `officialSkillOpt(...)` for Microsoft's SkillOpt: + +```bash +python -m pip install agent-eval-rpc +python -m pip install "skillopt @ git+https://github.com/microsoft/SkillOpt.git@61735e3922efc2b90c6d6cab561e62e98452ca90" +``` + +The source pins are deliberate. +The published GEPA wheel lacks Optimize Anything, and the published SkillOpt wheel lacks files required by `ReflACTTrainer`. +SkillOpt requires `optimizer: { model, baseUrl, apiKey, budget }`. +GEPA accepts the same optional `optimizer` block for its standard reflection engine. +Agent Eval proxies those model calls, enforces the nested budget, and records their cost. + +SkillOpt accepts one text surface. +GEPA accepts text or named components. +Any complete method from `@tangle-network/agent-eval` uses the same call. +For a skill, set `surface: 'skills'` and `skills.resourceName`. +For the complete profile, set `surface: 'agent-profile'`. +To optimize several named profile fields together, also provide `profileComponents.read` and `profileComponents.apply`. +Tools, MCP, hooks, subagents, curated instructions, and rollout policy are also exact profile coordinates. +Runtime does not choose an optimizer for them. + +Code is the exception. +It uses Runtime's isolated git worktrees and coding-agent candidate execution: ```ts -const skillResult = await improve(baseProfile, findings, { - surface: 'skills', - skills: { resourceName: 'incident-response' }, - scenarios, judge, agent, +const result = await improve(baseProfile, { + surface: 'code', + code: { repoRoot, baseRef, generator }, + scenarios, + judge, + agent, + budget, }) ``` @@ -155,7 +213,15 @@ const result = await proposeAgentImprovement({ runId, profile: liveProfile, analysis, - improvement: { surface: 'prompt', scenarios, judge, agent }, + improvement: { + surface: 'prompt', + method, + trainScenarios, + selectionScenarios, + testScenarios, + judges: [judge], + agent, + }, buildExperiment: ({ improvement }) => buildExperimentMaterial({ baseline, diff --git a/bench/README.md b/bench/README.md index c9514f7d..e50c95f9 100644 --- a/bench/README.md +++ b/bench/README.md @@ -26,6 +26,13 @@ pnpm install # tsx + link parent The judge needs only Docker; workers need a model key (Tangle router `TANGLE_API_KEY`, or a direct provider). +Live optimizer scripts require explicit token prices so cost records cannot be guessed. +Use the `REFLECT_` prefix for the general benchmark scripts and `GEPA_OPTIMIZER_` for the SWE arena seat. +For either prefix, set `INPUT_USD_PER_MILLION`, `CACHED_INPUT_USD_PER_MILLION`, `CACHE_WRITE_USD_PER_MILLION`, and `OUTPUT_USD_PER_MILLION`. +The SWE arena seat reads its model from `GEPA_OPTIMIZER_MODEL`, its URL from `GEPA_OPTIMIZER_BASE_URL`, and its key from `GEPA_OPTIMIZER_API_KEY`. +It falls back to the configured driver model, `ROUTER_BASE`, and `TANGLE_API_KEY`. +Missing prices fail before an optimizer model call. + Retain every official per-test log and report before the temporary evaluator directory is removed: ```ts diff --git a/bench/gen6-config.json b/bench/gen6-config.json deleted file mode 100644 index 3c3ca5b1..00000000 --- a/bench/gen6-config.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "_draft": "gen-6 DRAFT — do NOT launch. Adds the gepa-author engine seat (agent-eval external-GEPA adapter, PRs #408/#409). Launch preconditions: (1) an @tangle-network/agent-eval release exporting gepaOptimizationMethod (>0.123.5), (2) pip install 'agent-eval-rpc[gepa]' for the Python bridge, (3) operator approval. Provenance capture fails loud at t=0 on either missing runtime.", - "round": 4, - "instances": [ - "astropy__astropy-13033", - "django__django-11532", - "matplotlib__matplotlib-20826", - "pydata__xarray-4687", - "pytest-dev__pytest-6197", - "sphinx-doc__sphinx-9658" - ], - "holdoutInstances": [ - "astropy__astropy-14182", - "django__django-12774", - "django__django-14140", - "scikit-learn__scikit-learn-14894", - "sympy__sympy-20438", - "pytest-dev__pytest-7236" - ], - "loopsRepo": "/home/drew/code/loops", - "loopsBaseRef": "feat/supervisor-evidence-flow", - "armName": "R4", - "arm": { - "workerModel": "zai-coding-plan/glm-5.2", - "driverModel": "glm-5.2", - "budget": 40, - "maxSandboxes": 4, - "maxUsd": 8, - "maxDepth": 3, - "timeoutMs": 2800000 - }, - "verifyDir": "/home/drew/code/agent-runtime-gen5/bench/src/swe-arena/fixtures/verify", - "outDir": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen6", - "roundsDir": "/home/drew/code/supervisor-lab/.evolve/rounds", - "secretsDir": "/home/drew/company/devops/secrets", - "envFiles": [ - "agent-state.env", - "tangle-router.env" - ], - "generations": 1, - "populationSize": 5, - "repsPerInstance": 2, - "premeasuredBaselinePath": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen6/premeasured-baseline.json", - "maxShots": 3, - "proposerHarness": "claude", - "proposerTimeoutMs": 2400000, - "analystModels": [ - "glm-5.2", - "glm-5.2", - "glm-5.2" - ], - "seedArtifactRuns": [ - { - "iid": "astropy__astropy-13033", - "arm": "SUP4", - "dir": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/runs/astropy__astropy-13033/SUP4", - "patchPath": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/astropy__astropy-13033.sup4.patch", - "resolved": false - }, - { - "iid": "django__django-11532", - "arm": "SUP4", - "dir": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/runs/django__django-11532/SUP4", - "patchPath": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/django__django-11532.sup4.patch", - "resolved": false - }, - { - "iid": "matplotlib__matplotlib-20826", - "arm": "SUP4", - "dir": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/runs/matplotlib__matplotlib-20826/SUP4", - "patchPath": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/matplotlib__matplotlib-20826.sup4.patch", - "resolved": true - } - ], - "costGuardRatio": 1.2, - "dispatchTimeoutMs": 7200000, - "proposers": [ - { - "name": "claude-author", - "profile": "default-author.profile.json", - "harness": "claude" - }, - { - "name": "glm-author", - "harness": "opencode", - "model": "zai-coding-plan/glm-5.2" - }, - { - "name": "codex-author", - "harness": "codex" - }, - { - "name": "merge-author", - "profile": "default-author.profile.json", - "harness": "claude", - "merge": true - }, - { - "name": "gepa-author", - "engine": "gepa", - "surface": "extensions/pi/prompts/worker-coding-system.md", - "maxMetricCalls": 10, - "maxProposerCostUsd": 10 - } - ], - "prefilter": { - "enabled": true, - "smokeInstance": "cheapest-of-set", - "requireResolved": false - }, - "holdoutRepsPerInstance": 2, - "holdoutBaseline": "measure", - "paretoParents": [ - { - "commit": "cc0d95584c7ea14324cd57c21fe946c7c0f53827", - "label": "default-author", - "resolvedInstances": [ - "pydata__xarray-4687", - "sphinx-doc__sphinx-9658" - ], - "note": "mechanical patch-risk scan (patchRiskWarnings in src/worker-evidence.ts, threaded through extensions/pi/loops.ts) + never-reword / test-seam / run-the-neighbors worker rules + 3-check reviewer; pytest-dev__pytest-6197 split 0/1 across reps (near-miss)" - }, - { - "commit": "a7a2a982e51551de3a8e796ee2efc448ed405e6a", - "label": "prompts-author", - "resolvedInstances": [ - "django__django-11532", - "pydata__xarray-4687" - ], - "note": "prompt-only: hidden-suite bullet in the supervisor GOAL-authoring section (the django seam), frozen-behavior worker section + run-the-repo-tests discipline, 2-check reviewer; sphinx-doc__sphinx-9658 split 0/1 across reps (near-miss)" - } - ], - "scoreSplit": { - "publicCount": 4 - }, - "briefing": "map-toolbox-v1", - "activationGate": true, - "rolloutLedger": { - "enabled": true - }, - "lineage": true, - "priorEvidenceDirs": [ - "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen5", - "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen4" - ] -} diff --git a/bench/package.json b/bench/package.json index 9530bdf7..22272261 100644 --- a/bench/package.json +++ b/bench/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-bench", - "version": "0.3.7", + "version": "0.4.0", "type": "module", "description": "The unified benchmark suite for agent-runtime agents: 31 adapters (commit0, enterpriseops-gym, ragbench, crag, nomiracl, open-rag-bench, t2-ragbench, tau3-banking, bfcl, finresearchbench, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.", "repository": { @@ -40,7 +40,7 @@ "verify:pier": "tsx scripts/verify-pier-pair.mts" }, "dependencies": { - "@tangle-network/agent-eval": "0.123.5", + "@tangle-network/agent-eval": "0.126.0", "@tangle-network/agent-interface": "0.32.0", "@tangle-network/agent-knowledge": "^4.1.0", "@tangle-network/agent-runtime": "workspace:*", diff --git a/bench/scripts/trata-hedge/README.md b/bench/scripts/trata-hedge/README.md index 698e426d..9eb25e30 100644 --- a/bench/scripts/trata-hedge/README.md +++ b/bench/scripts/trata-hedge/README.md @@ -13,9 +13,9 @@ sparse (1 iff all themes). No deployable ground-truth checker — it's an **orac - **NOT** the verifier-grounded selector gate (our HumanEval/commit0 headline). The only checker is the judge itself, so selecting by it = selecting by the eval metric (an oracle, a Goodhart trap). See `docs/results.md`. -- **IS** admissible: (1) more-compute headroom (pass@k), (2) the **improvement loop** - (`selfImprove`/GEPA optimizing the analyst directive *against this judge*, held-out - gated — the legitimate use of a judge domain), with the honest caveat that an +- **IS** admissible: (1) more-compute headroom (pass@k), (2) prompt optimization + (`improve` with agent-eval's official GEPA method, using disjoint train, selection, + and final-test partitions), with the honest caveat that an LLM-judge can be *gamed*, so a held-out lift means "higher judge score," which without ground truth we can't independently certify as "better analysis." @@ -52,5 +52,6 @@ dotenvx run -f ~/company/devops/secrets/.env.keys -f ~/company/devops/secrets/ag ## Next steps 1. Agentic solver: our sandbox runtime browses the corpus + cites (fair baseline). -2. The improvement loop: `selfImprove` on the analyst directive vs this judge, held-out - gated, across the 102 tasks — with the judge-gaming caveat stated. +2. Prompt optimization: run `bench/src/trata-gepa.mts` across all 102 tasks with + explicit train, selection, and final-test partitions, and state the judge-gaming + caveat with every result. diff --git a/bench/src/benchmarks/humaneval.test.mts b/bench/src/benchmarks/humaneval.test.mts index 8087705c..cb75731c 100644 --- a/bench/src/benchmarks/humaneval.test.mts +++ b/bench/src/benchmarks/humaneval.test.mts @@ -13,8 +13,8 @@ describe('HumanEval Python isolation', () => { writeFileSync( fakeDocker, `#!/usr/bin/env node -const fs = require('node:fs') -const path = require('node:path') +import fs from 'node:fs' +import path from 'node:path' const args = process.argv.slice(2) if (args[0] === 'rm') process.exit(0) if (process.env.FAKE_DOCKER_MISSING === '1') { diff --git a/bench/src/live-improve-campaign-mbpp.mts b/bench/src/live-improve-campaign-mbpp.mts deleted file mode 100644 index b8896fbf..00000000 --- a/bench/src/live-improve-campaign-mbpp.mts +++ /dev/null @@ -1,641 +0,0 @@ -/** - * live-improve-campaign-mbpp — the kill-flow follow-up to live-improve-campaign.mts - * (HumanEval × Qwen2.5-7B: DEV baseline 90%, saturated, gate correctly HELD). Identical - * loop, moved to a config WITH headroom: sanitized MBPP, where the same worker measures - * 76.7% at the baseline recipe k5/r2/t6 against an 85.9% pass@5 bound (~9pts of room). - * The OPEN question this run answers: does dial-tuning ship a win when there is room - * to win? `improve()` (surface 'rollout-policy') tunes { k, repairRounds, testgen } - * and the library's own held-out gate makes the ship/hold call — a HOLD is a valid - * result; the gate is never loosened. - * - * Wiring is live-improve-campaign.mts verbatim except the dataset seams: - * - Tasks come from mbpp-structural.mts's loadMbpp (sanitized MBPP, MBPP_JSON env); - * prompt = its basePrompt (description + the shown official assert). - * - VISIBLE checks: the shown assert (test_list[0]) rides task.meta.visibleChecks so - * the strategy's default officialChecksFromMeta() source ranks it as the OFFICIAL - * check, lexicographically above the model-authored guesses (mbpp-structural's - * measured lesson: unweighted guesses flip selection negative). - * - HIDDEN grading: test_list[1:] (+ test_imports), script-side nonce-sentinel judge - * in docker --network=none, AFTER the strategy locks its artifact. - * - test_imports are prepended to every candidate before visible-check execution - * (a CheckRunner prelude wrapper) and inside the hidden program, mirroring the rig. - * - * Honesty split (identical to the prior run): - * - DEV = sanitized-MBPP usable-task index [0, DEV_N) and HELD-OUT = [DEV_N, - * DEV_N+HOLD_N) — fixed, disjoint slices, passed as explicit - * `budget.holdoutScenarios` so the library's own split machinery enforces - * disjointness (it throws on overlap). - * - Deterministic proposer; `analyzeGeneration: null` keeps the findings channel - * empty — the improver's context is ONLY the DEV composites the loop accumulates. - * - The gate decision is `result.gateDecision` from the library — never recomputed - * here. The recompute-from-raw block at the end cross-checks the held-out pass - * counts against the durable per-cell files; a mismatch voids any ship claim. - * - * Guards (all from the prior run): SMOKE=1 first (6 dev + 6 held-out tasks, 1 - * generation, population 2 — proves the full path completes before the burn); - * fail-loud model preflight; MAX_CALLS hard cap (default 8000 — the run aborts loud, - * durable provenance survives in RUN_DIR); dollars ceiling. - * - * Run (key via dotenvx; never in the shell history): - * cd ~/company/devops/secrets && dotenvx run -f agent-state.env -- bash -c ' \ - * cd /home/drew/code/agent-runtime-swe && \ - * MBPP_JSON=/abs/sanitized-mbpp.json npx tsx bench/src/live-improve-campaign-mbpp.mts' - */ - -import { execFile, execFileSync } from 'node:child_process' -import { randomBytes } from 'node:crypto' -import { appendFileSync, existsSync, mkdirSync, readdirSync, readFileSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import type { - DispatchContext, - JudgeConfig, - MutableSurface, - Scenario, -} from '@tangle-network/agent-eval/contract' -import type { AgentProfile } from '@tangle-network/agent-interface' -import { improve } from '../../src/improvement/improve' -import { - parseRolloutPolicy, - ROLLOUT_POLICY_EXTENSION, - serializeRolloutPolicy, - structuralRolloutPolicyFromProfile, -} from '../../src/improvement/rollout-policy' -import { - type AgenticRunResult, - type CheckExecChannel, - type CheckOutcome, - type CheckRunner, - createVerifierEnvironment, - runAgentic, - sandboxCheckRunner, - structuralRollout, - type StructuralRolloutResult, -} from '../../src/runtime/index' -import { basePrompt, loadMbpp, type MbppTask } from './mbpp-structural.mts' - -function must(name: string): string { - const v = process.env[name] - if (!v) throw new Error(`env ${name} is required`) - return v -} - -const SMOKE = process.env.SMOKE === '1' -const DEV_N = Number(process.env.DEV_N ?? (SMOKE ? 6 : 150)) -const HOLD_N = Number(process.env.HOLD_N ?? (SMOKE ? 6 : 150)) -const GENERATIONS = Number(process.env.GENERATIONS ?? (SMOKE ? 1 : 2)) -const POPULATION = Number(process.env.POPULATION ?? (SMOKE ? 2 : 4)) -const CONCURRENCY = Number(process.env.CONCURRENCY ?? 8) -const DOCKER_CONCURRENCY = Number(process.env.DOCKER_CONCURRENCY ?? 6) -// The worker of the measured MBPP basis: 76.7% repaired@1 at k5/r2/t6, 85.9% pass@5 -// bound — the headroom config the saturated HumanEval run lacked. -const MODEL = process.env.MODEL ?? 'Qwen/Qwen2.5-7B-Instruct-Turbo' -const BASE = process.env.ROUTER_BASE ?? 'https://api.together.xyz/v1' -const TEMP = Number(process.env.TEMPERATURE ?? 0.8) -const MAX_TOKENS = Number(process.env.MAX_TOKENS ?? 2500) -const DOLLARS = Number(process.env.DOLLARS ?? 15) -/** Hard LLM-call cap across every phase — the runaway guard. Hitting it aborts the - * process loud (durable provenance survives in RUN_DIR); it never silently degrades. */ -const MAX_CALLS = Number(process.env.MAX_CALLS ?? 8000) -const RUN_DIR = process.env.RUN_DIR ?? join(tmpdir(), `live-improve-campaign-mbpp-${Date.now()}`) - -const systemPrompt = 'You are an expert Python programmer.' -const dockerImage = 'python:3.12-slim' -const dockerTimeoutMs = Number(process.env.DOCKER_TIMEOUT_MS ?? 20000) - -// ── Docker: ONE semaphored --network=none exec channel for BOTH judges ─────────────── - -let dockerInFlight = 0 -const dockerWaiters: Array<() => void> = [] -async function withDockerSlot(fn: () => Promise): Promise { - if (dockerInFlight >= DOCKER_CONCURRENCY) await new Promise((r) => dockerWaiters.push(r)) - dockerInFlight += 1 - try { - return await fn() - } finally { - dockerInFlight -= 1 - dockerWaiters.shift()?.() - } -} - -const containerPrefix = `licm-${process.pid}` -let containerSeq = 0 - -function reapContainers(): void { - try { - const ids = execFileSync('docker', ['ps', '-aq', '--filter', `name=${containerPrefix}`], { - timeout: 10000, - }) - .toString() - .trim() - if (ids) execFileSync('docker', ['rm', '-f', ...ids.split('\n')], { timeout: 15000 }) - } catch { - /* reaper is best-effort by design */ - } -} -process.on('SIGINT', () => { - reapContainers() - process.exit(130) -}) -process.on('SIGTERM', () => { - reapContainers() - process.exit(143) -}) - -const dockerBox: CheckExecChannel = { - exec(command, options) { - const timeoutMs = options?.timeoutMs ?? dockerTimeoutMs - return withDockerSlot( - () => - new Promise((resolve, reject) => { - const name = `${containerPrefix}-${containerSeq++}` - let settled = false - const reap = () => execFile('docker', ['rm', '-f', name], () => {}) - const finish = (r: { exitCode: number; stdout: string; stderr: string }) => { - if (settled) return - settled = true - clearTimeout(backstop) - reap() - resolve(r) - } - const fail = (e: Error) => { - if (settled) return - settled = true - clearTimeout(backstop) - reap() - reject(e) - } - // execFile's timeout kills the docker CLIENT; a hung container could leave - // the callback unfired. The backstop guarantees resolution and the named - // reap kills the stray container. - const backstop = setTimeout( - () => finish({ exitCode: 124, stdout: '', stderr: 'timed out (backstop)' }), - timeoutMs + 3000, - ) - execFile( - 'docker', - ['run', '--rm', '--name', name, '--network=none', '--cpus=1', '--memory=512m', dockerImage, 'sh', '-c', command], - { timeout: timeoutMs, killSignal: 'SIGKILL', maxBuffer: 4 * 1024 * 1024 }, - (err, stdout, stderr) => { - if (err) { - const e = err as NodeJS.ErrnoException & { code?: number | string } - if (e.code === 'ENOENT') { - fail(new Error('docker binary not found on PATH')) - return - } - if (/cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(stderr ?? '')) { - fail(new Error(`docker daemon unreachable: ${(stderr ?? '').slice(0, 200)}`)) - return - } - finish({ exitCode: typeof e.code === 'number' ? e.code : 1, stdout: stdout ?? '', stderr: stderr ?? '' }) - return - } - finish({ exitCode: 0, stdout: stdout ?? '', stderr: stderr ?? '' }) - }, - ) - }), - ) - }, -} - -// ── The hidden nonce judge (script-side, AFTER the strategy locks its artifact) ────── -// MBPP hidden suite = test_list[1:] with test_imports; pass requires exit 0 AND the -// per-call nonce sentinel in stdout — a candidate printing a forged verdict cannot pass. - -function buildHiddenProgram(task: MbppTask, candidate: string, nonce: string): string { - return `${task.testImports.join('\n')}\n${candidate}\n\n${task.hiddenAsserts.join('\n')}\nprint("HIDDEN-${nonce} PASS")\n` -} - -async function runHiddenJudge( - task: MbppTask, - candidate: string, -): Promise<{ pass: number; detail?: string }> { - const nonce = randomBytes(8).toString('hex') - const b64 = Buffer.from(buildHiddenProgram(task, candidate, nonce), 'utf8').toString('base64') - const r = await dockerBox.exec(`printf '%s' '${b64}' | base64 -d | python3 -`, { - timeoutMs: dockerTimeoutMs, - }) - if (r.exitCode === 0 && r.stdout.includes(`HIDDEN-${nonce} PASS`)) return { pass: 1 } - return { pass: 0, detail: (r.stderr || r.stdout).slice(-300) || 'timed out (no output)' } -} - -// ── Scenarios: fixed disjoint slices of sanitized MBPP ──────────────────────────────── - -interface MbppScenario extends Scenario { - kind: 'mbpp' -} - -const taskById = new Map() -const scenarioId = (t: MbppTask) => `mbpp/${t.taskId}` - -// ── The real evaluator: one cell = one structuralRollout run + hidden grade ────────── - -interface CellArtifact { - taskId: string - policy: string - /** Hidden nonce-judge grade of the FINAL selected candidate: {0,1}. */ - pass: number - detail?: string - repairStop: string - shots: number - completions: number - authoredChecks: number - officialChecks: number - tokens: { input: number; output: number } - usd: number - ms: number -} - -interface ScoredCandidate { - candidate: string - outcome: CheckOutcome -} -function recordingRunner(inner: CheckRunner, log: ScoredCandidate[]): CheckRunner { - return { - async run(candidate, checks, ctx) { - const outcome = await inner.run(candidate, checks, ctx) - log.push({ candidate, outcome }) - return outcome - }, - } -} - -/** MBPP's test_imports must be in scope when the visible checks run (the rig prepends - * them to every judged program). The recorded candidate stays RAW — the hidden judge - * prepends the same imports itself. */ -function testImportsPrelude(inner: CheckRunner): CheckRunner { - return { - run(candidate, checks, ctx) { - const raw = ctx.task.meta?.testImports - const imports = Array.isArray(raw) ? raw.filter((i): i is string => typeof i === 'string') : [] - return inner.run(imports.length > 0 ? `${imports.join('\n')}\n${candidate}` : candidate, checks, ctx) - }, - } -} - -// Global spend meter (every cell of every phase — baseline, generations, holdout). -const spend = { cells: 0, llmCalls: 0, tokensIn: 0, tokensOut: 0, usd: 0, hiddenPass: 0 } - -async function evaluateCell( - surface: MutableSurface, - scenario: MbppScenario, - ctx: DispatchContext, -): Promise { - if (spend.llmCalls >= MAX_CALLS) { - console.error( - `\nBUDGET CAP HIT: ${spend.llmCalls} llm calls >= MAX_CALLS=${MAX_CALLS} — aborting loud; durable provenance in ${RUN_DIR}`, - ) - reapContainers() - process.exit(1) - } - const policy = parseRolloutPolicy(surface) - if (!policy) { - throw new Error(`agent: surface carries no valid rollout policy: ${String(surface).slice(0, 120)}`) - } - const task = taskById.get(scenario.id) - if (!task) throw new Error(`agent: unknown scenario id ${scenario.id}`) - - const scored: ScoredCandidate[] = [] - const strategy = structuralRollout({ - policy: { ...policy, temperature: TEMP }, - checkRunner: recordingRunner(testImportsPrelude(sandboxCheckRunner({ box: dockerBox })), scored), - }) - // INERT check: the strategy's harness-verified score channel carries no hidden - // signal — hidden grading happens below, after the rollout locks its artifact. - const inertSurface = createVerifierEnvironment({ - name: 'mbpp-inert', - check: () => ({ passes: 0, total: 1, errored: 0 }), - }) - const result = (await runAgentic({ - surface: inertSurface, - task: { - id: scenario.id, - systemPrompt, - userPrompt: basePrompt(task), - meta: { - entryPoint: task.entryPoint, - // The OFFICIAL check: the shown assert (test_list[0], printed in the prompt) - // feeds the strategy's default officialChecksFromMeta() source, so it ranks - // above the model-authored guesses in selection and repair. - visibleChecks: [task.shownAssert], - testImports: task.testImports, - }, - }, - routerBaseUrl: BASE, - routerKey: must('TOGETHER_API_KEY'), - model: MODEL, - temperature: TEMP, - maxTokens: MAX_TOKENS, - innerTurns: 2, - strategy, - // The strategy's documented sizing: k samples + repair rounds + the check-author consult. - budget: policy.k + policy.repairRounds + 1, - })) as AgenticRunResult & StructuralRolloutResult - - // Backend integrity: report REAL usage on every cell (expectUsage 'assert' upstream). - ctx.cost.observe(result.usd, 'together') - ctx.cost.observeTokens(result.tokens) - - const winner = result.selection.find((r) => r.selected) - if (!winner) { - // Zero candidates means EVERY shot for this cell returned null — the signature of a - // dead worker (credits exhausted / rate-limited / outage), not a hard task. The - // runtime swallows exhausted-retry shots as null, so without this the run degrades - // silently for hundreds of cells and then dies at the holdout with a cryptic empty- - // gate error. Probe the API once and abort LOUD with the real HTTP status so the - // operator sees the actual cause (e.g. HTTP 402 credit exceeded) immediately. - if (result.repairStop === 'no-candidates') { - await abortOnDeadWorker(scenario.id) - } - throw new Error(`${scenario.id}: no receipt marked selected (repairStop=${result.repairStop})`) - } - const rec = scored[winner.candidateIndex] - if (!rec) { - throw new Error( - `${scenario.id}: selected receipt #${winner.candidateIndex} has no recorded candidate (${scored.length} scored)`, - ) - } - - const hidden = await runHiddenJudge(task, rec.candidate) - - spend.cells += 1 - spend.llmCalls += result.completions - spend.tokensIn += result.tokens.input - spend.tokensOut += result.tokens.output - spend.usd += result.usd - spend.hiddenPass += hidden.pass - - const artifact: CellArtifact = { - taskId: scenario.id, - policy: serializeRolloutPolicy(policy), - pass: hidden.pass, - ...(hidden.detail ? { detail: hidden.detail } : {}), - repairStop: result.repairStop, - shots: result.shots, - completions: result.completions, - authoredChecks: result.authoredChecks, - officialChecks: result.officialChecks, - tokens: result.tokens, - usd: result.usd, - ms: result.ms, - } - appendFileSync( - join(RUN_DIR, 'cells.jsonl'), - `${JSON.stringify({ cellId: ctx.cellId, generation: ctx.generation ?? null, ...artifact, detail: undefined })}\n`, - ) - console.log( - ` [cell ${String(spend.cells).padStart(4)}] ${scenario.id.padEnd(10)} ${artifact.policy.padEnd(38)} hidden=${hidden.pass ? 'PASS' : 'fail'} ${result.repairStop} calls=${result.completions}`, - ) - return artifact -} - -// The in-loop judge is a deterministic transcriber of the script-side hidden grade — -// the grading itself never runs inside the strategy or the proposer's view. -const hiddenJudge: JudgeConfig = { - name: 'hidden-nonce-judge', - dimensions: [ - { key: 'hidden', description: 'MBPP hidden suite test_list[1:] (docker --network=none, nonce sentinel)' }, - ], - score: ({ artifact }) => ({ - dimensions: { hidden: artifact.pass }, - composite: artifact.pass, - notes: artifact.pass ? 'hidden PASS' : `hidden fail: ${(artifact.detail ?? '').slice(0, 160)}`, - }), -} - -// ── Fail-loud model preflight (cost gate: prove the worker is live before any burn) ── - -/** One direct 8-token probe of the worker. Returns HTTP status + a short body slice — - * the shared health check for the startup preflight and the mid-run dead-worker guard. */ -async function probeWorker(): Promise<{ ok: boolean; status: number; body: string; content: string }> { - const res = await fetch(`${BASE}/chat/completions`, { - method: 'POST', - headers: { Authorization: `Bearer ${must('TOGETHER_API_KEY')}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ - model: MODEL, - max_tokens: 16, - temperature: 0, - messages: [{ role: 'user', content: 'Reply with the single word: ready' }], - }), - }) - if (!res.ok) return { ok: false, status: res.status, body: (await res.text()).slice(0, 300), content: '' } - const d = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> } - return { ok: true, status: res.status, body: '', content: (d.choices?.[0]?.message?.content ?? '').trim() } -} - -async function preflightModel(): Promise { - const p = await probeWorker() - if (!p.ok) throw new Error(`model preflight FAILED: ${MODEL} @ ${BASE} → HTTP ${p.status}: ${p.body}`) - if (p.content === '') throw new Error(`model preflight FAILED: ${MODEL} returned empty content`) - console.log(` preflight: ${MODEL} is live (replied ${JSON.stringify(p.content.slice(0, 40))})`) -} - -/** Called when a cell produced zero candidates (every shot null). Probes the worker; if - * it is unhealthy (e.g. HTTP 402 credit exceeded, 429 rate limit, outage) the whole run - * is doomed — abort LOUD now rather than degrade through hundreds more null cells into - * a cryptic empty-holdout gate error. If the probe is HEALTHY the null was a one-off, so - * return and let the per-cell throw handle just this cell. */ -async function abortOnDeadWorker(scenarioId: string): Promise { - const p = await probeWorker() - if (p.ok && p.content !== '') return - console.error( - `\nDEAD WORKER: cell ${scenarioId} produced zero candidates and a direct probe returned ` + - `${p.ok ? `empty content` : `HTTP ${p.status}: ${p.body}`}. Every shot is failing — aborting ` + - `loud (durable provenance in ${RUN_DIR}). If this is HTTP 402, add Together credits and re-run.`, - ) - reapContainers() - process.exit(1) -} - -// ── Reporting helpers (read the library's own result objects; never re-decide) ─────── - -interface CampaignLike { - cells: Array<{ error?: string | null; judgeScores: Record }> -} -function passStats(campaign: CampaignLike): { passed: number; scored: number; errored: number; rate: number } { - let passed = 0 - let scored = 0 - let errored = 0 - for (const cell of campaign.cells) { - if (cell.error) { - errored += 1 - continue - } - scored += 1 - const scores = Object.values(cell.judgeScores) - const composite = scores.length === 0 ? 0 : scores.reduce((s, j) => s + j.composite, 0) / scores.length - if (composite >= 0.999) passed += 1 - } - return { passed, scored, errored, rate: scored > 0 ? passed / scored : 0 } -} -const pct = (x: number) => `${(100 * x).toFixed(1)}%` - -/** Recompute pass counts from the DURABLE per-cell files the loop's fs storage wrote - * (`//_/cached-result.json`, artifact.pass = the hidden - * nonce-judge grade). Independent of the in-memory campaign objects — the ship-claim - * cross-check. */ -function recomputeFromRaw(dir: string): { passed: number; scored: number; errored: number } | null { - if (!existsSync(dir)) return null - let passed = 0 - let scored = 0 - let errored = 0 - for (const entry of readdirSync(dir)) { - const file = join(dir, entry, 'cached-result.json') - if (!existsSync(file)) continue - const cell = JSON.parse(readFileSync(file, 'utf8')) as { - error?: string | null - artifact?: { pass?: number } - } - if (cell.error) { - errored += 1 - continue - } - scored += 1 - if ((cell.artifact?.pass ?? 0) >= 1) passed += 1 - } - return { passed, scored, errored } -} - -async function main(): Promise { - must('TOGETHER_API_KEY') - mkdirSync(RUN_DIR, { recursive: true }) - const started = Date.now() - - const { tasks: all, droppedShort, droppedEntry } = loadMbpp(DEV_N + HOLD_N, 0) - if (all.length !== DEV_N + HOLD_N) { - throw new Error(`expected ${DEV_N + HOLD_N} usable MBPP tasks, loaded ${all.length}`) - } - const devTasks = all.slice(0, DEV_N) - const holdTasks = all.slice(DEV_N) - for (const t of all) taskById.set(scenarioId(t), t) - - const toScenario = (t: MbppTask): MbppScenario => ({ id: scenarioId(t), kind: 'mbpp' }) - const scenarios = all.map(toScenario) - const holdoutScenarios = holdTasks.map(toScenario) - - const baselinePolicy = { k: 5, repairRounds: 2, testgen: 6 } - const profile: AgentProfile = { - name: 'mbpp-structural-worker', - extensions: { [ROLLOUT_POLICY_EXTENSION]: baselinePolicy }, - } - - console.log('=== LIVE self-improvement campaign · MBPP (headroom config) · improve() surface rollout-policy ===') - console.log(` worker: ${MODEL} @ ${BASE} (temp=${TEMP}, maxTokens=${MAX_TOKENS}, innerTurns=2)`) - console.log( - ` dataset: sanitized MBPP (${process.env.MBPP_JSON}); dropped at load: ${droppedShort.length} (<2 asserts), ${droppedEntry.length} (entry unresolved)`, - ) - console.log( - ` DEV slice : usable-task index [0, ${DEV_N}) — mbpp/${devTasks[0]?.taskId} .. mbpp/${devTasks[devTasks.length - 1]?.taskId} (n=${devTasks.length})`, - ) - console.log( - ` HELD-OUT slice : usable-task index [${DEV_N}, ${DEV_N + HOLD_N}) — mbpp/${holdTasks[0]?.taskId} .. mbpp/${holdTasks[holdTasks.length - 1]?.taskId} (n=${holdTasks.length})`, - ) - console.log(` baseline policy: ${JSON.stringify(baselinePolicy)} (measured basis: 76.7% repaired@1, 85.9% pass@5 bound)`) - console.log( - ` budget: generations=${GENERATIONS} population<=${POPULATION} reps=1 concurrency=${CONCURRENCY} docker<=${DOCKER_CONCURRENCY} ceiling=$${DOLLARS} maxCalls=${MAX_CALLS}`, - ) - console.log(` gate: library defaultProductionGate (paired bootstrap on held-out, ship iff CI.low > 0.05) — UNCHANGED`) - console.log(` runDir: ${RUN_DIR}`) - await preflightModel() - console.log(`\n profile BEFORE: ${JSON.stringify(profile)}\n`) - - const result = await improve(profile, [], { - surface: 'rollout-policy', - scenarios, - judge: hiddenJudge, - agent: evaluateCell, - budget: { - generations: GENERATIONS, - populationSize: POPULATION, - maxConcurrency: CONCURRENCY, - holdoutScenarios, - reps: 1, - dollars: DOLLARS, - }, - runDir: RUN_DIR, - // Deterministic proposer, empty findings channel: the improver's context is ONLY - // the DEV composites the loop accumulates — no distilled failure text, no trace - // paths, and (by the loop's own structure) never a held-out cell. - analyzeGeneration: null, - }) - - const loop = result.raw.raw - const wallMin = (Date.now() - started) / 60000 - - console.log('\n── DEV (train) results — what the improver saw ──') - const baseDev = passStats(loop.baselineCampaign) - console.log( - ` gen -1 baseline ${serializeRolloutPolicy(structuralRolloutPolicyFromProfile(profile)!).padEnd(38)} DEV ${baseDev.passed}/${baseDev.scored} = ${pct(baseDev.rate)} (errored ${baseDev.errored})`, - ) - for (const gen of loop.generations) { - const surfaceByHash = new Map(gen.surfaces.map((s) => [s.surfaceHash, s])) - const promotedHashes = new Set(gen.record.promoted) - for (const cand of gen.record.candidates) { - const s = surfaceByHash.get(cand.surfaceHash) - const stats = s ? passStats(s.campaign) : undefined - console.log( - ` gen ${String(gen.record.generationIndex).padStart(2)} ${String(cand.label ?? '').padEnd(16)} ${String(s?.surface ?? '?').padEnd(38)} DEV ${stats ? `${stats.passed}/${stats.scored} = ${pct(stats.rate)} (errored ${stats.errored})` : '?'}${promotedHashes.has(cand.surfaceHash) ? ' [promoted]' : ''}`, - ) - } - } - console.log(` training winner: ${String(loop.winnerSurface)}${loop.winnerLabel ? ` (${loop.winnerLabel})` : ''}`) - - console.log('\n── HELD-OUT gate — the library decides ──') - const baseHold = passStats(loop.baselineOnHoldout) - const winHold = passStats(loop.winnerOnHoldout) - console.log( - ` baseline on held-out : ${baseHold.passed}/${baseHold.scored} = ${pct(baseHold.rate)} (errored ${baseHold.errored})`, - ) - console.log( - ` winner on held-out : ${winHold.passed}/${winHold.scored} = ${pct(winHold.rate)} (errored ${winHold.errored})`, - ) - console.log(` gate decision: ${result.gateDecision.toUpperCase()} (lift ${result.lift >= 0 ? '+' : ''}${result.lift.toFixed(3)})`) - for (const reason of loop.gateResult.reasons) console.log(` reason: ${reason}`) - for (const g of loop.gateResult.contributingGates) { - console.log(` gate[${g.name}] passed=${g.passed} detail=${JSON.stringify(g.detail).slice(0, 300)}`) - } - - console.log('\n── recompute-from-raw check (durable cell files, artifact.pass) ──') - const rawBase = recomputeFromRaw(join(RUN_DIR, 'holdout-baseline')) - const rawWin = recomputeFromRaw(join(RUN_DIR, 'holdout-winner')) - if (rawBase && rawWin && rawBase.scored > 0 && rawWin.scored > 0) { - const rawLift = rawWin.passed / rawWin.scored - rawBase.passed / rawBase.scored - console.log(` holdout-baseline raw: ${rawBase.passed}/${rawBase.scored} = ${pct(rawBase.passed / rawBase.scored)} (errored ${rawBase.errored})`) - console.log(` holdout-winner raw: ${rawWin.passed}/${rawWin.scored} = ${pct(rawWin.passed / rawWin.scored)} (errored ${rawWin.errored})`) - console.log(` raw pass-rate lift: ${rawLift >= 0 ? '+' : ''}${rawLift.toFixed(3)} (loop-reported composite lift ${result.lift >= 0 ? '+' : ''}${result.lift.toFixed(3)})`) - const agree = rawBase.passed === baseHold.passed && rawWin.passed === winHold.passed - console.log(` agreement with library objects: ${agree ? 'EXACT' : 'MISMATCH — investigate before any ship claim'}`) - } else if (rawBase && rawBase.scored > 0 && !rawWin) { - // The library skips the winner holdout campaign when winner == baseline (empty - // diff) — nothing shipped, so there is nothing separate to recompute. - console.log( - ` holdout-baseline raw: ${rawBase.passed}/${rawBase.scored} = ${pct(rawBase.passed / rawBase.scored)} (errored ${rawBase.errored})`, - ) - console.log(' holdout-winner absent: winner == baseline, no separate winner campaign ran (ship impossible this run)') - } else { - console.log(' raw holdout cell files missing — cannot recompute (storage did not persist cells?)') - } - - console.log('\n── ship/hold outcome ──') - console.log(` shipped: ${result.shipped}`) - console.log(` profile AFTER : ${JSON.stringify(result.profile)}`) - if (result.shipped) { - console.log(` policy change : ${serializeRolloutPolicy(structuralRolloutPolicyFromProfile(profile)!)} → ${serializeRolloutPolicy(structuralRolloutPolicyFromProfile(result.profile)!)}`) - } else { - console.log(' policy change : none (gate held — baseline policy stays)') - } - - console.log('\n── spend / provenance ──') - console.log( - ` cells ${spend.cells} · llm calls ${spend.llmCalls} · tokens ${spend.tokensIn} in / ${spend.tokensOut} out · router-priced $${spend.usd.toFixed(4)} · loop-reported $${result.raw.totalCostUsd.toFixed(4)}`, - ) - console.log(` wall ${wallMin.toFixed(1)} min · runDir ${RUN_DIR} (cells.jsonl + campaign cells + loop provenance)`) - reapContainers() - process.exit(0) -} - -main().catch((e) => { - console.error(e) - reapContainers() - process.exit(1) -}) diff --git a/bench/src/live-improve-campaign.mts b/bench/src/live-improve-campaign.mts deleted file mode 100644 index e53538d5..00000000 --- a/bench/src/live-improve-campaign.mts +++ /dev/null @@ -1,500 +0,0 @@ -/** - * live-improve-campaign — the FIRST LIVE self-improvement campaign on the merged - * machinery: `improve()` (surface 'rollout-policy') tunes the structuralRollout dials - * { k, repairRounds, testgen } against REAL HumanEval with a REAL worker model, and the - * library's own held-out gate (defaultProductionGate: paired bootstrap over held-out - * scenarios, ship iff CI.low > deltaThreshold 0.05) makes the ship/hold call. No human - * picks winners; this script only wires the real evaluator into the loop and reports. - * - * Wiring template: rollout-policy.test.ts's end-to-end improve() run, with the fake - * judge gradient replaced by the real evaluator (the smoke's harness): - * agent(surface, scenario) = runAgentic(structuralRollout(parsed policy)) over an - * INERT verifier surface (no hidden signal reaches selection/repair), visible checks - * via the shipped sandboxCheckRunner over a docker --network=none exec channel, then - * SCRIPT-SIDE hidden grading of the locked winner candidate by the nonce-sentinel - * judge (hev-structural's runHiddenJudge pattern: pass requires exit 0 AND the - * per-call nonce in stdout — a candidate printing a forged verdict cannot pass). - * - * Honesty split: - * - DEV = HumanEval index [0, DEV_N) and HELD-OUT = [DEV_N, DEV_N+HOLD_N) — fixed, - * disjoint slices, passed as explicit `budget.holdoutScenarios` so the library's - * own train/holdout split machinery enforces disjointness (it throws on overlap). - * - The proposer is deterministic enumeration; `analyzeGeneration: null` keeps the - * findings channel empty, so the improver's context is ONLY the DEV composites the - * loop itself accumulates. Held-out cells run after all generations, gate-side only. - * - The gate decision is `result.gateDecision` from the library — never recomputed here. - * - * Run (key via dotenvx; never in the shell history): - * cd ~/company/devops/secrets && dotenvx run -f agent-state.env -- bash -c ' \ - * cd /home/drew/code/agent-runtime-swe && \ - * HUMANEVAL_GZ=/abs/HumanEval.jsonl.gz npx tsx bench/src/live-improve-campaign.mts' - * Smoke first (cost gate): SMOKE=1 shrinks to 6 dev + 6 held-out tasks, 1 generation, - * population 2 — proves the full path completes before the real burn. - */ - -import { execFile, execFileSync } from 'node:child_process' -import { randomBytes } from 'node:crypto' -import { appendFileSync, mkdirSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import type { - DispatchContext, - JudgeConfig, - MutableSurface, - Scenario, -} from '@tangle-network/agent-eval/contract' -import type { AgentProfile } from '@tangle-network/agent-interface' -import { improve } from '../../src/improvement/improve' -import { - parseRolloutPolicy, - ROLLOUT_POLICY_EXTENSION, - serializeRolloutPolicy, - structuralRolloutPolicyFromProfile, -} from '../../src/improvement/rollout-policy' -import { - type AgenticRunResult, - type CheckExecChannel, - type CheckOutcome, - type CheckRunner, - createVerifierEnvironment, - runAgentic, - sandboxCheckRunner, - structuralRollout, - type StructuralRolloutResult, -} from '../../src/runtime/index' -import { basePrompt, type HumanEvalTask, loadHumanEval } from './benchmarks/humaneval' - -function must(name: string): string { - const v = process.env[name] - if (!v) throw new Error(`env ${name} is required`) - return v -} - -const SMOKE = process.env.SMOKE === '1' -const DEV_N = Number(process.env.DEV_N ?? (SMOKE ? 6 : 60)) -const HOLD_N = Number(process.env.HOLD_N ?? (SMOKE ? 6 : 60)) -const GENERATIONS = Number(process.env.GENERATIONS ?? (SMOKE ? 1 : 2)) -const POPULATION = Number(process.env.POPULATION ?? (SMOKE ? 2 : 4)) -const CONCURRENCY = Number(process.env.CONCURRENCY ?? 8) -const DOCKER_CONCURRENCY = Number(process.env.DOCKER_CONCURRENCY ?? 6) -// Default worker: Qwen2.5-7B — the second model of the strategy's measured basis -// (Llama-3-8B/Qwen2.5-7B). The original smoke worker (Meta-Llama-3-8B-Instruct-Lite) -// and every other 8B Llama variant were retired from Together serverless -// (`model_not_available`, verified 2026-07); this is the closest live weak worker. -const MODEL = process.env.MODEL ?? 'Qwen/Qwen2.5-7B-Instruct-Turbo' -const BASE = process.env.ROUTER_BASE ?? 'https://api.together.xyz/v1' -const TEMP = Number(process.env.TEMPERATURE ?? 0.8) -const MAX_TOKENS = Number(process.env.MAX_TOKENS ?? 2500) -const DOLLARS = Number(process.env.DOLLARS ?? 15) -const RUN_DIR = process.env.RUN_DIR ?? join(tmpdir(), `live-improve-campaign-${Date.now()}`) - -const systemPrompt = 'You are an expert Python programmer.' -const dockerImage = 'python:3.12-slim' -const dockerTimeoutMs = Number(process.env.DOCKER_TIMEOUT_MS ?? 20000) - -// ── Docker: ONE semaphored --network=none exec channel for BOTH judges ─────────────── -// Visible checks (sandboxCheckRunner) and the hidden nonce judge each pipe a python -// program as `printf '%s' '' | base64 -d | python3 -`; every container passes -// through one global semaphore so task-level concurrency cannot stampede the daemon. - -let dockerInFlight = 0 -const dockerWaiters: Array<() => void> = [] -async function withDockerSlot(fn: () => Promise): Promise { - if (dockerInFlight >= DOCKER_CONCURRENCY) await new Promise((r) => dockerWaiters.push(r)) - dockerInFlight += 1 - try { - return await fn() - } finally { - dockerInFlight -= 1 - dockerWaiters.shift()?.() - } -} - -const containerPrefix = `lic-${process.pid}` -let containerSeq = 0 - -function reapContainers(): void { - try { - const ids = execFileSync('docker', ['ps', '-aq', '--filter', `name=${containerPrefix}`], { - timeout: 10000, - }) - .toString() - .trim() - if (ids) execFileSync('docker', ['rm', '-f', ...ids.split('\n')], { timeout: 15000 }) - } catch { - /* reaper is best-effort by design */ - } -} -process.on('SIGINT', () => { - reapContainers() - process.exit(130) -}) -process.on('SIGTERM', () => { - reapContainers() - process.exit(143) -}) - -const dockerBox: CheckExecChannel = { - exec(command, options) { - const timeoutMs = options?.timeoutMs ?? dockerTimeoutMs - return withDockerSlot( - () => - new Promise((resolve, reject) => { - const name = `${containerPrefix}-${containerSeq++}` - let settled = false - const reap = () => execFile('docker', ['rm', '-f', name], () => {}) - const finish = (r: { exitCode: number; stdout: string; stderr: string }) => { - if (settled) return - settled = true - clearTimeout(backstop) - reap() - resolve(r) - } - const fail = (e: Error) => { - if (settled) return - settled = true - clearTimeout(backstop) - reap() - reject(e) - } - // execFile's timeout kills the docker CLIENT; a hung container could leave - // the callback unfired. The backstop guarantees resolution and the named - // reap kills the stray container. - const backstop = setTimeout( - () => finish({ exitCode: 124, stdout: '', stderr: 'timed out (backstop)' }), - timeoutMs + 3000, - ) - execFile( - 'docker', - ['run', '--rm', '--name', name, '--network=none', '--cpus=1', '--memory=512m', dockerImage, 'sh', '-c', command], - { timeout: timeoutMs, killSignal: 'SIGKILL', maxBuffer: 4 * 1024 * 1024 }, - (err, stdout, stderr) => { - if (err) { - const e = err as NodeJS.ErrnoException & { code?: number | string } - if (e.code === 'ENOENT') { - fail(new Error('docker binary not found on PATH')) - return - } - if (/cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(stderr ?? '')) { - fail(new Error(`docker daemon unreachable: ${(stderr ?? '').slice(0, 200)}`)) - return - } - finish({ exitCode: typeof e.code === 'number' ? e.code : 1, stdout: stdout ?? '', stderr: stderr ?? '' }) - return - } - finish({ exitCode: 0, stdout: stdout ?? '', stderr: stderr ?? '' }) - }, - ) - }), - ) - }, -} - -// ── The hidden nonce judge (script-side, AFTER the strategy locks its artifact) ────── -// Pass requires the per-call nonce sentinel that check() prints AFTER succeeding — -// exit-0-before-check (a candidate calling sys.exit(0)) is a fail here, where trusting -// the exit code alone would score it a pass. Nothing from this run reaches the strategy. - -function buildHiddenProgram(task: HumanEvalTask, candidate: string, nonce: string): string { - return `${task.prompt}\n${candidate}\n\n${task.test}\n\ncheck(${task.entryPoint})\nprint("HIDDEN-${nonce} PASS")\n` -} - -async function runHiddenJudge( - task: HumanEvalTask, - candidate: string, -): Promise<{ pass: number; detail?: string }> { - const nonce = randomBytes(8).toString('hex') - const b64 = Buffer.from(buildHiddenProgram(task, candidate, nonce), 'utf8').toString('base64') - const r = await dockerBox.exec(`printf '%s' '${b64}' | base64 -d | python3 -`, { - timeoutMs: dockerTimeoutMs, - }) - if (r.exitCode === 0 && r.stdout.includes(`HIDDEN-${nonce} PASS`)) return { pass: 1 } - return { pass: 0, detail: (r.stderr || r.stdout).slice(-300) || 'timed out (no output)' } -} - -// ── Scenarios: fixed disjoint slices of HumanEval ───────────────────────────────────── - -interface HevScenario extends Scenario { - kind: 'humaneval' -} - -const taskById = new Map() - -// ── The real evaluator: one cell = one structuralRollout run + hidden grade ────────── - -interface CellArtifact { - taskId: string - policy: string - /** Hidden nonce-judge grade of the FINAL selected candidate: {0,1}. */ - pass: number - detail?: string - repairStop: string - shots: number - completions: number - authoredChecks: number - tokens: { input: number; output: number } - usd: number - ms: number -} - -interface ScoredCandidate { - candidate: string - outcome: CheckOutcome -} -function recordingRunner(inner: CheckRunner, log: ScoredCandidate[]): CheckRunner { - return { - async run(candidate, checks, ctx) { - const outcome = await inner.run(candidate, checks, ctx) - log.push({ candidate, outcome }) - return outcome - }, - } -} - -// Global spend meter (every cell of every phase — baseline, generations, holdout). -const spend = { cells: 0, llmCalls: 0, tokensIn: 0, tokensOut: 0, usd: 0, hiddenPass: 0 } - -async function evaluateCell( - surface: MutableSurface, - scenario: HevScenario, - ctx: DispatchContext, -): Promise { - const policy = parseRolloutPolicy(surface) - if (!policy) { - throw new Error(`agent: surface carries no valid rollout policy: ${String(surface).slice(0, 120)}`) - } - const task = taskById.get(scenario.id) - if (!task) throw new Error(`agent: unknown scenario id ${scenario.id}`) - - const scored: ScoredCandidate[] = [] - const strategy = structuralRollout({ - policy: { ...policy, temperature: TEMP }, - checkRunner: recordingRunner(sandboxCheckRunner({ box: dockerBox }), scored), - }) - // INERT check: the strategy's harness-verified score channel carries no hidden - // signal — hidden grading happens below, after the rollout locks its artifact. - const inertSurface = createVerifierEnvironment({ - name: 'humaneval-inert', - check: () => ({ passes: 0, total: 1, errored: 0 }), - }) - const result = (await runAgentic({ - surface: inertSurface, - task: { - id: task.taskId, - systemPrompt, - userPrompt: basePrompt(task), - meta: { entryPoint: task.entryPoint }, - }, - routerBaseUrl: BASE, - routerKey: must('TOGETHER_API_KEY'), - model: MODEL, - temperature: TEMP, - maxTokens: MAX_TOKENS, - innerTurns: 2, - strategy, - // The strategy's documented sizing: k samples + repair rounds + the check-author consult. - budget: policy.k + policy.repairRounds + 1, - })) as AgenticRunResult & StructuralRolloutResult - - // Backend integrity: report REAL usage on every cell (expectUsage 'assert' upstream). - ctx.cost.observe(result.usd, 'together') - ctx.cost.observeTokens(result.tokens) - - const winner = result.selection.find((r) => r.selected) - if (!winner) { - throw new Error(`${task.taskId}: no receipt marked selected (repairStop=${result.repairStop})`) - } - const rec = scored[winner.candidateIndex] - if (!rec) { - throw new Error( - `${task.taskId}: selected receipt #${winner.candidateIndex} has no recorded candidate (${scored.length} scored)`, - ) - } - - const hidden = await runHiddenJudge(task, rec.candidate) - - spend.cells += 1 - spend.llmCalls += result.completions - spend.tokensIn += result.tokens.input - spend.tokensOut += result.tokens.output - spend.usd += result.usd - spend.hiddenPass += hidden.pass - - const artifact: CellArtifact = { - taskId: task.taskId, - policy: serializeRolloutPolicy(policy), - pass: hidden.pass, - ...(hidden.detail ? { detail: hidden.detail } : {}), - repairStop: result.repairStop, - shots: result.shots, - completions: result.completions, - authoredChecks: result.authoredChecks, - tokens: result.tokens, - usd: result.usd, - ms: result.ms, - } - appendFileSync( - join(RUN_DIR, 'cells.jsonl'), - `${JSON.stringify({ cellId: ctx.cellId, generation: ctx.generation ?? null, ...artifact, detail: undefined })}\n`, - ) - console.log( - ` [cell ${String(spend.cells).padStart(3)}] ${task.taskId.padEnd(14)} ${artifact.policy.padEnd(38)} hidden=${hidden.pass ? 'PASS' : 'fail'} ${result.repairStop} calls=${result.completions}`, - ) - return artifact -} - -// The in-loop judge is a deterministic transcriber of the script-side hidden grade — -// the grading itself never runs inside the strategy or the proposer's view. -const hiddenJudge: JudgeConfig = { - name: 'hidden-nonce-judge', - dimensions: [ - { key: 'hidden', description: 'HumanEval hidden check() suite (docker --network=none, nonce sentinel)' }, - ], - score: ({ artifact }) => ({ - dimensions: { hidden: artifact.pass }, - composite: artifact.pass, - notes: artifact.pass ? 'hidden PASS' : `hidden fail: ${(artifact.detail ?? '').slice(0, 160)}`, - }), -} - -// ── Reporting helpers (read the library's own result objects; never re-decide) ─────── - -interface CampaignLike { - cells: Array<{ error?: string | null; judgeScores: Record }> -} -function passStats(campaign: CampaignLike): { passed: number; scored: number; errored: number; rate: number } { - let passed = 0 - let scored = 0 - let errored = 0 - for (const cell of campaign.cells) { - if (cell.error) { - errored += 1 - continue - } - scored += 1 - const scores = Object.values(cell.judgeScores) - const composite = scores.length === 0 ? 0 : scores.reduce((s, j) => s + j.composite, 0) / scores.length - if (composite >= 0.999) passed += 1 - } - return { passed, scored, errored, rate: scored > 0 ? passed / scored : 0 } -} -const pct = (x: number) => `${(100 * x).toFixed(1)}%` - -async function main(): Promise { - must('TOGETHER_API_KEY') - mkdirSync(RUN_DIR, { recursive: true }) - const started = Date.now() - - const all = await loadHumanEval(DEV_N + HOLD_N, 0) - if (all.length !== DEV_N + HOLD_N) { - throw new Error(`expected ${DEV_N + HOLD_N} tasks, loaded ${all.length}`) - } - const devTasks = all.slice(0, DEV_N) - const holdTasks = all.slice(DEV_N) - for (const t of all) taskById.set(t.taskId, t) - - const toScenario = (t: HumanEvalTask): HevScenario => ({ id: t.taskId, kind: 'humaneval' }) - const scenarios = all.map(toScenario) - const holdoutScenarios = holdTasks.map(toScenario) - - const baselinePolicy = { k: 5, repairRounds: 2, testgen: 6 } - const profile: AgentProfile = { - name: 'humaneval-structural-worker', - extensions: { [ROLLOUT_POLICY_EXTENSION]: baselinePolicy }, - } - - console.log('=== LIVE self-improvement campaign · improve() surface rollout-policy ===') - console.log(` worker: ${MODEL} @ ${BASE} (temp=${TEMP}, maxTokens=${MAX_TOKENS}, innerTurns=2)`) - console.log( - ` DEV slice : HumanEval index [0, ${DEV_N}) — ${devTasks[0]?.taskId} .. ${devTasks[devTasks.length - 1]?.taskId} (n=${devTasks.length})`, - ) - console.log( - ` HELD-OUT slice : HumanEval index [${DEV_N}, ${DEV_N + HOLD_N}) — ${holdTasks[0]?.taskId} .. ${holdTasks[holdTasks.length - 1]?.taskId} (n=${holdTasks.length})`, - ) - console.log(` baseline policy: ${JSON.stringify(baselinePolicy)}`) - console.log( - ` budget: generations=${GENERATIONS} population<=${POPULATION} reps=1 concurrency=${CONCURRENCY} docker<=${DOCKER_CONCURRENCY} ceiling=$${DOLLARS}`, - ) - console.log(` gate: library defaultProductionGate (paired bootstrap on held-out, ship iff CI.low > 0.05)`) - console.log(` runDir: ${RUN_DIR}`) - console.log(`\n profile BEFORE: ${JSON.stringify(profile)}\n`) - - const result = await improve(profile, [], { - surface: 'rollout-policy', - scenarios, - judge: hiddenJudge, - agent: evaluateCell, - budget: { - generations: GENERATIONS, - populationSize: POPULATION, - maxConcurrency: CONCURRENCY, - holdoutScenarios, - reps: 1, - dollars: DOLLARS, - }, - runDir: RUN_DIR, - // Deterministic proposer, empty findings channel: the improver's context is ONLY - // the DEV composites the loop accumulates — no distilled failure text, no trace - // paths, and (by the loop's own structure) never a held-out cell. - analyzeGeneration: null, - }) - - const loop = result.raw.raw - const wallMin = (Date.now() - started) / 60000 - - console.log('\n── DEV (train) results — what the improver saw ──') - const baseDev = passStats(loop.baselineCampaign) - console.log( - ` gen -1 baseline ${serializeRolloutPolicy(structuralRolloutPolicyFromProfile(profile)!).padEnd(38)} DEV ${baseDev.passed}/${baseDev.scored} = ${pct(baseDev.rate)} (errored ${baseDev.errored})`, - ) - for (const gen of loop.generations) { - const surfaceByHash = new Map(gen.surfaces.map((s) => [s.surfaceHash, s])) - const promotedHashes = new Set(gen.record.promoted) - for (const cand of gen.record.candidates) { - const s = surfaceByHash.get(cand.surfaceHash) - const stats = s ? passStats(s.campaign) : undefined - console.log( - ` gen ${String(gen.record.generationIndex).padStart(2)} ${String(cand.label ?? '').padEnd(16)} ${String(s?.surface ?? '?').padEnd(38)} DEV ${stats ? `${stats.passed}/${stats.scored} = ${pct(stats.rate)} (errored ${stats.errored})` : '?'}${promotedHashes.has(cand.surfaceHash) ? ' [promoted]' : ''}`, - ) - } - } - console.log(` training winner: ${String(loop.winnerSurface)}${loop.winnerLabel ? ` (${loop.winnerLabel})` : ''}`) - - console.log('\n── HELD-OUT gate — the library decides ──') - const baseHold = passStats(loop.baselineOnHoldout) - const winHold = passStats(loop.winnerOnHoldout) - console.log( - ` baseline on held-out : ${baseHold.passed}/${baseHold.scored} = ${pct(baseHold.rate)} (errored ${baseHold.errored})`, - ) - console.log( - ` winner on held-out : ${winHold.passed}/${winHold.scored} = ${pct(winHold.rate)} (errored ${winHold.errored})`, - ) - console.log(` gate decision: ${result.gateDecision.toUpperCase()} (lift ${result.lift >= 0 ? '+' : ''}${result.lift.toFixed(3)})`) - for (const reason of loop.gateResult.reasons) console.log(` reason: ${reason}`) - for (const g of loop.gateResult.contributingGates) { - console.log(` gate[${g.name}] passed=${g.passed} detail=${JSON.stringify(g.detail).slice(0, 300)}`) - } - - console.log('\n── ship/hold outcome ──') - console.log(` shipped: ${result.shipped}`) - console.log(` profile AFTER : ${JSON.stringify(result.profile)}`) - if (result.shipped) { - console.log(` policy change : ${serializeRolloutPolicy(structuralRolloutPolicyFromProfile(profile)!)} → ${serializeRolloutPolicy(structuralRolloutPolicyFromProfile(result.profile)!)}`) - } else { - console.log(' policy change : none (gate held — baseline policy stays)') - } - - console.log('\n── spend / provenance ──') - console.log( - ` cells ${spend.cells} · llm calls ${spend.llmCalls} · tokens ${spend.tokensIn} in / ${spend.tokensOut} out · router-priced $${spend.usd.toFixed(4)} · loop-reported $${result.raw.totalCostUsd.toFixed(4)}`, - ) - console.log(` wall ${wallMin.toFixed(1)} min · runDir ${RUN_DIR} (cells.jsonl + campaign cells + loop provenance)`) - reapContainers() - process.exit(0) -} - -main().catch((e) => { - console.error(e) - reapContainers() - process.exit(1) -}) diff --git a/bench/src/official-optimizer-config.mts b/bench/src/official-optimizer-config.mts index da7a4d7f..d060f203 100644 --- a/bench/src/official-optimizer-config.mts +++ b/bench/src/official-optimizer-config.mts @@ -27,7 +27,7 @@ function positiveInteger( export function requiredTokenPricing( env: NodeJS.ProcessEnv, - prefix: 'WORKER' | 'REFLECT', + prefix: string, ) { return { inputUsdPerMillion: requiredNonNegativeNumber( @@ -56,20 +56,22 @@ export function officialOptimizerModel(options: { apiKey: string maxCostUsd: number maxOutputTokensPerRequest: number + envPrefix?: string }) { const { env } = options + const envPrefix = options.envPrefix ?? 'REFLECT' return { model: options.model, baseUrl: options.baseUrl, apiKey: options.apiKey, budget: { maxCostUsd: options.maxCostUsd, - maxRequests: positiveInteger(env, 'REFLECT_MAX_REQUESTS', 100), - maxRequestBytes: positiveInteger(env, 'REFLECT_MAX_REQUEST_BYTES', 2_000_000), - maxResponseBytes: positiveInteger(env, 'REFLECT_MAX_RESPONSE_BYTES', 2_000_000), + maxRequests: positiveInteger(env, `${envPrefix}_MAX_REQUESTS`, 100), + maxRequestBytes: positiveInteger(env, `${envPrefix}_MAX_REQUEST_BYTES`, 2_000_000), + maxResponseBytes: positiveInteger(env, `${envPrefix}_MAX_RESPONSE_BYTES`, 2_000_000), maxOutputTokensPerRequest: options.maxOutputTokensPerRequest, - requestTimeoutMs: positiveInteger(env, 'REFLECT_REQUEST_TIMEOUT_MS', 300_000), - pricing: requiredTokenPricing(env, 'REFLECT'), + requestTimeoutMs: positiveInteger(env, `${envPrefix}_REQUEST_TIMEOUT_MS`, 300_000), + pricing: requiredTokenPricing(env, envPrefix), }, } } diff --git a/bench/src/official-optimizer-config.test.mts b/bench/src/official-optimizer-config.test.mts new file mode 100644 index 00000000..382100ce --- /dev/null +++ b/bench/src/official-optimizer-config.test.mts @@ -0,0 +1,88 @@ +import { describe, expect, it } from 'vitest' +import { + assertCompleteCost, + officialOptimizerModel, + requiredTokenPricing, +} from './official-optimizer-config.mts' + +const pricingEnv = { + OPT_INPUT_USD_PER_MILLION: '1', + OPT_CACHED_INPUT_USD_PER_MILLION: '0.1', + OPT_CACHE_WRITE_USD_PER_MILLION: '1.25', + OPT_OUTPUT_USD_PER_MILLION: '5', +} + +describe('official optimizer configuration', () => { + it('builds a bounded model configuration from an arbitrary environment prefix', () => { + const model = officialOptimizerModel({ + env: { + ...pricingEnv, + OPT_MAX_REQUESTS: '7', + OPT_MAX_REQUEST_BYTES: '1000', + OPT_MAX_RESPONSE_BYTES: '2000', + OPT_REQUEST_TIMEOUT_MS: '3000', + }, + envPrefix: 'OPT', + model: 'test-model', + baseUrl: 'http://127.0.0.1:8080/v1', + apiKey: 'test-key', + maxCostUsd: 2, + maxOutputTokensPerRequest: 4000, + }) + + expect(model).toEqual({ + model: 'test-model', + baseUrl: 'http://127.0.0.1:8080/v1', + apiKey: 'test-key', + budget: { + maxCostUsd: 2, + maxRequests: 7, + maxRequestBytes: 1000, + maxResponseBytes: 2000, + maxOutputTokensPerRequest: 4000, + requestTimeoutMs: 3000, + pricing: { + inputUsdPerMillion: 1, + cachedInputUsdPerMillion: 0.1, + cacheWriteUsdPerMillion: 1.25, + outputUsdPerMillion: 5, + }, + }, + }) + }) + + it('requires every token price instead of inventing cost data', () => { + expect(() => + requiredTokenPricing( + { + ...pricingEnv, + OPT_OUTPUT_USD_PER_MILLION: undefined, + }, + 'OPT', + ), + ).toThrow('env OPT_OUTPUT_USD_PER_MILLION is required') + }) + + it('rejects invalid request limits before constructing the optimizer', () => { + expect(() => + officialOptimizerModel({ + env: { ...pricingEnv, OPT_MAX_REQUESTS: '0' }, + envPrefix: 'OPT', + model: 'test-model', + baseUrl: 'http://127.0.0.1:8080/v1', + apiKey: 'test-key', + maxCostUsd: 2, + maxOutputTokensPerRequest: 4000, + }), + ).toThrow('env OPT_MAX_REQUESTS must be a positive integer') + }) + + it('rejects incomplete cost records', () => { + expect(() => + assertCompleteCost('official optimizer', { + accountingComplete: false, + incompleteReasons: ['provider omitted usage'], + }), + ).toThrow('official optimizer: cost accounting is incomplete: provider omitted usage') + }) +}) diff --git a/bench/src/profiles.ts b/bench/src/profiles.ts index 7830ef14..09c43476 100644 --- a/bench/src/profiles.ts +++ b/bench/src/profiles.ts @@ -5,8 +5,8 @@ * "operator type" or "analyst type" — there are profiles + the operator toolbox they use to manage * each other (in-process via the Scope, in a sandbox via the same verbs exposed as MCP tools). * - * The directory is OPTIMIZED over time by the agent-eval RSI loop (`runImprovementLoop`): `gepaProposer` - * evolves the prompts, a `skillOptProposer` evolves `skills`, Pareto-tracked + holdout-gated. + * Complete agent-eval methods optimize these profiles over explicit train, + * selection, and final-test partitions. * The trace-analyst's findings are the optimizer's input — the loop improves the profile that the * findings say is weak. */ diff --git a/bench/src/swe-arena/activation.mts b/bench/src/swe-arena/activation.mts index fb181c5f..ea8f53f5 100644 --- a/bench/src/swe-arena/activation.mts +++ b/bench/src/swe-arena/activation.mts @@ -26,7 +26,6 @@ import { run } from './proc.ts' export const ACTIVATION_PREDICATE_RELPATH = '.improve/activation.json' export interface ActivationPredicate { - version: 'v1' /** One sentence: which mechanism this proves fired. */ description: string kind: 'grep' | 'script' @@ -53,7 +52,6 @@ export function parseActivationPredicate(raw: string): ParsedPredicate { return { ok: false, error: 'must be a JSON object' } } const p = value as Record - if (p.version !== 'v1') return { ok: false, error: `version must be "v1", got ${JSON.stringify(p.version)}` } if (typeof p.description !== 'string' || p.description.trim().length === 0) { return { ok: false, error: 'description must be a non-empty string' } } @@ -90,13 +88,12 @@ export function activationPredicateInstruction(): string { '', 'Template (kind "grep" — a RegExp tested over the run artifacts):', ' {', - ' "version": "v1",', ' "description": "patchRiskWarnings emitted in at least one settle",', ' "kind": "grep",', ' "pattern": "patchRiskWarnings|patch-risk",', ' "files": ["journal.jsonl", "workers/"]', ' }', - 'Or kind "script": {"version":"v1","description":"...","kind":"script","script":"grep -rq NEW_SECTION ws/.loops"}', + 'Or kind "script": {"description":"...","kind":"script","script":"grep -rq NEW_SECTION ws/.loops"}', '(exit 0 in any run dir = fired).', 'Pick a pattern that can ONLY appear when your mechanism ran — not one that matches the diff itself.', ].join('\n') diff --git a/bench/src/swe-arena/activation.test.mts b/bench/src/swe-arena/activation.test.mts index 09dc3a19..1d445873 100644 --- a/bench/src/swe-arena/activation.test.mts +++ b/bench/src/swe-arena/activation.test.mts @@ -26,7 +26,6 @@ describe('parseActivationPredicate', () => { it('accepts a valid grep predicate (with optional files filters)', () => { const parsed = parseActivationPredicate( JSON.stringify({ - version: 'v1', description: 'patchRiskWarnings emitted in at least one settle', kind: 'grep', pattern: 'patchRiskWarnings|patch-risk', @@ -38,7 +37,7 @@ describe('parseActivationPredicate', () => { it('accepts a valid script predicate', () => { const parsed = parseActivationPredicate( - JSON.stringify({ version: 'v1', description: 'x', kind: 'script', script: 'grep -rq X .' }), + JSON.stringify({ description: 'x', kind: 'script', script: 'grep -rq X .' }), ) expect(parsed.ok).toBe(true) }) @@ -46,13 +45,12 @@ describe('parseActivationPredicate', () => { it.each([ ['not json', 'not valid JSON'], ['[]', 'JSON object'], - [JSON.stringify({ version: 'v2', description: 'x', kind: 'grep', pattern: 'a' }), 'version'], - [JSON.stringify({ version: 'v1', description: '', kind: 'grep', pattern: 'a' }), 'description'], - [JSON.stringify({ version: 'v1', description: 'x', kind: 'grep' }), 'pattern'], - [JSON.stringify({ version: 'v1', description: 'x', kind: 'grep', pattern: '(' }), 'RegExp'], - [JSON.stringify({ version: 'v1', description: 'x', kind: 'grep', pattern: 'a', files: [1] }), 'files'], - [JSON.stringify({ version: 'v1', description: 'x', kind: 'script' }), 'script'], - [JSON.stringify({ version: 'v1', description: 'x', kind: 'sql' }), 'kind'], + [JSON.stringify({ description: '', kind: 'grep', pattern: 'a' }), 'description'], + [JSON.stringify({ description: 'x', kind: 'grep' }), 'pattern'], + [JSON.stringify({ description: 'x', kind: 'grep', pattern: '(' }), 'RegExp'], + [JSON.stringify({ description: 'x', kind: 'grep', pattern: 'a', files: [1] }), 'files'], + [JSON.stringify({ description: 'x', kind: 'script' }), 'script'], + [JSON.stringify({ description: 'x', kind: 'sql' }), 'kind'], ])('rejects %s', (raw, want) => { const parsed = parseActivationPredicate(raw) expect(parsed.ok).toBe(false) @@ -84,7 +82,6 @@ describe('runActivationPredicate', () => { }) const grep = (pattern: string, files?: string[]) => ({ - version: 'v1' as const, description: 'd', kind: 'grep' as const, pattern, @@ -127,7 +124,7 @@ describe('runActivationPredicate', () => { }) it('script kind fires on rc=0 in any run dir, not-fired otherwise', async () => { - const script = (s: string) => ({ version: 'v1' as const, description: 'd', kind: 'script' as const, script: s }) + const script = (s: string) => ({ description: 'd', kind: 'script' as const, script: s }) const hit = await runActivationPredicate(script('grep -q patchRiskWarnings driver.log'), [runA, runB]) expect(hit.fired).toBe(true) expect(hit.evidence[0]).toContain(runB) @@ -261,11 +258,11 @@ describe('activation-predicate prefilter', () => { await mkdir(join(args.worktreePath, '.improve'), { recursive: true }) await writeFile( join(args.worktreePath, ACTIVATION_PREDICATE_RELPATH), - JSON.stringify({ version: 'v1', description: 'd', kind: 'grep', pattern: 'x' }), + JSON.stringify({ description: 'd', kind: 'grep', pattern: 'x' }), ) } else if (proposer.name === 'invalid-predicate') { await mkdir(join(args.worktreePath, '.improve'), { recursive: true }) - await writeFile(join(args.worktreePath, ACTIVATION_PREDICATE_RELPATH), '{"version":"v1"}') + await writeFile(join(args.worktreePath, ACTIVATION_PREDICATE_RELPATH), '{}') } return { applied: true, summary: `${proposer.name} edit` } }, diff --git a/bench/src/swe-arena/gepa-seat.mts b/bench/src/swe-arena/gepa-seat.mts index 330a07f3..fb78c05c 100644 --- a/bench/src/swe-arena/gepa-seat.mts +++ b/bench/src/swe-arena/gepa-seat.mts @@ -1,5 +1,5 @@ /** - * GEN-6 GEPA proposer seat using agent-eval's official + * GEPA proposer seat using agent-eval's official * `gepaOptimizationMethod` as one author in the swe-arena fan-out. * * Two-tier evaluator, the critical shape: @@ -30,7 +30,7 @@ * (`gepa_bridge.py` `_validate_input`: `if "testSet" in value ... raise`). * This module never mentions holdout instances to begin with. * - * RUNTIME SEAM (fails at provenance time, before a candidate slot is used): + * OPTIONAL RUNTIME (fails at provenance time, before a candidate slot is used): * - Python: `agent_eval_rpc.gepa_bridge` + a GEPA build with * `optimize_anything`/`OptimizeAnythingConfig` must import — * `probeGepaRuntime` throws with the pip install instruction otherwise. @@ -54,6 +54,7 @@ import { type OptimizationMethodProvenance, type Scenario, } from '@tangle-network/agent-eval/campaign' +import { officialOptimizerModel } from '../official-optimizer-config.mts' import { ACTIVATION_PREDICATE_RELPATH, type ActivationPredicate } from './activation.mts' import { changeSpaceViolations, type OuterLoopConfig } from './outer-loop.mts' import type { AuthorFn, ProposerSpec, SmokeRunner, SmokeVerdict } from './proposer-fanout.mts' @@ -206,7 +207,7 @@ export function innerSmokeComposite(verdict: Pick { return { name: 'gepa-inner-smoke', - judgeVersion: 'gepa-inner-smoke.v1', + judgeVersion: 'gepa-inner-smoke', dimensions: [ { key: 'resolved', description: 'Official SWE-bench judge verdict for the smoke cell (1 resolved / 0 not).' }, { key: 'verifyPass', description: 'Committed verify fixture passed for the smoke cell (tiebreak).' }, @@ -552,7 +553,7 @@ function assertCompleteCost( } // --------------------------------------------------------------------------- -// Mechanical activation predicate (gen-5 activation gate). +// Mechanical activation predicate. // --------------------------------------------------------------------------- const escapeRegExp = (s: string): string => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') @@ -577,7 +578,6 @@ export function mechanicalActivationPredicate( if (added.length === 0) return null const line = added.reduce((a, b) => (b.length > a.length ? b : a)) return { - version: 'v1', description: `gepa-author surface change fired: candidate text from ${surface} appears in run artifacts`, kind: 'grep', pattern: escapeRegExp(line), @@ -594,8 +594,12 @@ export interface GepaSeatDeps { * split's public set; re-asserted here fail-closed). */ smokeInstanceId: string scoreSplit: Pick | null - /** Test seam. Default: agent-eval's official GEPA method. */ + /** Test override. Default: agent-eval's official GEPA method. */ methodFactory?: GepaMethodFactory + /** Explicit model override. Production otherwise resolves the metered model from env. */ + optimizer?: NonNullable< + GepaOptimizationMethodConfig['optimizer'] + > log?: (msg: string) => void } @@ -683,6 +687,24 @@ export function gepaSeatAuthor(config: OuterLoopConfig, deps: GepaSeatDeps): Aut const factory: GepaMethodFactory = deps.methodFactory ?? gepaOptimizationMethod + const optimizer = + deps.optimizer ?? + (deps.methodFactory + ? undefined + : officialOptimizerModel({ + env: process.env, + envPrefix: 'GEPA_OPTIMIZER', + model: process.env.GEPA_OPTIMIZER_MODEL ?? config.arm.driverModel, + baseUrl: + process.env.GEPA_OPTIMIZER_BASE_URL ?? + process.env.ROUTER_BASE ?? + 'https://router.tangle.tools/v1', + apiKey: process.env.GEPA_OPTIMIZER_API_KEY ?? process.env.TANGLE_API_KEY ?? '', + maxCostUsd: spec.maxProposerCostUsd ?? DEFAULT_MAX_PROPOSER_COST_USD, + maxOutputTokensPerRequest: Number( + process.env.GEPA_OPTIMIZER_MAX_OUTPUT_TOKENS ?? 16_384, + ), + })) const method = factory({ name: `gepa-seat:${spec.name}`, recipe, @@ -695,7 +717,8 @@ export function gepaSeatAuthor(config: OuterLoopConfig, deps: GepaSeatDeps): Aut ].join('|'), background, describeScenario: (scenario) => ({ id: scenario.id }), - // Ceiling, not expectation: every inner call is a real arm cell. + ...(optimizer ? { optimizer } : {}), + // Upper bound, not expectation: every inner call is a real arm cell. timeoutMs: budget * config.dispatchTimeoutMs, resume: 'if-compatible', trustResumeState: true, diff --git a/bench/src/swe-arena/gepa-seat.test.mts b/bench/src/swe-arena/gepa-seat.test.mts index c3e351ef..11a75ba7 100644 --- a/bench/src/swe-arena/gepa-seat.test.mts +++ b/bench/src/swe-arena/gepa-seat.test.mts @@ -1,6 +1,7 @@ import { spawnSync } from 'node:child_process' import { existsSync } from 'node:fs' import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises' +import { createServer } from 'node:http' import { tmpdir } from 'node:os' import { join } from 'node:path' import type { @@ -47,7 +48,7 @@ const seat = (over: Partial = {}): ProposerSpec => ({ // --------------------------------------------------------------------------- describe('validateGepaSeat', () => { - it('accepts the gen-6 draft seat shape and isGepaSeat discriminates on engine', () => { + it('accepts an engine seat and isGepaSeat discriminates on engine', () => { expect(() => validateGepaSeat(seat())).not.toThrow() expect(() => validateGepaSeat(seat({ engine: 'omni', maxMetricCalls: 8 }))).not.toThrow() expect(isGepaSeat(seat())).toBe(true) @@ -155,7 +156,7 @@ describe('inner smoke score', () => { expect(innerSmokeComposite(verdict({ resolved: false, verifyPass: true }))).toBe(0.25) expect(innerSmokeComposite(verdict({ resolved: true, verifyPass: false }))).toBe(1) expect(innerSmokeComposite(verdict({ resolved: true, verifyPass: true }))).toBe(1.25) - // Pre-gen-6 verdicts without the field score as no verify signal. + // Older verdicts without the field score as no verify signal. expect(innerSmokeComposite(verdict({ resolved: true }))).toBe(1) }) @@ -265,7 +266,7 @@ describe('captureProposerProvenance with a gepa seat', () => { // --------------------------------------------------------------------------- describe('mechanicalActivationPredicate', () => { - it('targets the longest added line and produces a parseable v1 grep predicate', () => { + it('targets the longest added line and produces a parseable grep predicate', () => { const seed = 'alpha\nshared line stays here\n' const winner = 'alpha\nshared line stays here\nAlways run the neighboring test file before finalizing.\nshort\n' const predicate = mechanicalActivationPredicate(seed, winner, SURFACE)! @@ -419,6 +420,24 @@ describe('recordGepaSeatInnerRun', () => { // --------------------------------------------------------------------------- const fakeCtx = {} as unknown as DispatchContext +const testOptimizer = { + model: 'optimizer-model', + baseUrl: 'http://127.0.0.1:1/v1', + apiKey: 'optimizer-key', + budget: { + maxCostUsd: 1, + maxRequests: 10, + maxRequestBytes: 100_000, + maxResponseBytes: 100_000, + maxOutputTokensPerRequest: 2_000, + pricing: { + inputUsdPerMillion: 1, + cachedInputUsdPerMillion: 0.1, + cacheWriteUsdPerMillion: 1.25, + outputUsdPerMillion: 5, + }, + }, +} const fullProvenance = ( runId = 'gepa-run', @@ -572,6 +591,7 @@ describe('fanOutLoopsGenerator with the gepa seat', () => { smokeInstanceId: 'astropy__astropy-13033', scoreSplit: { privateInstances: ['django__django-11532'] }, gepaMethodFactory: fakeGepaFactory([LOSER, WINNER], observed), + gepaOptimizer: testOptimizer, }) const result = await gen.generate(generatorArgs(0)) @@ -599,6 +619,7 @@ describe('fanOutLoopsGenerator with the gepa seat', () => { evaluationId: `swe-arena-gepa-seat|smoke=astropy__astropy-13033|judge=gepa-inner-smoke|dispatchTimeoutMs=${config.dispatchTimeoutMs}`, recipe: { kind: 'engine', run: { engine: 'gepa', maxEvaluations: 5 } }, + optimizer: testOptimizer, resume: 'if-compatible', trustResumeState: true, }) @@ -742,8 +763,8 @@ describe('fanOutLoopsGenerator with the gepa seat', () => { // bridge. The TypeScript adapter is a compile-time package dependency. // --------------------------------------------------------------------------- -const pythonBridgeReady = (): { ok: boolean; reason: string } => { - const probe = spawnSync(DEFAULT_GEPA_PYTHON, [ +const pythonBridgeReady = (python: string): { ok: boolean; reason: string } => { + const probe = spawnSync(python, [ '-c', 'import agent_eval_rpc.gepa_bridge; from gepa.optimize_anything import optimize_anything, OptimizeAnythingConfig', ]) @@ -754,15 +775,44 @@ const pythonBridgeReady = (): { ok: boolean; reason: string } => { } describe('integration: real adapter roundtrip', () => { - it('runs the bridge but rejects its unmetered winner when the full runtime is installed', async (ctx) => { - const python = pythonBridgeReady() - if (!python.ok) { - ctx.skip(`skip-with-reason: ${python.reason}`) + it('runs a metered optimizer through the real bridge and applies its winner', async (ctx) => { + const python = process.env.AGENT_EVAL_TEST_PYTHON ?? DEFAULT_GEPA_PYTHON + const pythonRuntime = pythonBridgeReady(python) + if (!pythonRuntime.ok) { + ctx.skip(`skip-with-reason: ${pythonRuntime.reason}`) return } - // Full runtime present: drive the REAL adapter with a stub smoke runner - // (no arm spend) over a synthetic tiny surface in a temp repo. + const winner = 'tiny synthetic surface\nAlways run the neighboring test before finalizing.' + const modelServer = createServer((_request, response) => { + response.writeHead(200, { 'content-type': 'application/json' }) + response.end( + JSON.stringify({ + choices: [ + { + message: { + role: 'assistant', + content: `\`\`\`\n${winner}\`\`\``, + }, + }, + ], + usage: { + prompt_tokens: 20, + completion_tokens: 20, + total_tokens: 40, + }, + }), + ) + }) + await new Promise((resolve, reject) => { + modelServer.once('error', reject) + modelServer.listen(0, '127.0.0.1', resolve) + }) + const address = modelServer.address() + if (!address || typeof address === 'string') { + throw new Error('test optimizer server did not bind') + } + const loopsRepo = await mkdtemp(join(tmpdir(), 'gepa-int-repo-')) const outDir = await mkdtemp(join(tmpdir(), 'gepa-int-out-')) try { @@ -776,24 +826,23 @@ describe('integration: real adapter roundtrip', () => { const driverWt = join(outDir, 'driver-wt') await runOk('git', ['-C', loopsRepo, 'worktree', 'add', '--detach', driverWt, 'HEAD']) - let innerCalls = 0 const gen = fanOutLoopsGenerator( { ...defaultRound4Config(), loopsRepo, outDir, populationSize: 1, - proposers: [seat({ maxMetricCalls: 4 })], + proposers: [seat({ maxMetricCalls: 10, python })], prefilter: { enabled: true, smokeInstance: 'cheapest-of-set', requireResolved: false }, }, { - smokeRunner: async () => { - innerCalls += 1 + smokeRunner: async ({ scratchPath }) => { + const candidate = await readFile(join(scratchPath, SURFACE), 'utf8') return { iid: 'astropy__astropy-13033', pass: true, reason: 'stub smoke (integration)', - resolved: innerCalls > 1, + resolved: candidate === winner, patchLines: 1, wallS: 0, verifyPass: true, @@ -801,28 +850,48 @@ describe('integration: real adapter roundtrip', () => { }, smokeInstanceId: 'astropy__astropy-13033', scoreSplit: null, + gepaOptimizer: { + model: 'test-optimizer', + baseUrl: `http://127.0.0.1:${address.port}/v1`, + apiKey: 'local-test-key', + budget: { + maxCostUsd: 1, + maxRequests: 10, + maxRequestBytes: 100_000, + maxResponseBytes: 100_000, + maxOutputTokensPerRequest: 2_000, + pricing: { + inputUsdPerMillion: 1, + cachedInputUsdPerMillion: 0.1, + cacheWriteUsdPerMillion: 1.25, + outputUsdPerMillion: 5, + }, + }, + }, }, ) - await expect( - gen.generate({ - worktreePath: driverWt, - report: undefined, - findings: [], - maxShots: 1, - signal: new AbortController().signal, - generation: 0, - candidateIndex: 0, - }), - ).rejects.toThrow(/cost accounting is incomplete/) - // The bridge ran and its result was retained even though activation was refused. + const result = await gen.generate({ + worktreePath: driverWt, + report: undefined, + findings: [], + maxShots: 1, + signal: new AbortController().signal, + generation: 0, + candidateIndex: 0, + }) const inner = JSON.parse( await readFile(join(outDir, 'gepa-seat', 'gen0-gepa-author', 'inner-provenance.json'), 'utf8'), ) + expect(result.applied, `${result.summary}\n${JSON.stringify(inner.innerScores, null, 2)}`).toBe(true) + expect(await readFile(join(driverWt, SURFACE), 'utf8')).toBe(winner) expect(inner.innerCallCount).toBeGreaterThanOrEqual(1) - expect(inner.accountingComplete).toBe(false) - expect(inner.incompleteReasons.length).toBeGreaterThan(0) - expect((await runOk('git', ['-C', driverWt, 'status', '--porcelain'])).stdout.trim()).toBe('') + expect(inner.accountingComplete).toBe(true) + expect(inner.incompleteReasons).toEqual([]) + expect(inner.tokenUsage.calls).toBeGreaterThan(0) } finally { + await new Promise((resolve, reject) => + modelServer.close((error) => (error ? reject(error) : resolve())), + ) await rm(outDir, { recursive: true, force: true }) await rm(loopsRepo, { recursive: true, force: true }) } diff --git a/bench/src/swe-arena/lineage-record.mts b/bench/src/swe-arena/lineage-record.mts deleted file mode 100644 index 640b3080..00000000 --- a/bench/src/swe-arena/lineage-record.mts +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Gen-5 lineage DAG wiring (SOTA adoption #3) — the hand-rolled parent fields - * stop being the only ancestry record: every candidate becomes a - * `LineageNode` in agent-eval's Lineage DAG (campaign/lineage.ts — - * multi-parent merges, tracks, deterministic content-hash ids, append-only - * fsLineageStore), every generation appends, and the library's - * `heuristicGovernor` reads the graph to decide per-generation continuation - * (extend the winner / branch on stall / merge frontier tips / prune). The - * governor's decision is RECORDED in the round summary — the operator (or a - * future automated outer loop) acts on it; this run never acts on it itself. - * - * Store location: `/.evolve/lineage.jsonl` (an .evolve-compatible - * path). The existing staircase rows are UNCHANGED and still written — the - * observatory depends on them; the DAG is additive. - * - * Node semantics: - * - the BASELINE incumbent is the root (track 'baseline', proposer - * 'baseline'), - * - configured Pareto parents (prior-run frontier commits) become nodes off - * the root under their own tracks, so a merge can name them as parents, - * - each evaluated candidate is a node under its proposer-named track: - * single-parent (root) for regular authors, MULTI-PARENT (the Pareto - * parent nodes) for the merge seat, - * - score = combined resolved fraction; scoreVector = per-instance - * fail-closed AND-verdicts over the lexicographically sorted instance ids. - */ - -import { join } from 'node:path' -import { - Lineage, - fsLineageStore, - heuristicGovernor, - lineageNodeId, - type GovernorOp, - type LineageNode, -} from '@tangle-network/agent-eval/campaign' - -export const LINEAGE_STORE_RELPATH = join('.evolve', 'lineage.jsonl') - -export interface LineageCandidateInput { - /** Proposer name — the node's track and proposer. */ - label: string - /** Loops commit; null (never-committed candidate) is skipped with a note. */ - commit: string | null - resolvedCount: number - /** Per-instance AND-verdicts (missing instance = fail-closed false). */ - verdicts: Record - merge: boolean - /** Staircase verdict — recorded as the node's rationale. */ - verdict: string -} - -export interface LineageRecordArgs { - outDir: string - runId: string - instances: readonly string[] - baseline: { commit: string; resolvedCount: number; verdicts: Record } - paretoParents: Array<{ label: string; commit: string; resolvedInstances: string[] }> - candidates: LineageCandidateInput[] -} - -export interface LineageRecordResult { - path: string - /** Node ids appended by THIS call (idempotent re-runs append zero). */ - appended: string[] - /** Nodes skipped because the candidate never became a commit. */ - skipped: string[] - governor: GovernorOp - nodesTotal: number -} - -const scoreVectorOf = (instances: readonly string[], verdicts: Record): number[] => - [...instances].sort().map((iid) => (verdicts[iid] === true ? 1 : 0)) - -const scoreOf = (resolvedCount: number, instanceCount: number): number => - instanceCount > 0 ? resolvedCount / instanceCount : 0 - -/** Record one generation into the lineage DAG and ask the governor for the - * continuation decision. Append-only + idempotent: node ids are content - * hashes, so re-recording an identical generation appends nothing. */ -export async function recordLineageGeneration(args: LineageRecordArgs): Promise { - const path = join(args.outDir, LINEAGE_STORE_RELPATH) - const store = fsLineageStore(path) - const lineage = await store.load() - const appended: string[] = [] - const skipped: string[] = [] - - const ensure = async (input: Parameters[0]): Promise => { - const id = lineageNodeId({ - parentIds: input.parentIds, - track: input.track, - surface: input.surface, - proposer: input.proposer, - }) - const existed = lineage.all().some((n) => n.id === id) - const node = lineage.addNode(input) - if (!existed) { - await store.append(node) - appended.push(node.id) - } - return node - } - - const n = args.instances.length - const root = await ensure({ - parentIds: [], - track: 'baseline', - surface: `loops@${args.baseline.commit}`, - score: scoreOf(args.baseline.resolvedCount, n), - scoreVector: scoreVectorOf(args.instances, args.baseline.verdicts), - proposer: 'baseline', - }) - - // Prior-run frontier commits become referenceable parent nodes. - const parentNodeByLabel = new Map() - for (const parent of args.paretoParents) { - const verdicts: Record = {} - for (const iid of parent.resolvedInstances) verdicts[iid] = true - const node = await ensure({ - parentIds: [root.id], - track: parent.label, - surface: `loops@${parent.commit}`, - score: scoreOf(parent.resolvedInstances.length, n), - scoreVector: scoreVectorOf(args.instances, verdicts), - proposer: parent.label, - }) - parentNodeByLabel.set(parent.label, node) - } - - for (const cand of args.candidates) { - if (cand.commit === null) { - skipped.push(cand.label) - continue - } - const parentIds = cand.merge - ? [...parentNodeByLabel.values()].map((p) => p.id) - : [root.id] - if (cand.merge && parentIds.length < 2) { - throw new Error( - `lineage: merge candidate ${cand.label} needs >=2 pareto parent nodes, got ${parentIds.length}`, - ) - } - await ensure({ - parentIds, - track: cand.label, - surface: `loops@${cand.commit}`, - score: scoreOf(cand.resolvedCount, n), - scoreVector: scoreVectorOf(args.instances, cand.verdicts), - proposer: cand.merge ? 'merge' : cand.label, - rationale: `run ${args.runId}: ${cand.verdict}`, - }) - } - - // The continuation decision — recorded, not acted on here. budgetRemaining=1: - // the operator approves one next generation at a time. - const governor = await heuristicGovernor().decide({ - lineage, - step: lineage.all().length, - budgetRemaining: 1, - prunedTracks: [], - }) - - return { path, appended, skipped, governor, nodesTotal: lineage.all().length } -} diff --git a/bench/src/swe-arena/lineage-record.test.mts b/bench/src/swe-arena/lineage-record.test.mts deleted file mode 100644 index ba8a1b21..00000000 --- a/bench/src/swe-arena/lineage-record.test.mts +++ /dev/null @@ -1,115 +0,0 @@ -/** - * Gen-5 lineage DAG wiring: baseline root + pareto-parent nodes + candidate - * nodes (multi-parent merge), append-only idempotence at the - * .evolve-compatible store, and the recorded governor decision. - */ - -import { readFile, rm } from 'node:fs/promises' -import { mkdtemp } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { Lineage } from '@tangle-network/agent-eval/campaign' -import { LINEAGE_STORE_RELPATH, recordLineageGeneration, type LineageRecordArgs } from './lineage-record.mts' - -const SIX = ['a-1', 'b-2', 'c-3', 'd-4', 'e-5', 'f-6'] - -describe('recordLineageGeneration', () => { - let outDir: string - - beforeEach(async () => { - outDir = await mkdtemp(join(tmpdir(), 'lineage-rec-')) - }) - - afterEach(async () => { - await rm(outDir, { recursive: true, force: true }) - }) - - const args = (): LineageRecordArgs => ({ - outDir, - runId: 'r4-test', - instances: SIX, - baseline: { commit: 'base'.repeat(10), resolvedCount: 1, verdicts: { 'a-1': true } }, - paretoParents: [ - { label: 'default-author', commit: 'p1p1'.repeat(10), resolvedInstances: ['d-4', 'f-6'] }, - { label: 'prompts-author', commit: 'p2p2'.repeat(10), resolvedInstances: ['b-2', 'd-4'] }, - ], - candidates: [ - { - label: 'claude-author', - commit: 'c1c1'.repeat(10), - resolvedCount: 2, - verdicts: { 'a-1': true, 'd-4': true }, - merge: false, - verdict: 'accepted', - }, - { - label: 'merge-author', - commit: 'c2c2'.repeat(10), - resolvedCount: 3, - verdicts: { 'a-1': true, 'b-2': true, 'd-4': true }, - merge: true, - verdict: 'accepted', - }, - { - label: 'killed-author', - commit: null, - resolvedCount: 0, - verdicts: {}, - merge: false, - verdict: 'rejected-prefilter', - }, - ], - }) - - it('records root + parents + candidates at the .evolve-compatible store; merge is MULTI-PARENT', async () => { - const result = await recordLineageGeneration(args()) - expect(result.path).toBe(join(outDir, LINEAGE_STORE_RELPATH)) - expect(result.path).toContain('.evolve') - // root + 2 parents + 2 committed candidates (killed-author skipped). - expect(result.nodesTotal).toBe(5) - expect(result.appended).toHaveLength(5) - expect(result.skipped).toEqual(['killed-author']) - - const raw = await readFile(result.path, 'utf8') - const lineage = Lineage.fromJSONL(raw) - const root = lineage.roots() - expect(root).toHaveLength(1) - expect(root[0]!.track).toBe('baseline') - expect(root[0]!.score).toBeCloseTo(1 / 6) - - const merge = lineage.all().find((n) => n.track === 'merge-author')! - expect(merge.parentIds).toHaveLength(2) - const parentTracks = merge.parentIds.map((id) => lineage.all().find((n) => n.id === id)!.track).sort() - expect(parentTracks).toEqual(['default-author', 'prompts-author']) - expect(merge.proposer).toBe('merge') - expect(merge.score).toBeCloseTo(3 / 6) - - const single = lineage.all().find((n) => n.track === 'claude-author')! - expect(single.parentIds).toEqual([root[0]!.id]) - expect(single.rationale).toContain('accepted') - // scoreVector: sorted instance order, fail-closed 0 for missing verdicts. - expect(single.scoreVector).toEqual([1, 0, 0, 1, 0, 0]) - }) - - it('is idempotent: re-recording the identical generation appends ZERO nodes', async () => { - const first = await recordLineageGeneration(args()) - const second = await recordLineageGeneration(args()) - expect(first.appended).toHaveLength(5) - expect(second.appended).toHaveLength(0) - expect(second.nodesTotal).toBe(5) - const raw = await readFile(second.path, 'utf8') - expect(raw.trim().split('\n')).toHaveLength(5) - }) - - it('returns a governor continuation decision (recorded, never acted on here)', async () => { - const result = await recordLineageGeneration(args()) - expect(['extend', 'branch', 'merge', 'prune', 'stop']).toContain(result.governor.op) - }) - - it('fails loud when a merge candidate has fewer than 2 parent nodes', async () => { - const bad = args() - bad.paretoParents = bad.paretoParents.slice(0, 1) - await expect(recordLineageGeneration(bad)).rejects.toThrow(/needs >=2/) - }) -}) diff --git a/bench/src/swe-arena/outer-loop.mts b/bench/src/swe-arena/outer-loop.mts index 8223f0d1..bba0b4cd 100644 --- a/bench/src/swe-arena/outer-loop.mts +++ b/bench/src/swe-arena/outer-loop.mts @@ -17,7 +17,7 @@ * `rawTraceDistiller` path-context so the coding agent also greps the raw * traces itself (`rawTraceContext: true` names the mechanism; an explicit * `analyzeGeneration` wins, so the distiller is composed in directly). - * (b) PROPOSE — `improvementDriver` + a change-space-constrained + * (b) PROPOSE — Runtime's code candidate driver + a change-space-constrained * `agenticGenerator` edit an isolated git worktree of loops. The DECLARED * CHANGE-SPACE is enforced twice: in the generator's verifier (feedback → * next shot) and fail-closed in the dispatch below (an out-of-space @@ -152,7 +152,6 @@ import { type ScoreSplit, type ScoreSplitConfig, } from './score-split.mts' -import { recordLineageGeneration, type LineageCandidateInput } from './lineage-record.mts' import { campaignCoordsFromCellPath, createSettleCapture, @@ -640,7 +639,7 @@ export interface OuterLoopConfig { * When set, `populationSize` MUST equal `proposers.length` (one candidate * slot per proposer — enforced at launch). Unset = the legacy * single-author generator (`proposerHarness` + bare invocation). - * GEN-6: a spec with `engine` set is a GEPA seat (gepa-seat.mts) — the + * A spec with `engine` set is a GEPA seat (gepa-seat.mts); the * agent-eval external-GEPA adapter optimizes ONE change-space file as a * string against the pre-filter smoke cell; requires `prefilter.enabled`. */ proposers?: ProposerSpec[] @@ -681,10 +680,6 @@ export interface OuterLoopConfig { * emit tangle.rollout.v1 lines live after each cell judges, with label-v2 * rewards. Default path: /rollout-ledger.jsonl. */ rolloutLedger?: { enabled: boolean; path?: string; opencodeDb?: string } - /** GEN-5 lineage DAG (lineage-record.mts): record every candidate as a - * LineageNode at /.evolve/lineage.jsonl and put the governor's - * continuation decision in the round summary. */ - lineage?: boolean /** GEN-5 evidence map: prior run outDirs whose arm-runs/judge/candidate * evidence the authors may mine (rendered into the evidence index). */ priorEvidenceDirs?: string[] @@ -966,9 +961,6 @@ export function defaultGen4Config( // never-fired mechanisms are quarantined even on an improved score. // 4. SETTLE-TIME ROLLOUT LEDGER — tangle.rollout.v1 lines live per cell, // label v2 (contribution-aware workers, baseline-relative proposers). -// 5. LINEAGE DAG — agent-eval Lineage at /.evolve/lineage.jsonl + -// governor continuation decision in the round summary; staircase rows -// unchanged (observatory contract). // --------------------------------------------------------------------------- export function defaultGen5Config( @@ -985,7 +977,6 @@ export function defaultGen5Config( briefing: AUTHOR_BRIEFING_VERSION, activationGate: true, rolloutLedger: { enabled: true }, - lineage: true, priorEvidenceDirs: [join(hh, 'gen4'), join(hh, 'gen3')], } } @@ -1632,7 +1623,7 @@ export async function runRound(config: OuterLoopConfig, signal?: AbortSignal): P resolved: outcome.resolved, patchLines: outcome.armRes.patch_lines, wallS, - // GEN-6: the GEPA seat's inner-score tiebreak. + // The GEPA seat's inner-score tiebreak. verifyPass: outcome.armRes.verify_pass, } await mkdir(armOutDir, { recursive: true }) @@ -2031,7 +2022,7 @@ export async function runRound(config: OuterLoopConfig, signal?: AbortSignal): P ...(smokeRunner ? { smokeRunner } : {}), ...(paretoParents.length > 0 ? { parents: paretoParents } : {}), ...(briefingCtx !== undefined ? { briefing: briefingCtx } : {}), - // GEN-6: the GEPA seat's inner evaluator rides the SAME public-only + // The GEPA seat's inner evaluator uses the same public-only // smoke instance; the split guards the never-surfaced invariant at // the bridge boundary too. ...(smokeIid !== null ? { smokeInstanceId: smokeIid } : {}), @@ -2047,7 +2038,7 @@ export async function runRound(config: OuterLoopConfig, signal?: AbortSignal): P const profile = { name: 'loops-pi-supervisor' } as Parameters[0] signal?.throwIfAborted() log(`round ${config.round} runId=${runId}: improve(surface:'code') over ${config.loopsRepo}@${config.loopsBaseRef}`) - const result = await improve(profile, [], { + const result = await improve(profile, { surface: 'code', // analyzeGeneration wins over this flag; the composite above embeds // rawTraceDistiller directly so the raw-trace mechanism stays active. @@ -2131,10 +2122,8 @@ export async function runRound(config: OuterLoopConfig, signal?: AbortSignal): P } } - // Collected per-candidate facts for the gen-5 machinery (activation by - // surface hash for the winner brief, lineage nodes, proposer v2 rewards). + // Collected per-candidate facts for activation and proposer rewards. const activationBySurface = new Map() - const lineageCandidates: LineageCandidateInput[] = [] interface ProposerOutcomeFact { generation: number candidateIndex: number @@ -2278,14 +2267,6 @@ export async function runRound(config: OuterLoopConfig, signal?: AbortSignal): P }) const label = cand.label ?? cs?.candidateCommit?.slice(0, 10) ?? cand.surfaceHash.slice(0, 10) - lineageCandidates.push({ - label, - commit: cs?.candidateCommit ?? null, - resolvedCount: candResolved, - verdicts, - merge: config.proposers?.find((p) => p.name === cand.label)?.merge === true, - verdict, - }) proposerFacts.push({ generation: g, candidateIndex: candIndex, @@ -2374,42 +2355,6 @@ export async function runRound(config: OuterLoopConfig, signal?: AbortSignal): P } } - // GEN-5 lineage DAG: record baseline root + pareto parents + every - // evaluated candidate (multi-parent for the merge seat) at the - // .evolve-compatible store, and ask the governor for the continuation - // decision (recorded below — never acted on inside this run). - let lineageResult: Awaited> | null = null - if (config.lineage === true) { - try { - const baselineCommit = - baselineCells.find((c) => c.artifact !== null && c.artifact.kind === 'swe-arm')?.artifact?.commit ?? - config.loopsBaseRef - lineageResult = await recordLineageGeneration({ - outDir: config.outDir, - runId, - instances: config.instances, - baseline: { - commit: baselineCommit, - resolvedCount: measuredBaselineCount, - verdicts: instanceVerdictsFromCells(baselineCells, config.instances, reps), - }, - paretoParents: (config.paretoParents ?? []).map((p) => ({ - label: p.label, - commit: p.commit, - resolvedInstances: p.resolvedInstances, - })), - candidates: lineageCandidates, - }) - log( - `lineage: ${lineageResult.appended.length} node(s) appended (total ${lineageResult.nodesTotal}) → ` + - `${lineageResult.path}; governor decision: ${JSON.stringify(lineageResult.governor)}` + - `${lineageResult.skipped.length > 0 ? `; skipped (no commit): ${lineageResult.skipped.join(', ')}` : ''}`, - ) - } catch (cause) { - log(`lineage: recording FAILED: ${(cause as Error).message}`) - } - } - const winnerSurface = result.raw.winner.surface const winnerCs = typeof winnerSurface === 'object' && winnerSurface !== null && winnerSurface.kind === 'code' @@ -2535,17 +2480,6 @@ export async function runRound(config: OuterLoopConfig, signal?: AbortSignal): P : { version: AUTHOR_BRIEFING_VERSION, indexPath: briefingCtx.indexPath, textSource: briefingCtx.briefingSource }, // GEN-5: settle-time rollout ledger location (tangle.rollout.v1, label v2). rolloutLedger: settleCapture === null ? null : { path: settleCapture.path, capture: 'settle-time', labels: 'v2' }, - // GEN-5: lineage DAG + the governor's recorded continuation decision. - lineage: - lineageResult === null - ? null - : { - path: lineageResult.path, - nodesTotal: lineageResult.nodesTotal, - appended: lineageResult.appended.length, - skipped: lineageResult.skipped, - governor: lineageResult.governor, - }, // Honest run-wide spend from the lib's CostLedger: per-channel rollups // (agent = arm cells, judge = official-judge calls, driver = proposer // shots), token totals, and accounting-completeness flags. diff --git a/bench/src/swe-arena/proposer-fanout.mts b/bench/src/swe-arena/proposer-fanout.mts index 5eea50cc..80a223cc 100644 --- a/bench/src/swe-arena/proposer-fanout.mts +++ b/bench/src/swe-arena/proposer-fanout.mts @@ -64,7 +64,13 @@ import { parseActivationPredicate, } from './activation.mts' import { briefingPromptSection, type BriefingContext } from './briefing.mts' -import { gepaSeatAuthor, isGepaSeat, validateGepaSeat, type GepaMethodFactory } from './gepa-seat.mts' +import { + gepaSeatAuthor, + isGepaSeat, + validateGepaSeat, + type GepaMethodFactory, + type GepaSeatDeps, +} from './gepa-seat.mts' import type { ScoreSplit } from './score-split.mts' import { run, runOk } from './proc.ts' @@ -78,7 +84,7 @@ export interface ProposerSpec { /** Path to an `AgentProfile` JSON. Absolute, or relative to this module's * `profiles/` directory. Omitted = bare profile (legacy invocation). */ profile?: string - /** Required for harness-authored seats. ABSENT on a GEN-6 engine seat + /** Required for harness-authored seats. Absent on an engine seat * (`engine` set) — enforced both ways at generator construction. */ harness?: 'claude' | 'codex' | 'opencode' /** GEN-4 pinned model id, threaded to the harness CLI as `-m ` via @@ -98,21 +104,20 @@ export interface ProposerSpec { /** Which diagnosis findings this proposer sees. Protocol/steering and * raw-trace-context findings always pass through. Default 'all'. */ diagnosisSlice?: 'all' | 'mechanics' | 'prompts' - /** GEN-6 GEPA seat: this seat is an ENGINE invocation (agent-eval's + /** GEPA seat: this seat is an engine invocation (agent-eval's * external-GEPA adapter), not a harness CLI. `gepa` = one bounded engine - * run; `omni` = GEPA's published best-of-then-continue shape. Validation + + * run; `omni` = GEPA's official Omni recipe. Validation + * authoring live in gepa-seat.mts. */ engine?: 'gepa' | 'omni' - /** GEN-6 GEPA seat: the ONE repo-relative change-space file GEPA optimizes + /** GEPA seat: the one repo-relative change-space file GEPA optimizes * as a string; the rest of the loops repo stays at the incumbent commit. */ surface?: string - /** GEN-6 GEPA seat: total inner-evaluation budget (each inner call is one + /** GEPA seat: total inner-evaluation budget (each inner call is one * real smoke arm cell). Default 10. */ maxMetricCalls?: number - /** GEN-6 GEPA seat: requested GEPA proposer spend cap in USD (the adapter - * reports GEPA's own model/CLI spend without an agent-eval receipt). */ + /** GEPA seat: hard cap for metered optimizer-model spend. */ maxProposerCostUsd?: number - /** GEN-6 GEPA seat: python executable for the bridge. Default 'python3'. */ + /** GEPA seat: Python executable for the bridge. Default 'python3'. */ python?: string } @@ -137,8 +142,8 @@ export interface SmokeVerdict { resolved: boolean | null patchLines: number wallS: number - /** Committed verify fixture passed (GEN-6: the GEPA seat's inner-score - * tiebreak). Absent on errored smokes and pre-gen-6 records. */ + /** Committed verify fixture passed, used as the GEPA seat's inner-score + * tiebreak. Absent on errored smokes and older records. */ verifyPass?: boolean } @@ -485,16 +490,17 @@ export interface FanOutDeps { parents?: ParetoParentContext[] /** GEN-5: MAP+TOOLBOX briefing context threaded into every author prompt. */ briefing?: BriefingContext - /** GEN-6: the resolved PUBLIC smoke instance — the GEPA seat's inner + /** The resolved public smoke instance used by the GEPA seat's inner * evaluator target. Required (with `smokeRunner`) when a gepa seat is * configured and no custom `author` is injected. */ smokeInstanceId?: string - /** GEN-6: the gen-5 score split; private instance ids never reach the GEPA + /** The score split; private instance ids never reach the GEPA * bridge (gepa-seat.mts asserts, fail-closed). Null/omitted = no split. */ scoreSplit?: Pick | null - /** GEN-6 test seam: the adapter factory (default: checked dynamic import of - * agent-eval's gepaOptimizationMethod, loud when the install predates it). */ + /** GEPA method override for tests. Default: agent-eval's official GEPA method. */ gepaMethodFactory?: GepaMethodFactory + /** Explicit GEPA optimizer model override, primarily for isolated tests. */ + gepaOptimizer?: GepaSeatDeps['optimizer'] log?: (msg: string) => void } @@ -509,7 +515,7 @@ function defaultAuthor(config: OuterLoopConfig, deps: FanOutDeps): AuthorFn { // its own profile, lens prompt, and shot-receipt home. const inners = new Map() const ledgers = new Map() - // GEN-6: the GEPA seat authors through the agent-eval adapter, not a + // The GEPA seat authors through the agent-eval adapter, not a // harness CLI. Its inner evaluator is the SAME injected smoke runner the // pre-filter uses (presence enforced at generator construction). let gepaAuthor: AuthorFn | undefined @@ -520,6 +526,7 @@ function defaultAuthor(config: OuterLoopConfig, deps: FanOutDeps): AuthorFn { smokeInstanceId: deps.smokeInstanceId!, scoreSplit: deps.scoreSplit ?? null, ...(deps.gepaMethodFactory ? { methodFactory: deps.gepaMethodFactory } : {}), + ...(deps.gepaOptimizer ? { optimizer: deps.gepaOptimizer } : {}), ...(deps.log ? { log: deps.log } : {}), }) return gepaAuthor(proposer, args) @@ -586,7 +593,7 @@ export function fanOutLoopsGenerator(config: OuterLoopConfig, deps: FanOutDeps = `only ${(deps.parents ?? []).length} pareto parent(s) materialized — a merge seat needs >=2`, ) } - // GEN-6: engine seats are validated fail-closed at construction, and the + // Engine seats are validated at construction, and the // default author path requires the pre-filter smoke runner — it IS the GEPA // seat's inner evaluator (spec: score = smoke resolve + verify-pass // tiebreak). A custom injected `author` owns its own evaluator. diff --git a/bench/src/swe-arena/proposer-fanout.test.mts b/bench/src/swe-arena/proposer-fanout.test.mts index ab39e7a3..17a159da 100644 --- a/bench/src/swe-arena/proposer-fanout.test.mts +++ b/bench/src/swe-arena/proposer-fanout.test.mts @@ -567,7 +567,6 @@ describe('defaultGen5Config', () => { expect(config.briefing).toBe('map-toolbox-v1') expect(config.activationGate).toBe(true) expect(config.rolloutLedger).toEqual({ enabled: true }) - expect(config.lineage).toBe(true) expect(config.priorEvidenceDirs!.some((d) => d.includes('gen4'))).toBe(true) }) diff --git a/bench/src/swe-arena/proposer-provenance.mts b/bench/src/swe-arena/proposer-provenance.mts index 2199b342..94f20d45 100644 --- a/bench/src/swe-arena/proposer-provenance.mts +++ b/bench/src/swe-arena/proposer-provenance.mts @@ -27,21 +27,19 @@ import { join } from 'node:path' import { DEFAULT_GEPA_PYTHON, isGepaSeat, - loadGepaMethodFactory, probeGepaRuntime, - type CampaignModuleImport, } from './gepa-seat.mts' import { run } from './proc.ts' import type { ProposerSpec } from './proposer-fanout.mts' export interface ProposerModelProvenance { name: string - /** Absent on a GEN-6 engine seat (see `engine`). */ + /** Absent on an engine seat (see `engine`). */ harness: ProposerSpec['harness'] /** Explicit model pin from the spec (threaded as `-m`), or null when the * seat runs the CLI's own resolved default. */ pinnedModel: string | null - /** ` --version` stdout (trimmed). For a GEN-6 gepa seat this is + /** ` --version` stdout (trimmed). For a GEPA seat this is * the bridge python's `--version` output — the runtime that authors. */ harnessVersion: string /** claude seats only: the settings default model the logged-in CLI resolves @@ -51,13 +49,13 @@ export interface ProposerModelProvenance { /** codex seats only: `codex login status` stdout (trimmed). */ authStatus: string | null merge: boolean - /** GEN-6 gepa seat: the engine name from the spec. */ + /** GEPA seat: the engine name from the spec. */ engine?: 'gepa' | 'omni' - /** GEN-6 gepa seat: the ONE change-space file GEPA optimizes. */ + /** GEPA seat: the one change-space file GEPA optimizes. */ surface?: string - /** GEN-6 gepa seat: installed gepa version ('source' for a source pin). */ + /** GEPA seat: installed GEPA version ('source' for a source pin). */ gepaVersion?: string - /** GEN-6 gepa seat: the Python bridge module the seat runs. */ + /** GEPA seat: the Python bridge module the seat runs. */ bridge?: string } @@ -92,17 +90,16 @@ export function claudeSettingsModel( } /** Capture per-proposer model provenance. Throws when any configured harness - * binary is missing/broken, when a codex seat is not logged in, or — GEN-6 — - * when a gepa seat's runtime is incomplete: the installed agent-eval must - * export `gepaOptimizationMethod` and the Python bridge + GEPA engine must - * import (probeGepaRuntime carries the exact install instructions). A dead - * seat fails the launch at t=0, never a mid-run candidate slot. */ + * binary is missing/broken, when a codex seat is not logged in, or when + * a GEPA seat's Python bridge or engine cannot import + * (`probeGepaRuntime` carries the exact install instructions). A dead seat + * fails the launch at t=0, never a mid-run candidate slot. The TypeScript + * adapter is a pinned package dependency and therefore checked at install. */ export async function captureProposerProvenance( proposers: ProposerSpec[], deps: { exec?: VersionExec readSettingsModel?: () => string | null - importCampaign?: CampaignModuleImport } = {}, ): Promise { const exec = deps.exec ?? defaultExec @@ -112,8 +109,6 @@ export async function captureProposerProvenance( const gepaBySeat = new Map() const gepaSeats = proposers.filter(isGepaSeat) if (gepaSeats.length > 0) { - // Node side first: the adapter export (fails loud with upgrade hint). - await loadGepaMethodFactory(...(deps.importCampaign ? [deps.importCampaign] : [])) for (const seat of gepaSeats) { gepaBySeat.set(seat.name, await probeGepaRuntime(seat.python ?? DEFAULT_GEPA_PYTHON, exec, seat.name)) } diff --git a/bench/src/swe-code-improve.mts b/bench/src/swe-code-improve.mts index f7430a09..2fb707e5 100644 --- a/bench/src/swe-code-improve.mts +++ b/bench/src/swe-code-improve.mts @@ -108,7 +108,6 @@ async function main(): Promise { if (!routerKey) throw new Error('TANGLE_API_KEY required (worker calls the router)') const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1' const workerModel = process.env.WORKER_MODEL ?? 'glm-4.6' - const reflectModel = process.env.REFLECT_MODEL ?? 'glm-4.6' const trainIds = (process.env.TRAIN_IDS ?? 'psf__requests-2931,pallets__flask-5014').split(',').map((s) => s.trim()).filter(Boolean) const holdoutIds = (process.env.HOLDOUT_IDS ?? 'psf__requests-1142,psf__requests-1921').split(',').map((s) => s.trim()).filter(Boolean) const generations = Number(process.env.GENERATIONS ?? 1) @@ -127,7 +126,7 @@ async function main(): Promise { const allIds = [...new Set([...trainIds, ...holdoutIds])] console.log('=== META-HARNESS on the SWE scaffold — improve(surface:code) ===') - console.log(`worker=${workerModel} reflect=${reflectModel} router=${routerBaseUrl} runTool=${enableRun}`) + console.log(`worker=${workerModel} router=${routerBaseUrl} runTool=${enableRun}`) console.log(`train=[${trainIds.join(', ')}] holdout=[${holdoutIds.join(', ')}]`) console.log(`generations=${generations} population=${population} innerTurns=${innerTurns} maxTokens=${maxTokens}`) console.log(`repoRoot=${REPO_ROOT} baseRef=${baseRef}`) @@ -156,15 +155,25 @@ async function main(): Promise { const worktreeRef = isCode ? String((surface as { worktreeRef?: string }).worktreeRef ?? '') : '' const rootDir = isCode && worktreeRef && existsSync(worktreeRef) ? worktreeRef : SWE_MAIN_ROOT const t0 = Date.now() - const r = await runEmit(rootDir, scenario.id, workerEnv, emitTimeoutMs) + const paid = await ctx.cost.runPaidCall({ + channel: 'agent', + actor: 'swe-scaffold-worker', + model: workerModel, + execute: () => runEmit(rootDir, scenario.id, workerEnv, emitTimeoutMs), + receipt: (result) => { + const usageUnknown = result.tokIn === 0 && result.tokOut === 0 + return { + model: workerModel, + inputTokens: result.tokIn, + outputTokens: result.tokOut, + ...(result.usd > 0 ? { actualCostUsd: result.usd } : {}), + ...(usageUnknown ? { usageUnknown: true, costUnknown: result.usd <= 0 } : {}), + } + }, + }) + if (!paid.succeeded) throw paid.error + const r = paid.value const hasPatch = r.patch.trim().length > 0 - // Report real usage; floor a patch-bearing zero-usage cell so the stub-guard cannot abort on a - // router telemetry gap (lift is judge-derived, so this only affects cost accounting). - const zeroUsage = r.tokIn === 0 && r.tokOut === 0 - ctx.cost.observe(zeroUsage && hasPatch ? Math.max(r.usd, 0.0001) : r.usd, workerModel) - ctx.cost.observeTokens( - zeroUsage && hasPatch ? { input: Math.max(r.tokIn, 1), output: Math.max(r.tokOut, 1) } : { input: r.tokIn, output: r.tokOut }, - ) const files = hasPatch ? [...r.patch.matchAll(/^diff --git a\/(\S+)/gm)].map((m) => m[1]) : [] console.log( ` [measure] ${isCode ? 'cand' : 'base'} ${scenario.id} patch=${r.patch.length}b files=[${files.join(', ') || 'none'}] ` + @@ -173,7 +182,7 @@ async function main(): Promise { return hasPatch ? r.patch : null } - const judge: JudgeConfig = { + const judge: JudgeConfig = { name: 'swebench-docker', dimensions: [{ key: 'resolved', description: 'FAIL_TO_PASS + PASS_TO_PASS resolved by the official swebench Docker harness' }], async score({ artifact, scenario }) { @@ -290,7 +299,7 @@ async function main(): Promise { const scenarios: Scenario[] = allIds.map((id) => ({ id, kind: 'swe-bench-verified' })) const holdoutScenarios: Scenario[] = holdoutIds.map((id) => ({ id, kind: 'swe-bench-verified' })) - const out = await improve(profile, [], { + const out = await improve(profile, { surface: 'code', gate: 'holdout', code: { repoRoot: REPO_ROOT, baseRef, worktreeDir, generator }, @@ -301,25 +310,17 @@ async function main(): Promise { agent, expectUsage: 'warn', budget: { generations, populationSize: population, holdoutScenarios, maxConcurrency: 1, reps: 1 }, - llm: { baseUrl: routerBaseUrl, apiKey: routerKey, model: reflectModel }, }) console.log('\n=== RESULT ===') - console.log(`gateDecision=${out.gateDecision} shipped=${out.shipped} lift=${out.lift}`) + console.log(`decision=${out.decision} lift=${out.lift}`) console.log(`baseline holdout composite = ${out.raw.baseline.compositeMean}`) console.log(`winner holdout composite = ${out.raw.winner.compositeMean}`) console.log(`baseline per-scenario: ${JSON.stringify(out.raw.baseline.perScenario)}`) console.log(`winner per-scenario: ${JSON.stringify(out.raw.winner.perScenario)}`) if (out.raw.winner.label) console.log(`winner label: ${out.raw.winner.label}`) - if (out.raw.winner.summary) console.log(`winner summary: ${out.raw.winner.summary}`) - for (const gen of out.raw.generations ?? []) { - console.log(`\n-- generation ${gen.record.generationIndex} candidates --`) - for (const c of gen.record.candidates) { - const perScenario = (c as { scenarios?: Array<{ scenarioId: string; composite: number }> }).scenarios ?? [] - const detail = perScenario.map((s) => `${s.scenarioId}=${s.composite}`).join(' ') - console.log(` candidate ${c.surfaceHash.slice(0, 8)} composite=${c.composite}${c.label ? ` "${c.label}"` : ''} [${detail}]`) - } - } + if (out.raw.winner.rationale) console.log(`winner rationale: ${out.raw.winner.rationale}`) + console.log(`generations explored: ${out.generationsExplored ?? 0}`) } main().catch((e) => { diff --git a/docs/agent-managed-compute/architecture.md b/docs/agent-managed-compute/architecture.md index 9976be2d..2d76f29f 100644 --- a/docs/agent-managed-compute/architecture.md +++ b/docs/agent-managed-compute/architecture.md @@ -129,7 +129,14 @@ runInteraction({ store, }) -improve(profile, findings, options) +improve(profile, { + method, + trainScenarios, + selectionScenarios, + testScenarios, + judges, + agent, +}) ``` `runAgent` is the common path for one root profile that may dynamically delegate. diff --git a/docs/api/index.md b/docs/api/index.md index b622c79c..1ca5c9fe 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -1133,6 +1133,52 @@ Defined in: [src/errors.ts:117](https://github.com/tangle-network/agent-runtime/ *** +### OfficialOptimizerUnavailableError + +Defined in: src/improvement/official-optimizers.ts:46 + +Missing optional Python dependencies for an official optimizer. + +#### Extends + +- `ConfigError` + +#### Constructors + +##### Constructor + +> **new OfficialOptimizerUnavailableError**(`optimizer`, `cause`): [`OfficialOptimizerUnavailableError`](#officialoptimizerunavailableerror) + +Defined in: src/improvement/official-optimizers.ts:49 + +###### Parameters + +###### optimizer + +`"gepa"` \| `"skillopt"` + +###### cause + +`unknown` + +###### Returns + +[`OfficialOptimizerUnavailableError`](#officialoptimizerunavailableerror) + +###### Overrides + +`ConfigError.constructor` + +#### Properties + +##### optimizer + +> `readonly` **optimizer**: `"gepa"` \| `"skillopt"` + +Defined in: src/improvement/official-optimizers.ts:47 + +*** + ### InMemoryRuntimeSessionStore Defined in: [src/sessions.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/sessions.ts#L111) @@ -5544,7 +5590,7 @@ Content type for the response. ### VerifyResult -Defined in: [src/improvement/agentic-generator.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L75) +Defined in: [src/improvement/agentic-generator.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L74) Outcome of verifying a candidate worktree. `feedback` (compiler errors, failing test output) is fed into the next shot when `ok` is false. @@ -5555,31 +5601,25 @@ Outcome of verifying a candidate worktree. `feedback` (compiler errors, > **ok**: `boolean` -Defined in: [src/improvement/agentic-generator.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L76) +Defined in: [src/improvement/agentic-generator.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L75) ##### feedback? > `optional` **feedback?**: `string` -Defined in: [src/improvement/agentic-generator.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L77) +Defined in: [src/improvement/agentic-generator.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L76) *** ### AgenticGeneratorShotReceipt -Defined in: [src/improvement/agentic-generator.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L88) +Defined in: [src/improvement/agentic-generator.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L87) -`@tangle-network/agent-runtime` improvement — the CODE-surface proposer for -agent-eval's improvement loop. +`@tangle-network/agent-runtime` improvement. -The public entry point is `improve()`, a profile-aware facade over agent-eval's -`selfImprove`. This module also supplies the runtime-specific code candidate -producer, which mutates an isolated git worktree via a pluggable -`CandidateGenerator`: - - `reflectiveGenerator` — cheap, no sandbox, applies pre-drafted patches - - `agenticGenerator` — full coding harness in the worktree, multi-shot - - `driverLoopGenerator` — the driver→worker atom: an LLM driver authors, - observes, rates, and steers the harness sessions (default for tool/mcp) +The public entry point is `improve()`. Complete agent-eval methods optimize +profile surfaces. Runtime owns only code candidates that mutate an isolated +git worktree through a pluggable `CandidateGenerator`. #### Properties @@ -5587,19 +5627,19 @@ producer, which mutates an isolated git worktree via a pluggable > `readonly` **generation**: `number` \| `null` -Defined in: [src/improvement/agentic-generator.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L89) +Defined in: [src/improvement/agentic-generator.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L88) ##### candidateIndex > `readonly` **candidateIndex**: `number` \| `null` -Defined in: [src/improvement/agentic-generator.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L90) +Defined in: [src/improvement/agentic-generator.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L89) ##### shot > `readonly` **shot**: `number` -Defined in: [src/improvement/agentic-generator.ts:92](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L92) +Defined in: [src/improvement/agentic-generator.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L91) One-based shot number within this candidate. @@ -5607,67 +5647,67 @@ One-based shot number within this candidate. > `readonly` **maxShots**: `number` -Defined in: [src/improvement/agentic-generator.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L93) +Defined in: [src/improvement/agentic-generator.ts:92](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L92) ##### harness > `readonly` **harness**: [`LocalHarness`](mcp.md#localharness) -Defined in: [src/improvement/agentic-generator.ts:94](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L94) +Defined in: [src/improvement/agentic-generator.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L93) ##### model > `readonly` **model**: `string` \| `null` -Defined in: [src/improvement/agentic-generator.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L95) +Defined in: [src/improvement/agentic-generator.ts:94](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L94) ##### reasoningEffort > `readonly` **reasoningEffort**: `ReasoningEffort` \| `null` -Defined in: [src/improvement/agentic-generator.ts:96](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L96) +Defined in: [src/improvement/agentic-generator.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L95) ##### promptSha256 > `readonly` **promptSha256**: `` `sha256:${string}` `` -Defined in: [src/improvement/agentic-generator.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L97) +Defined in: [src/improvement/agentic-generator.ts:96](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L96) ##### startedAt > `readonly` **startedAt**: `string` -Defined in: [src/improvement/agentic-generator.ts:98](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L98) +Defined in: [src/improvement/agentic-generator.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L97) ##### completedAt > `readonly` **completedAt**: `string` -Defined in: [src/improvement/agentic-generator.ts:99](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L99) +Defined in: [src/improvement/agentic-generator.ts:98](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L98) ##### durationMs > `readonly` **durationMs**: `number` -Defined in: [src/improvement/agentic-generator.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L100) +Defined in: [src/improvement/agentic-generator.ts:99](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L99) ##### exitCode > `readonly` **exitCode**: `number` \| `null` -Defined in: [src/improvement/agentic-generator.ts:101](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L101) +Defined in: [src/improvement/agentic-generator.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L100) ##### timedOut > `readonly` **timedOut**: `boolean` -Defined in: [src/improvement/agentic-generator.ts:102](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L102) +Defined in: [src/improvement/agentic-generator.ts:101](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L101) ##### aborted? > `readonly` `optional` **aborted?**: `boolean` -Defined in: [src/improvement/agentic-generator.ts:104](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L104) +Defined in: [src/improvement/agentic-generator.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L103) True when caller cancellation reached the author process; absent in older receipts. @@ -5675,43 +5715,43 @@ True when caller cancellation reached the author process; absent in older receip > `readonly` **killedBySignal**: `Signals` \| `null` -Defined in: [src/improvement/agentic-generator.ts:105](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L105) +Defined in: [src/improvement/agentic-generator.ts:104](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L104) ##### stdoutBytes > `readonly` **stdoutBytes**: `number` \| `null` -Defined in: [src/improvement/agentic-generator.ts:106](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L106) +Defined in: [src/improvement/agentic-generator.ts:105](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L105) ##### stdoutSha256 > `readonly` **stdoutSha256**: `` `sha256:${string}` `` \| `null` -Defined in: [src/improvement/agentic-generator.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L107) +Defined in: [src/improvement/agentic-generator.ts:106](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L106) ##### stderrBytes > `readonly` **stderrBytes**: `number` \| `null` -Defined in: [src/improvement/agentic-generator.ts:108](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L108) +Defined in: [src/improvement/agentic-generator.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L107) ##### stderrSha256 > `readonly` **stderrSha256**: `` `sha256:${string}` `` \| `null` -Defined in: [src/improvement/agentic-generator.ts:109](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L109) +Defined in: [src/improvement/agentic-generator.ts:108](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L108) ##### usage > `readonly` **usage**: [`CodexTokenUsage`](mcp.md#codextokenusage) \| `null` -Defined in: [src/improvement/agentic-generator.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L110) +Defined in: [src/improvement/agentic-generator.ts:109](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L109) ##### profileWorkspacePlanDigest > `readonly` **profileWorkspacePlanDigest**: `string` \| `null` -Defined in: [src/improvement/agentic-generator.ts:112](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L112) +Defined in: [src/improvement/agentic-generator.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L111) Digest of the exact profile-file workspace plan applied for this shot. @@ -5719,13 +5759,13 @@ Digest of the exact profile-file workspace plan applied for this shot. > `readonly` **profileWorkspaceFileCount**: `number` -Defined in: [src/improvement/agentic-generator.ts:113](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L113) +Defined in: [src/improvement/agentic-generator.ts:112](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L112) ##### costCallId > `readonly` **costCallId**: `string` \| `null` -Defined in: [src/improvement/agentic-generator.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L115) +Defined in: [src/improvement/agentic-generator.ts:114](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L114) Shared run-ledger call id for this exact shot. @@ -5733,7 +5773,7 @@ Shared run-ledger call id for this exact shot. > `readonly` **costBasis**: `"unknown"` \| `"provider-reported"` \| `"estimated-pricing"` -Defined in: [src/improvement/agentic-generator.ts:117](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L117) +Defined in: [src/improvement/agentic-generator.ts:116](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L116) Whether dollars came from the provider, the pricing table, or are unknown. @@ -5741,13 +5781,13 @@ Whether dollars came from the provider, the pricing table, or are unknown. > `readonly` **costUsd**: `number` \| `null` -Defined in: [src/improvement/agentic-generator.ts:118](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L118) +Defined in: [src/improvement/agentic-generator.ts:117](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L117) ##### costUsdKnown > `readonly` **costUsdKnown**: `boolean` -Defined in: [src/improvement/agentic-generator.ts:120](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L120) +Defined in: [src/improvement/agentic-generator.ts:119](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L119) True only for a provider-reported amount, never for a pricing estimate. @@ -5755,31 +5795,25 @@ True only for a provider-reported amount, never for a pricing estimate. > `readonly` **evidence**: [`CodexExecutionEvidence`](mcp.md#codexexecutionevidence) \| `null` -Defined in: [src/improvement/agentic-generator.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L121) +Defined in: [src/improvement/agentic-generator.ts:120](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L120) ##### error > `readonly` **error**: \{ `name`: `string`; `message`: `string`; \} \| `null` -Defined in: [src/improvement/agentic-generator.ts:122](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L122) +Defined in: [src/improvement/agentic-generator.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L121) *** ### AgenticGeneratorOptions -Defined in: [src/improvement/agentic-generator.ts:165](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L165) +Defined in: [src/improvement/agentic-generator.ts:164](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L164) -`@tangle-network/agent-runtime` improvement — the CODE-surface proposer for -agent-eval's improvement loop. +`@tangle-network/agent-runtime` improvement. -The public entry point is `improve()`, a profile-aware facade over agent-eval's -`selfImprove`. This module also supplies the runtime-specific code candidate -producer, which mutates an isolated git worktree via a pluggable -`CandidateGenerator`: - - `reflectiveGenerator` — cheap, no sandbox, applies pre-drafted patches - - `agenticGenerator` — full coding harness in the worktree, multi-shot - - `driverLoopGenerator` — the driver→worker atom: an LLM driver authors, - observes, rates, and steers the harness sessions (default for tool/mcp) +The public entry point is `improve()`. Complete agent-eval methods optimize +profile surfaces. Runtime owns only code candidates that mutate an isolated +git worktree through a pluggable `CandidateGenerator`. #### Properties @@ -5787,7 +5821,7 @@ producer, which mutates an isolated git worktree via a pluggable > `optional` **harness?**: [`LocalHarness`](mcp.md#localharness) -Defined in: [src/improvement/agentic-generator.ts:167](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L167) +Defined in: [src/improvement/agentic-generator.ts:166](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L166) Local coding harness to run in the worktree. Default `claude`. @@ -5795,7 +5829,7 @@ Local coding harness to run in the worktree. Default `claude`. > `optional` **profile?**: `AgentProfile` -Defined in: [src/improvement/agentic-generator.ts:170](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L170) +Defined in: [src/improvement/agentic-generator.ts:169](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L169) Author profile rendered through the canonical harness mapper. Required for reproducible Codex so model and reasoning settings are explicit. @@ -5804,7 +5838,7 @@ Author profile rendered through the canonical harness mapper. Required > `optional` **codexReproducible?**: `boolean` -Defined in: [src/improvement/agentic-generator.ts:173](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L173) +Defined in: [src/improvement/agentic-generator.ts:172](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L172) Run Codex with isolated configuration, exact prompt evidence, and required terminal token usage. Requires `harness: 'codex'` and `profile`. @@ -5813,7 +5847,7 @@ Run Codex with isolated configuration, exact prompt evidence, and required > `optional` **codexReadDeniedPaths?**: readonly `string`[] \| ((`worktreePath`) => readonly `string`[]) -Defined in: [src/improvement/agentic-generator.ts:176](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L176) +Defined in: [src/improvement/agentic-generator.ts:175](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L175) Absolute paths reproducible Codex must not read. A function can derive candidate-specific paths after the driver creates its worktree. @@ -5822,7 +5856,7 @@ Absolute paths reproducible Codex must not read. A function can derive > `optional` **onShotCompleted?**: (`receipt`, `execution`) => `void` \| `Promise`\<`void`\> -Defined in: [src/improvement/agentic-generator.ts:181](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L181) +Defined in: [src/improvement/agentic-generator.ts:180](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L180) Awaited once for every attempted author shot, including process failures. The second argument preserves the exact harness result, including stdout @@ -5847,7 +5881,7 @@ Awaited once for every attempted author shot, including process failures. > `optional` **onShotDisposition?**: (`receipt`, `disposition`) => `void` \| `Promise`\<`void`\> -Defined in: [src/improvement/agentic-generator.ts:187](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L187) +Defined in: [src/improvement/agentic-generator.ts:186](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L186) Awaited after worktree inspection and before the shot is accepted, retried, or discarded. Throwing aborts the candidate. @@ -5870,7 +5904,7 @@ Awaited after worktree inspection and before the shot is accepted, > `optional` **maximumCharge?**: `MaximumCharge` -Defined in: [src/improvement/agentic-generator.ts:195](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L195) +Defined in: [src/improvement/agentic-generator.ts:194](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L194) Optional hard upper bound passed to the run-wide CostLedger before each author shot. This MUST be enforced by the provider or executor; a planning @@ -5881,7 +5915,7 @@ Optional hard upper bound passed to the run-wide CostLedger before each > `optional` **timeoutMs?**: `number` -Defined in: [src/improvement/agentic-generator.ts:197](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L197) +Defined in: [src/improvement/agentic-generator.ts:196](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L196) Per-shot wall-clock timeout (ms). Default = `runLocalHarness` default (5m). @@ -5889,7 +5923,7 @@ Per-shot wall-clock timeout (ms). Default = `runLocalHarness` default (5m). > `optional` **buildPrompt?**: (`args`) => `string` -Defined in: [src/improvement/agentic-generator.ts:200](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L200) +Defined in: [src/improvement/agentic-generator.ts:199](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L199) Build the harness task prompt from the report + findings. Override for domain phrasing; the default turns findings into a concrete coder task. @@ -5914,7 +5948,7 @@ Build the harness task prompt from the report + findings. Override for > `optional` **verify?**: [`Verifier`](#verifier) -Defined in: [src/improvement/agentic-generator.ts:206](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L206) +Defined in: [src/improvement/agentic-generator.ts:205](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L205) Verify the worktree after each dirtying shot. When set, a candidate that fails verification is NOT returned — the failure feeds the next shot @@ -5926,7 +5960,7 @@ Verify the worktree after each dirtying shot. When set, a candidate that > `optional` **runHarness?**: (`options`) => `Promise`\<[`LocalHarnessResult`](mcp.md#localharnessresult)\> -Defined in: [src/improvement/agentic-generator.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L208) +Defined in: [src/improvement/agentic-generator.ts:207](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L207) Test seam — inject the harness runner (defaults to `runLocalHarness`). @@ -5967,7 +6001,7 @@ returning an incomplete reproducibility receipt. > `optional` **isDirty?**: (`worktreePath`) => `boolean` -Defined in: [src/improvement/agentic-generator.ts:210](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L210) +Defined in: [src/improvement/agentic-generator.ts:209](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L209) Test seam — inject the worktree-dirty check (defaults to `git status`). @@ -6031,7 +6065,7 @@ OTLP `service.name` on every emitted span. Default `'campaign'`. Defined in: [src/improvement/campaign-otlp.ts:159](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/campaign-otlp.ts#L159) -The `selfImprove`/`improve()` run root — the SAME `runDir` the loop +The code-improvement run root — the SAME `runDir` the loop records under (`/baseline/...`, `/gen-/candidate-/...`). Must be a real path; a `mem://` run records nothing to resolve. @@ -6240,9 +6274,49 @@ Defined in: [src/improvement/findings.ts:74](https://github.com/tangle-network/a *** +### ImproveMethodContext + +Defined in: [src/improvement/improve.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L77) + +#### Properties + +##### profile + +> `readonly` **profile**: `Readonly`\<`AgentProfile`\> + +Defined in: [src/improvement/improve.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L79) + +Validated baseline profile. + +##### surface + +> `readonly` **surface**: [`ImproveProfileSurface`](#improveprofilesurface) + +Defined in: [src/improvement/improve.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L81) + +Exact profile coordinate being optimized. + +##### baselineSurface + +> `readonly` **baselineSurface**: `MutableSurface` + +Defined in: [src/improvement/improve.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L83) + +Exact bytes supplied to the optimization method. + +##### findings + +> `readonly` **findings**: readonly `unknown`[] + +Defined in: [src/improvement/improve.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L85) + +Findings produced before this search, if any. + +*** + ### ImproveSkillsOptions -Defined in: [src/improvement/improve.ts:157](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L157) +Defined in: [src/improvement/improve.ts:169](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L169) #### Properties @@ -6250,15 +6324,65 @@ Defined in: [src/improvement/improve.ts:157](https://github.com/tangle-network/a > **resourceName**: `string` -Defined in: [src/improvement/improve.ts:159](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L159) +Defined in: [src/improvement/improve.ts:171](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L171) `name` of one inline entry in `profile.resources.skills`. *** +### ImproveProfileComponents + +Defined in: [src/improvement/improve.ts:175](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L175) + +Caller-owned mapping for optimizing several profile fields as one candidate. + +#### Methods + +##### read() + +> **read**(`profile`): `Readonly`\<`Record`\<`string`, `string`\>\> + +Defined in: [src/improvement/improve.ts:177](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L177) + +Extract the exact named text components optimized together. + +###### Parameters + +###### profile + +`Readonly`\<`AgentProfile`\> + +###### Returns + +`Readonly`\<`Record`\<`string`, `string`\>\> + +##### apply() + +> **apply**(`profile`, `components`): `AgentProfile` + +Defined in: [src/improvement/improve.ts:179](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L179) + +Apply a complete winning component map to a detached profile. + +###### Parameters + +###### profile + +`Readonly`\<`AgentProfile`\> + +###### components + +`Readonly`\<`Record`\<`string`, `string`\>\> + +###### Returns + +`AgentProfile` + +*** + ### ImproveCodeOptions -Defined in: [src/improvement/improve.ts:162](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L162) +Defined in: [src/improvement/improve.ts:182](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L182) #### Properties @@ -6266,7 +6390,7 @@ Defined in: [src/improvement/improve.ts:162](https://github.com/tangle-network/a > **repoRoot**: `string` -Defined in: [src/improvement/improve.ts:164](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L164) +Defined in: [src/improvement/improve.ts:184](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L184) Repo root candidate worktrees fork from. @@ -6274,7 +6398,7 @@ Repo root candidate worktrees fork from. > `optional` **baseRef?**: `string` -Defined in: [src/improvement/improve.ts:166](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L166) +Defined in: [src/improvement/improve.ts:186](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L186) Base ref candidates fork from. Default `main`. @@ -6282,7 +6406,7 @@ Base ref candidates fork from. Default `main`. > `optional` **worktreeDir?**: `string` -Defined in: [src/improvement/improve.ts:168](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L168) +Defined in: [src/improvement/improve.ts:188](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L188) Directory worktrees are created under. Default `/.worktrees`. @@ -6290,7 +6414,7 @@ Directory worktrees are created under. Default `/.worktrees`. > `optional` **worktree?**: `WorktreeAdapter` -Defined in: [src/improvement/improve.ts:171](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L171) +Defined in: [src/improvement/improve.ts:191](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L191) Git-compatible adapter override, primarily for tests. Candidate advancement still requires normal Git worktree and commit semantics. @@ -6299,7 +6423,7 @@ Git-compatible adapter override, primarily for tests. Candidate advancement > `optional` **harness?**: [`LocalHarness`](mcp.md#localharness) -Defined in: [src/improvement/improve.ts:173](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L173) +Defined in: [src/improvement/improve.ts:193](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L193) Coding harness the agentic generator runs in each worktree. Default `claude`. @@ -6307,7 +6431,7 @@ Coding harness the agentic generator runs in each worktree. Default `claude`. > `optional` **verify?**: [`Verifier`](#verifier) -Defined in: [src/improvement/improve.ts:176](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L176) +Defined in: [src/improvement/improve.ts:196](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L196) Verify a candidate worktree before it becomes a measurable surface; failures feed the next shot (see `agenticGenerator.verify` / `commandVerifier`). @@ -6316,7 +6440,7 @@ Verify a candidate worktree before it becomes a measurable surface; failures > `optional` **timeoutMs?**: `number` -Defined in: [src/improvement/improve.ts:178](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L178) +Defined in: [src/improvement/improve.ts:198](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L198) Per-shot wall-clock timeout for the harness (ms). @@ -6324,7 +6448,7 @@ Per-shot wall-clock timeout for the harness (ms). > `optional` **generator?**: [`CandidateGenerator`](#candidategenerator) -Defined in: [src/improvement/improve.ts:181](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L181) +Defined in: [src/improvement/improve.ts:201](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L201) Byte-producer override — the test seam and the escape hatch for custom candidate production. When set, `harness`/`verify`/`timeoutMs` are unused. @@ -6333,7 +6457,7 @@ Byte-producer override — the test seam and the escape hatch for custom ### ImprovementCandidate -Defined in: [src/improvement/improve.ts:184](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L184) +Defined in: [src/improvement/improve.ts:204](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L204) #### Properties @@ -6341,7 +6465,7 @@ Defined in: [src/improvement/improve.ts:184](https://github.com/tangle-network/a > **surface**: [`ImproveSurface`](#improvesurface) -Defined in: [src/improvement/improve.ts:186](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L186) +Defined in: [src/improvement/improve.ts:206](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L206) Surface searched by this run. @@ -6349,7 +6473,7 @@ Surface searched by this run. > **value**: `MutableSurface` -Defined in: [src/improvement/improve.ts:188](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L188) +Defined in: [src/improvement/improve.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L208) Exact winning value returned by agent-eval. @@ -6357,246 +6481,490 @@ Exact winning value returned by agent-eval. > `optional` **profile?**: `AgentProfile` -Defined in: [src/improvement/improve.ts:190](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L190) +Defined in: [src/improvement/improve.ts:210](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L210) Detached profile candidate when the surface maps directly to AgentProfile. *** -### ImproveResult +### ImproveCost -Defined in: [src/improvement/improve.ts:193](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L193) +Defined in: [src/improvement/improve.ts:214](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L214) -#### Type Parameters +Normalized spend reported for one Runtime improvement run. -##### TScenario +#### Properties -`TScenario` *extends* `Scenario` +##### totalCostUsd -##### TArtifact +> **totalCostUsd**: `number` -`TArtifact` +Defined in: [src/improvement/improve.ts:215](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L215) -#### Properties +##### accountingComplete -##### candidate +> **accountingComplete**: `boolean` -> **candidate**: [`ImprovementCandidate`](#improvementcandidate) +Defined in: [src/improvement/improve.ts:216](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L216) -Defined in: [src/improvement/improve.ts:195](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L195) +##### incompleteReasons -Frozen candidate only. Live state is changed through an approved activation. +> **incompleteReasons**: `string`[] -##### decision +Defined in: [src/improvement/improve.ts:217](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L217) -> **decision**: `"ship"` \| `"hold"` \| `"need_more_work"` \| `"model_ceiling"` \| `"arch_ceiling"` +*** -Defined in: [src/improvement/improve.ts:197](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L197) +### ImproveLineage -Held-out decision for this search result. +Defined in: [src/improvement/improve.ts:221](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L221) -##### lift? +Optimizer ancestry sealed into downstream candidate experiments. -> `optional` **lift?**: `number` +#### Properties -Defined in: [src/improvement/improve.ts:201](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L201) +##### runId -Held-out lift (`winner − baseline` composite). Absent iff - `budget.holdout === 'deferred'` — no held-out measurement ran, so there - is no lift to report (never a fabricated 0). +> **runId**: `string` -##### raw +Defined in: [src/improvement/improve.ts:223](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L223) -> **raw**: `SelfImproveResult`\<`TScenario`, `TArtifact`\> +Upstream optimizer run when reported, otherwise this Runtime optimization invocation. -Defined in: [src/improvement/improve.ts:205](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L205) +##### developmentSplitDigest -Full `selfImprove` result for advanced inspection. For code runs, - `raw.winner.surface.worktreeRef` remains live after return whether the - candidate passed or held; call `dispose()` after consuming it. +> **developmentSplitDigest**: `` `sha256:${string}` `` -#### Methods +Defined in: [src/improvement/improve.ts:225](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L225) -##### dispose() +Exact train-plus-selection scenario payloads exposed to candidate selection. -> **dispose**(): `Promise`\<`void`\> +*** -Defined in: [src/improvement/improve.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L208) +### ImproveMethodResult -Release resources owned by this result. Idempotent; currently disposes - the returned code worktree and is a no-op for profile-only surfaces. +Defined in: [src/improvement/improve.ts:250](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L250) -###### Returns +#### Extends -`Promise`\<`void`\> +- `ImproveResultBase` -*** +#### Properties -### CandidateGenerator +##### candidate -Defined in: [src/improvement/improvement-driver.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L42) +> **candidate**: [`ImprovementCandidate`](#improvementcandidate) -The byte-producing seam — the ONE thing that differs between the cheap - reflective path and the full agentic path. A generator makes (uncommitted) - changes inside `worktreePath`; the driver commits them via the worktree - adapter's `finalize`. +Defined in: [src/improvement/improve.ts:230](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L230) -#### Properties +Frozen candidate only. Live state is changed through an approved activation. -##### kind +###### Inherited from -> **kind**: `string` +`ImproveResultBase.candidate` -Defined in: [src/improvement/improvement-driver.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L43) +##### cost -##### proposesWithoutFindings? +> **cost**: [`ImproveCost`](#improvecost) -> `optional` **proposesWithoutFindings?**: `boolean` +Defined in: [src/improvement/improve.ts:238](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L238) -Defined in: [src/improvement/improvement-driver.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L54) +Full search and final-test spend. -Whether this generator can produce a candidate from an EMPTY findings set - and no phase-2 report — i.e. it draws its change signal from the repo and - the raw-trace filesystem context on disk, not only from pre-summarized - findings. An agentic coder (`agenticGenerator`) sets this: the seed repo + - raw traces ARE the signal, so it must still run the full `populationSize` - when the distiller yielded nothing (this is the meta-harness contract — the - agent diagnoses from the raw traces itself). A patch-applier - (`reflectiveGenerator`) leaves it unset — with no findings there is no - patch to draft, so the driver short-circuits rather than spin up worktrees - for a guaranteed no-op. Default `false`. +###### Inherited from -#### Methods +`ImproveResultBase.cost` -##### generate() +##### durationMs -> **generate**(`args`): `Promise`\<\{ `applied`: `boolean`; `summary`: `string`; `label?`: `string`; `rationale?`: `string`; \}\> +> **durationMs**: `number` -Defined in: [src/improvement/improvement-driver.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L55) +Defined in: [src/improvement/improve.ts:240](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L240) -###### Parameters +Full wall-clock duration. -###### args +###### Inherited from -###### worktreePath +`ImproveResultBase.durationMs` -`string` +##### lineage -The candidate worktree — a clean checkout of the current incumbent. +> **lineage**: [`ImproveLineage`](#improvelineage) -###### report +Defined in: [src/improvement/improve.ts:242](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L242) -`unknown` +Optimizer ancestry used when sealing a candidate experiment. -Phase-2 research report (analyst findings + diff), opaque. +###### Inherited from -###### findings +`ImproveResultBase.lineage` -`AnalystFinding`[] +##### generationsExplored? -Findings resolved from the report or the loop context. +> `optional` **generationsExplored?**: `number` -###### dataset? +Defined in: [src/improvement/improve.ts:244](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L244) -`LabeledScenarioStore` +Number of generations explored by Runtime's code path. -Handle to all captured data, to ground the change. +###### Inherited from -###### maxShots +`ImproveResultBase.generationsExplored` -`number` +##### mode -DEPTH: max iterations the generator may take (agentic uses this; the - reflective generator ignores it). +> **mode**: `"method"` -###### signal +Defined in: [src/improvement/improve.ts:251](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L251) -`AbortSignal` +##### method -###### generation? +> **method**: `string` -`number` +Defined in: [src/improvement/improve.ts:252](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L252) -Improvement-loop coordinates. Present when called through improvementDriver. +##### provenance? -###### candidateIndex? +> `optional` **provenance?**: `OptimizationMethodProvenance` -`number` +Defined in: [src/improvement/improve.ts:254](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L254) -###### costLedger? +External optimizer package and resumable run identity, when reported. -`CostLedgerHandle` +##### decision -Shared run-wide paid-call account supplied by agent-eval 0.117+. +> **decision**: `"ship"` \| `"hold"` -###### costPhase? +Defined in: [src/improvement/improve.ts:255](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L255) -`string` +Final-test decision for this search result. -Receipt attribution phase supplied alongside `costLedger`. +###### Overrides + +`ImproveResultBase.decision` + +##### lift + +> **lift**: `number` + +Defined in: [src/improvement/improve.ts:256](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L256) + +Final-test lift when one was measured. + +###### Overrides + +`ImproveResultBase.lift` + +##### liftInterval + +> **liftInterval**: `object` + +Defined in: [src/improvement/improve.ts:257](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L257) + +Paired final-test confidence interval for method-based profile runs. + +###### low + +> **low**: `number` + +###### high + +> **high**: `number` + +###### Overrides + +`ImproveResultBase.liftInterval` + +##### raw + +> **raw**: `OptimizationMethodComparison` + +Defined in: [src/improvement/improve.ts:258](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L258) + +#### Methods + +##### dispose() + +> **dispose**(): `Promise`\<`void`\> + +Defined in: [src/improvement/improve.ts:247](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L247) + +Release resources owned by this result. Idempotent; currently disposes + the returned code worktree and is a no-op for profile-only surfaces. ###### Returns -`Promise`\<\{ `applied`: `boolean`; `summary`: `string`; `label?`: `string`; `rationale?`: `string`; \}\> +`Promise`\<`void`\> + +###### Inherited from + +`ImproveResultBase.dispose` *** -### ImprovementDriverOptions +### ImproveCodeResult + +Defined in: [src/improvement/improve.ts:261](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L261) + +#### Extends + +- `ImproveResultBase` + +#### Type Parameters + +##### TScenario + +`TScenario` *extends* `Scenario` + +##### TArtifact -Defined in: [src/improvement/improvement-driver.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L88) +`TArtifact` #### Properties -##### worktree +##### candidate -> **worktree**: `WorktreeAdapter` +> **candidate**: [`ImprovementCandidate`](#improvementcandidate) -Defined in: [src/improvement/improvement-driver.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L89) +Defined in: [src/improvement/improve.ts:230](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L230) -##### generator +Frozen candidate only. Live state is changed through an approved activation. -> **generator**: [`CandidateGenerator`](#candidategenerator) +###### Inherited from -Defined in: [src/improvement/improvement-driver.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L90) +`ImproveResultBase.candidate` -##### baseRef? +##### decision -> `optional` **baseRef?**: `string` +> **decision**: `"ship"` \| `"hold"` \| `"need_more_work"` \| `"model_ceiling"` \| `"arch_ceiling"` -Defined in: [src/improvement/improvement-driver.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L93) +Defined in: [src/improvement/improve.ts:232](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L232) -Root ref for first-generation/direct callers. Default `main`. - Later code generations retain the incumbent's original root. +Final-test decision for this search result. -*** +###### Inherited from -### ManagedImprovementDriver +`ImproveResultBase.decision` -Defined in: [src/improvement/improvement-driver.ts:96](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L96) +##### lift? -#### Extends +> `optional` **lift?**: `number` + +Defined in: [src/improvement/improve.ts:234](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L234) + +Final-test lift when one was measured. + +###### Inherited from + +`ImproveResultBase.lift` + +##### liftInterval? + +> `optional` **liftInterval?**: `object` + +Defined in: [src/improvement/improve.ts:236](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L236) + +Paired final-test confidence interval for method-based profile runs. + +###### low + +> **low**: `number` + +###### high + +> **high**: `number` + +###### Inherited from + +`ImproveResultBase.liftInterval` + +##### cost + +> **cost**: [`ImproveCost`](#improvecost) + +Defined in: [src/improvement/improve.ts:238](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L238) + +Full search and final-test spend. + +###### Inherited from + +`ImproveResultBase.cost` + +##### durationMs + +> **durationMs**: `number` + +Defined in: [src/improvement/improve.ts:240](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L240) + +Full wall-clock duration. + +###### Inherited from -- `SurfaceProposer`\<`AnalystFinding`\> +`ImproveResultBase.durationMs` + +##### lineage + +> **lineage**: [`ImproveLineage`](#improvelineage) + +Defined in: [src/improvement/improve.ts:242](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L242) + +Optimizer ancestry used when sealing a candidate experiment. + +###### Inherited from + +`ImproveResultBase.lineage` + +##### generationsExplored? + +> `optional` **generationsExplored?**: `number` + +Defined in: [src/improvement/improve.ts:244](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L244) + +Number of generations explored by Runtime's code path. + +###### Inherited from + +`ImproveResultBase.generationsExplored` + +##### mode + +> **mode**: `"code"` + +Defined in: [src/improvement/improve.ts:263](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L263) + +##### raw + +> **raw**: `SelfImproveResult`\<`TScenario`, `TArtifact`\> + +Defined in: [src/improvement/improve.ts:264](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L264) #### Methods -##### cleanup() +##### dispose() + +> **dispose**(): `Promise`\<`void`\> + +Defined in: [src/improvement/improve.ts:247](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L247) + +Release resources owned by this result. Idempotent; currently disposes + the returned code worktree and is a no-op for profile-only surfaces. + +###### Returns + +`Promise`\<`void`\> + +###### Inherited from + +`ImproveResultBase.dispose` + +*** + +### CandidateGenerator + +Defined in: [src/improvement/improvement-driver.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L29) + +The byte-producing seam — the ONE thing that differs between the cheap + reflective path and the full agentic path. A generator makes (uncommitted) + changes inside `worktreePath`; the driver commits them via the worktree + adapter's `finalize`. + +#### Properties + +##### kind + +> **kind**: `string` + +Defined in: [src/improvement/improvement-driver.ts:30](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L30) + +##### proposesWithoutFindings? + +> `optional` **proposesWithoutFindings?**: `boolean` + +Defined in: [src/improvement/improvement-driver.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L41) + +Whether this generator can produce a candidate from an EMPTY findings set + and no phase-2 report — i.e. it draws its change signal from the repo and + the raw-trace filesystem context on disk, not only from pre-summarized + findings. An agentic coder (`agenticGenerator`) sets this: the seed repo + + raw traces ARE the signal, so it must still run the full `populationSize` + when the distiller yielded nothing (this is the meta-harness contract — the + agent diagnoses from the raw traces itself). A patch-applier + (`reflectiveGenerator`) leaves it unset — with no findings there is no + patch to draft, so the driver short-circuits rather than spin up worktrees + for a guaranteed no-op. Default `false`. + +#### Methods -> **cleanup**(`retainWorktreeRefs?`): `Promise`\<`void`\> +##### generate() -Defined in: [src/improvement/improvement-driver.ts:98](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L98) +> **generate**(`args`): `Promise`\<\{ `applied`: `boolean`; `summary`: `string`; `label?`: `string`; `rationale?`: `string`; \}\> -Remove every owned candidate except explicitly retained finalized winners. +Defined in: [src/improvement/improvement-driver.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L42) ###### Parameters -###### retainWorktreeRefs? +###### args -readonly `string`[] +###### worktreePath + +`string` + +The candidate worktree — a clean checkout of the current incumbent. + +###### report + +`unknown` + +Phase-2 research report (analyst findings + diff), opaque. + +###### findings + +`AnalystFinding`[] + +Findings resolved from the report or the loop context. + +###### dataset? + +`LabeledScenarioStore` + +Handle to all captured data, to ground the change. + +###### maxShots + +`number` + +DEPTH: max iterations the generator may take (agentic uses this; the + reflective generator ignores it). + +###### signal + +`AbortSignal` + +###### generation? + +`number` + +Generation coordinates supplied by Runtime's internal code candidate driver. + +###### candidateIndex? + +`number` + +###### costLedger? + +`CostLedgerHandle` + +Shared run-wide paid-call account supplied by agent-eval 0.117+. + +###### costPhase? + +`string` + +Receipt attribution phase supplied alongside `costLedger`. ###### Returns -`Promise`\<`void`\> +`Promise`\<\{ `applied`: `boolean`; `summary`: `string`; `label?`: `string`; `rationale?`: `string`; \}\> *** @@ -6646,33 +7014,37 @@ Minimum tools the server must expose to pass. Default 1. *** -### ProfileDiffProposerOptions +### OfficialOptimizerContextOptions -Defined in: [src/improvement/profile-diff-proposer.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/profile-diff-proposer.ts#L24) +Defined in: src/improvement/official-optimizers.ts:22 -#### Type Parameters +Runtime context appended to an official optimizer's own configuration. -##### TFindings +#### Properties -`TFindings` = `unknown` +##### background? -#### Methods +> `optional` **background?**: `string` -##### proposeDiffs() +Defined in: src/improvement/official-optimizers.ts:24 -> **proposeDiffs**(`context`): readonly `AgentProfileDiff`[] \| `Promise`\ +Context supplied to the optimizer before Runtime appends the profile surface and findings. -Defined in: [src/improvement/profile-diff-proposer.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/profile-diff-proposer.ts#L25) +##### includeFindings? -###### Parameters +> `optional` **includeFindings?**: `boolean` -###### context +Defined in: src/improvement/official-optimizers.ts:26 -[`ProfileDiffProposerContext`](#profilediffproposercontext)\<`TFindings`\> +Include current trace or analyst findings in the optimizer background. Default true. -###### Returns +##### maxFindingsChars? + +> `optional` **maxFindingsChars?**: `number` -readonly `AgentProfileDiff`[] \| `Promise`\ +Defined in: src/improvement/official-optimizers.ts:28 + +Reject oversized serialized findings before starting Python. Default 50,000 characters. *** @@ -6733,7 +7105,7 @@ Findings to fall back to when the generation had NO failing cells, so a ### ReflectiveGeneratorOptions -Defined in: [src/improvement/reflective-generator.ts:21](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/reflective-generator.ts#L21) +Defined in: [src/improvement/reflective-generator.ts:20](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/reflective-generator.ts#L20) #### Properties @@ -6741,7 +7113,7 @@ Defined in: [src/improvement/reflective-generator.ts:21](https://github.com/tang > **improvementProposalSource**: [`ImprovementProposalSource`](analyst-loop.md#improvementproposalsource)\<[`SurfaceImprovementEdit`](agent.md#surfaceimprovementedit)\> -Defined in: [src/improvement/reflective-generator.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/reflective-generator.ts#L22) +Defined in: [src/improvement/reflective-generator.ts:21](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/reflective-generator.ts#L21) *** @@ -7341,7 +7713,7 @@ Defined in: [src/knowledge/supervised-update.ts:80](https://github.com/tangle-ne ### DelegatedLoopResult -Defined in: [src/loop-runner.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L67) +Defined in: [src/loop-runner.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L61) **`Experimental`** @@ -7360,7 +7732,7 @@ Uniform result — never throws from a registered runner; a > **mode**: `"code"` \| `"review"` \| `"research"` \| `"audit"` \| `"self-improve"` -Defined in: [src/loop-runner.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L68) +Defined in: [src/loop-runner.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L62) **`Experimental`** @@ -7368,7 +7740,7 @@ Defined in: [src/loop-runner.ts:68](https://github.com/tangle-network/agent-runt > **ok**: `boolean` -Defined in: [src/loop-runner.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L69) +Defined in: [src/loop-runner.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L63) **`Experimental`** @@ -7376,7 +7748,7 @@ Defined in: [src/loop-runner.ts:69](https://github.com/tangle-network/agent-runt > `optional` **output?**: `T` -Defined in: [src/loop-runner.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L70) +Defined in: [src/loop-runner.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L64) **`Experimental`** @@ -7384,7 +7756,7 @@ Defined in: [src/loop-runner.ts:70](https://github.com/tangle-network/agent-runt > `optional` **error?**: `string` -Defined in: [src/loop-runner.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L71) +Defined in: [src/loop-runner.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L65) **`Experimental`** @@ -7392,7 +7764,7 @@ Defined in: [src/loop-runner.ts:71](https://github.com/tangle-network/agent-runt > **durationMs**: `number` -Defined in: [src/loop-runner.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L72) +Defined in: [src/loop-runner.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L66) **`Experimental`** @@ -7400,7 +7772,7 @@ Defined in: [src/loop-runner.ts:72](https://github.com/tangle-network/agent-runt ### RunDelegatedLoopOptions -Defined in: [src/loop-runner.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L76) +Defined in: [src/loop-runner.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L70) **`Experimental`** @@ -7410,7 +7782,7 @@ Defined in: [src/loop-runner.ts:76](https://github.com/tangle-network/agent-runt > `optional` **signal?**: `AbortSignal` -Defined in: [src/loop-runner.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L77) +Defined in: [src/loop-runner.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L71) **`Experimental`** @@ -7418,7 +7790,7 @@ Defined in: [src/loop-runner.ts:77](https://github.com/tangle-network/agent-runt > `optional` **now?**: () => `number` -Defined in: [src/loop-runner.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L79) +Defined in: [src/loop-runner.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L73) **`Experimental`** @@ -7432,7 +7804,7 @@ Clock override for deterministic tests. ### WorktreeLoopRunnerOptions -Defined in: [src/loop-runner.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L121) +Defined in: [src/loop-runner.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L115) **`Experimental`** @@ -7444,7 +7816,7 @@ Options for the local-repo `code` runner over the GENERIC recursive path. > **repoRoot**: `string` -Defined in: [src/loop-runner.ts:123](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L123) +Defined in: [src/loop-runner.ts:117](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L117) **`Experimental`** @@ -7454,7 +7826,7 @@ Absolute path to the local git checkout each worktree is cut from. > **taskPrompt**: `string` -Defined in: [src/loop-runner.ts:125](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L125) +Defined in: [src/loop-runner.ts:119](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L119) **`Experimental`** @@ -7464,7 +7836,7 @@ The instruction handed to every authored harness (composed under each profile's > **harnesses**: readonly [`AuthoredHarness`](runtime.md#authoredharness)[] -Defined in: [src/loop-runner.ts:127](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L127) +Defined in: [src/loop-runner.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L121) **`Experimental`** @@ -7474,7 +7846,7 @@ The supervisor-authored harness profiles — one fanout item (one worktree-CLI l > **budget**: [`Budget`](runtime.md#budget-12) -Defined in: [src/loop-runner.ts:129](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L129) +Defined in: [src/loop-runner.ts:123](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L123) **`Experimental`** @@ -7484,7 +7856,7 @@ Conserved budget pool bounding the fanout (equal-k holds by construction). > `optional` **testCmd?**: `string` -Defined in: [src/loop-runner.ts:131](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L131) +Defined in: [src/loop-runner.ts:125](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L125) **`Experimental`** @@ -7494,7 +7866,7 @@ Shell command run in each worktree to derive the tests-PASS signal. > `optional` **typecheckCmd?**: `string` -Defined in: [src/loop-runner.ts:133](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L133) +Defined in: [src/loop-runner.ts:127](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L127) **`Experimental`** @@ -7504,7 +7876,7 @@ Shell command run in each worktree to derive the typecheck-PASS signal. > `optional` **require?**: readonly (`"tests"` \| `"typecheck"`)[] -Defined in: [src/loop-runner.ts:135](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L135) +Defined in: [src/loop-runner.ts:129](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L129) **`Experimental`** @@ -7514,7 +7886,7 @@ Which verification signals the deliverable REQUIRES present-and-passing (default > `optional` **maxDiffLines?**: `number` -Defined in: [src/loop-runner.ts:137](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L137) +Defined in: [src/loop-runner.ts:131](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L131) **`Experimental`** @@ -7524,7 +7896,7 @@ Diff-size cap (lines). > `optional` **forbiddenPaths?**: `string`[] -Defined in: [src/loop-runner.ts:139](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L139) +Defined in: [src/loop-runner.ts:133](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L133) **`Experimental`** @@ -7534,7 +7906,7 @@ Literal path prefixes the patch must not touch (the secret-floor is always on re > `optional` **winnerStrategy?**: [`WinnerStrategy`](runtime.md#winnerstrategy) -Defined in: [src/loop-runner.ts:141](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L141) +Defined in: [src/loop-runner.ts:135](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L135) **`Experimental`** @@ -7544,7 +7916,7 @@ Winner-selection strategy among gated candidates. Default `highest-score`. > `optional` **runGit?**: [`GitRunner`](mcp.md#gitrunner) -Defined in: [src/loop-runner.ts:143](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L143) +Defined in: [src/loop-runner.ts:137](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L137) **`Experimental`** @@ -7554,7 +7926,7 @@ Test seams forwarded to the worktree-CLI leaves so the runner drives offline. > `optional` **runHarness?**: (`options`) => `Promise`\<[`LocalHarnessResult`](mcp.md#localharnessresult)\> -Defined in: [src/loop-runner.ts:144](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L144) +Defined in: [src/loop-runner.ts:138](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L138) **`Experimental`** @@ -7595,7 +7967,7 @@ returning an incomplete reproducibility receipt. > `optional` **runCommand?**: `WorktreeCheckRunner` -Defined in: [src/loop-runner.ts:145](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L145) +Defined in: [src/loop-runner.ts:139](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L139) **`Experimental`** @@ -7603,7 +7975,7 @@ Defined in: [src/loop-runner.ts:145](https://github.com/tangle-network/agent-run ### VetoedFact -Defined in: [src/loop-runner.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L208) +Defined in: [src/loop-runner.ts:202](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L202) **`Experimental`** @@ -7615,7 +7987,7 @@ A fact rejected at the KB gate — surfaced, never dropped. > **candidate**: [`FactCandidate`](mcp.md#factcandidate) -Defined in: [src/loop-runner.ts:209](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L209) +Defined in: [src/loop-runner.ts:203](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L203) **`Experimental`** @@ -7623,7 +7995,7 @@ Defined in: [src/loop-runner.ts:209](https://github.com/tangle-network/agent-run > `optional` **vetoedBy?**: `string` -Defined in: [src/loop-runner.ts:210](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L210) +Defined in: [src/loop-runner.ts:204](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L204) **`Experimental`** @@ -7631,7 +8003,7 @@ Defined in: [src/loop-runner.ts:210](https://github.com/tangle-network/agent-run > `optional` **reason?**: `string` -Defined in: [src/loop-runner.ts:211](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L211) +Defined in: [src/loop-runner.ts:205](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L205) **`Experimental`** @@ -7639,7 +8011,7 @@ Defined in: [src/loop-runner.ts:211](https://github.com/tangle-network/agent-run ### ResearchLoopResult -Defined in: [src/loop-runner.ts:215](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L215) +Defined in: [src/loop-runner.ts:209](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L209) **`Experimental`** @@ -7649,7 +8021,7 @@ Defined in: [src/loop-runner.ts:215](https://github.com/tangle-network/agent-run > **accepted**: [`FactCandidate`](mcp.md#factcandidate)[] -Defined in: [src/loop-runner.ts:217](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L217) +Defined in: [src/loop-runner.ts:211](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L211) **`Experimental`** @@ -7659,7 +8031,7 @@ Facts that passed the fail-closed gate — safe to write to the KB. > **vetoed**: [`VetoedFact`](#vetoedfact)[] -Defined in: [src/loop-runner.ts:219](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L219) +Defined in: [src/loop-runner.ts:213](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L213) **`Experimental`** @@ -7669,7 +8041,7 @@ Facts the gate vetoed in the final round — escalate, do not silently drop. > **rounds**: `number` -Defined in: [src/loop-runner.ts:221](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L221) +Defined in: [src/loop-runner.ts:215](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L215) **`Experimental`** @@ -7679,7 +8051,7 @@ Research rounds actually run. ### ResearchLoopRunnerOptions -Defined in: [src/loop-runner.ts:225](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L225) +Defined in: [src/loop-runner.ts:219](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L219) **`Experimental`** @@ -7691,7 +8063,7 @@ Options for the default `research` runner. > **research**: (`round`, `vetoed`) => `Promise`\<[`FactCandidate`](mcp.md#factcandidate)[]\> -Defined in: [src/loop-runner.ts:232](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L232) +Defined in: [src/loop-runner.ts:226](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L226) **`Experimental`** @@ -7718,7 +8090,7 @@ Returns fact candidates carrying their grounding (`verbatimPassage` + > `optional` **gate?**: [`CreateKbGateOptions`](mcp.md#createkbgateoptions) -Defined in: [src/loop-runner.ts:234](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L234) +Defined in: [src/loop-runner.ts:228](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L228) **`Experimental`** @@ -7728,7 +8100,7 @@ Gate config (extra judges, self-artifact kinds, …). The floor is always on. > `optional` **maxRounds?**: `number` -Defined in: [src/loop-runner.ts:236](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L236) +Defined in: [src/loop-runner.ts:230](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L230) **`Experimental`** @@ -11441,7 +11813,7 @@ Defined in: [src/conversation/types.ts:246](https://github.com/tangle-network/ag > **Verifier** = (`worktreePath`, `signal?`) => `Promise`\<[`VerifyResult`](#verifyresult)\> \| [`VerifyResult`](#verifyresult) -Defined in: [src/improvement/agentic-generator.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L83) +Defined in: [src/improvement/agentic-generator.ts:82](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L82) Verifies the edited worktree. Sync or async; throws only on a setup fault (a candidate that fails verification returns `{ok:false}`, it does not @@ -11467,7 +11839,7 @@ Verifies the edited worktree. Sync or async; throws only on a setup fault > **AgenticGeneratorShotExecution** = `Readonly`\<`Omit`\<[`LocalHarnessResult`](mcp.md#localharnessresult), `"usage"` \| `"evidence"`\> & `object`\> -Defined in: [src/improvement/agentic-generator.ts:129](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L129) +Defined in: [src/improvement/agentic-generator.ts:128](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L128) Frozen exact harness result for an author shot: full streams, process state, token usage, and execution-policy evidence. @@ -11476,55 +11848,196 @@ Frozen exact harness result for an author shot: full streams, process state, *** -### AgenticGeneratorShotDisposition +### AgenticGeneratorShotDisposition + +> **AgenticGeneratorShotDisposition** = \{ `kind`: `"clean"`; `worktreePath`: `string`; \} \| \{ `kind`: `"rejected"`; `worktreePath`: `string`; `stage`: `"raw-trace-evidence"` \| `"verification"`; `feedback`: `string` \| `null`; \} \| \{ `kind`: `"accepted"`; `worktreePath`: `string`; `verified`: `boolean`; \} \| \{ `kind`: `"setup-error"`; `worktreePath`: `string`; `stage`: `"worktree-inspection"` \| `"raw-trace-evidence"` \| `"verification"`; `error`: \{ `name`: `string`; `message`: `string`; \}; \} + +Defined in: [src/improvement/agentic-generator.ts:141](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L141) + +Worktree decision emitted before a completed shot is retried, accepted, or + discarded. The callback runs while `worktreePath` is still available, so + callers can persist the exact diff. + +*** + +### ImproveSurface + +> **ImproveSurface** = `"prompt"` \| `"skills"` \| `"tools"` \| `"mcp"` \| `"hooks"` \| `"subagents"` \| `"agent-profile"` \| `"memory"` \| `"code"` \| `"rollout-policy"` + +Defined in: [src/improvement/improve.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L63) + +The executable agent lever `improve` optimizes. Profile fields remain + portable AgentProfile coordinates; implementation and orchestration files + use the code surface so a winner can be sealed into an exact candidate. + `rollout-policy` is the inference-time structuralRollout dials + (`profile.extensions['structural-rollout']`). + +*** + +### ImproveProfileSurface + +> **ImproveProfileSurface** = `Exclude`\<[`ImproveSurface`](#improvesurface), `"code"`\> + +Defined in: [src/improvement/improve.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L75) + +*** + +### ImproveMethodFactory + +> **ImproveMethodFactory**\<`TScenario`, `TArtifact`\> = (`context`) => `OptimizationMethod`\<`TScenario`, `TArtifact`\> + +Defined in: [src/improvement/improve.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L89) + +Build a complete method after trace findings are available. + +#### Type Parameters + +##### TScenario + +`TScenario` *extends* `Scenario` + +##### TArtifact + +`TArtifact` + +#### Parameters + +##### context + +[`ImproveMethodContext`](#improvemethodcontext) + +#### Returns + +`OptimizationMethod`\<`TScenario`, `TArtifact`\> + +*** + +### ImproveMethodSource + +> **ImproveMethodSource**\<`TScenario`, `TArtifact`\> = `OptimizationMethod`\<`TScenario`, `TArtifact`\> \| [`ImproveMethodFactory`](#improvemethodfactory)\<`TScenario`, `TArtifact`\> + +Defined in: [src/improvement/improve.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L93) + +#### Type Parameters + +##### TScenario + +`TScenario` *extends* `Scenario` + +##### TArtifact + +`TArtifact` + +*** + +### ImproveMethodOptions + +> **ImproveMethodOptions**\<`TScenario`, `TArtifact`\> = `Omit`\<`CompareOptimizationMethodsOptions`\<`TScenario`, `TArtifact`\>, `"baselineSurface"` \| `"dispatchWithSurface"` \| `"methods"` \| `"optimizationConcurrency"`\> & `object` + +Defined in: [src/improvement/improve.ts:98](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L98) + +Complete-method configuration for every non-code profile surface. + +#### Type Declaration + +##### surface? + +> `optional` **surface?**: [`ImproveProfileSurface`](#improveprofilesurface) + +Exact profile coordinate optimized by `method`. Default `'prompt'`. + +##### method + +> **method**: [`ImproveMethodSource`](#improvemethodsource)\<`TScenario`, `TArtifact`\> + +A complete optimizer or a factory that can incorporate current findings. + +##### agent + +> **agent**: (`surface`, `scenario`, `ctx`) => `Promise`\<`TArtifact`\> + +Runs one candidate surface on one scenario. + +###### Parameters + +###### surface + +`MutableSurface` + +###### scenario + +`TScenario` + +###### ctx + +`Parameters`\<`CompareOptimizationMethodsOptions`\<`TScenario`, `TArtifact`\>\[`"dispatchWithSurface"`\]\>\[`2`\] + +###### Returns + +`Promise`\<`TArtifact`\> + +##### findings? + +> `optional` **findings?**: readonly `unknown`[] + +Trace or analyst findings available to a method factory. + +##### skills? + +> `optional` **skills?**: [`ImproveSkillsOptions`](#improveskillsoptions) + +Select the exact inline skill document for `surface: 'skills'`. + +##### profileComponents? + +> `optional` **profileComponents?**: [`ImproveProfileComponents`](#improveprofilecomponents) + +Map a profile to named text components and apply the winning components. +Valid only with `surface: 'agent-profile'`. -> **AgenticGeneratorShotDisposition** = \{ `kind`: `"clean"`; `worktreePath`: `string`; \} \| \{ `kind`: `"rejected"`; `worktreePath`: `string`; `stage`: `"raw-trace-evidence"` \| `"verification"`; `feedback`: `string` \| `null`; \} \| \{ `kind`: `"accepted"`; `worktreePath`: `string`; `verified`: `boolean`; \} \| \{ `kind`: `"setup-error"`; `worktreePath`: `string`; `stage`: `"worktree-inspection"` \| `"raw-trace-evidence"` \| `"verification"`; `error`: \{ `name`: `string`; `message`: `string`; \}; \} +##### minimumLift? -Defined in: [src/improvement/agentic-generator.ts:142](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L142) +> `optional` **minimumLift?**: `number` -Worktree decision emitted before a completed shot is retried, accepted, or - discarded. The callback runs while `worktreePath` is still available, so - callers can persist the exact diff. +Ship only when the paired final-test interval is entirely above this lift. Default `0`. -*** +#### Type Parameters -### ImproveSurface +##### TScenario -> **ImproveSurface** = `"prompt"` \| `"skills"` \| `"tools"` \| `"mcp"` \| `"hooks"` \| `"subagents"` \| `"agent-profile"` \| `"memory"` \| `"code"` \| `"rollout-policy"` +`TScenario` *extends* `Scenario` -Defined in: [src/improvement/improve.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L89) +##### TArtifact -The executable agent lever `improve` optimizes. Profile fields remain - portable AgentProfile coordinates; implementation and orchestration files - use the code surface so a winner can be sealed into an exact candidate. - `rollout-policy` is the inference-time structuralRollout dials - (`profile.extensions['structural-rollout']`). +`TArtifact` *** -### ImproveOptions +### ImproveCodeRunOptions -> **ImproveOptions**\<`TScenario`, `TArtifact`\> = `Omit`\<`SelfImproveOptions`\<`TScenario`, `TArtifact`\>, `"analyzeGeneration"` \| `"baselineSurface"` \| `"findings"` \| `"gate"` \| `"proposer"`\> & `object` +> **ImproveCodeRunOptions**\<`TScenario`, `TArtifact`\> = `Omit`\<`SelfImproveOptions`\<`TScenario`, `TArtifact`\>, `"analyzeGeneration"` \| `"baselineSurface"` \| `"budget"` \| `"findings"` \| `"gate"` \| `"llm"` \| `"method"` \| `"mutationPrimitives"` \| `"proposer"` \| `"proposerTarget"` \| `"selectionScenarios"`\> & `object` -Defined in: [src/improvement/improve.ts:101](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L101) +Defined in: [src/improvement/improve.ts:128](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L128) + +Runtime-owned code search in isolated git worktrees. #### Type Declaration -##### surface? +##### surface + +> **surface**: `"code"` -> `optional` **surface?**: [`ImproveSurface`](#improvesurface) +##### budget? -Which profile lever to optimize. Default `'prompt'`. Selects the default - generator + the baseline-surface extraction shape. +> `optional` **budget?**: `Omit`\<`SelfImproveBudget`, `"selectionFraction"`\> -##### generator? +Local code-search budget. Method-only selection controls do not apply. + +##### findings? -> `optional` **generator?**: `SurfaceProposer` +> `optional` **findings?**: readonly `unknown`[] -The `SurfaceProposer` that mutates a profile surface. When unset, the facade - picks the default for prompt, skills, and memory; surfaces - with no default REQUIRE this (fail-loud otherwise). Forbidden for code; - use `code.generator` so the runtime owns candidate cleanup. +Findings supplied to Runtime's code candidate driver. ##### gate? @@ -11533,61 +12046,26 @@ The `SurfaceProposer` that mutates a profile surface. When unset, the facade Gate mode. `'holdout'` (default) runs the held-out promotion gate; `'none'` is a baseline-only run (`budget.generations = 0`). -##### allowedModels? - -> `optional` **allowedModels?**: readonly `string`[] - -Restrict the run to this subset of models. When set, the reflection model - (`llm.model`, or the default when unset) must be a member, or `improve()` throws - a `ConfigError` before the generator is built. Unset = unrestricted. - ##### analyzeGeneration? > `optional` **analyzeGeneration?**: `SelfImproveOptions`\<`TScenario`, `TArtifact`\>\[`"analyzeGeneration"`\] \| `null` -Per-generation findings producer passthrough (see selfImprove.analyzeGeneration). - DEFAULT: with a real (non-`mem://`) `runDir`, the raw-trace distiller - (`rawTraceDistiller`) — typed `AnalystFinding`s pointing the proposer at the - prior generation's actual on-disk traces; for in-memory runs (no traces on - disk to point at), the built-in failure distiller — the worst-scoring/errored - cells distilled into typed `AnalystFinding`s for the NEXT proposal round. - Pass your own producer to replace either; pass `null` to disable and keep the - static `findings` all the way through. +Per-generation findings producer for Runtime's code search. +Pass your own producer to replace the code-trace distiller; pass `null` +to keep the static findings for every generation. ##### rawTraceContext? > `optional` **rawTraceContext?**: `boolean` -META-HARNESS mode: instead of the distilled findings, feed the proposer - RAW-TRACE FILESYSTEM CONTEXT — the PATHS into the prior generation's - real run traces under `runDir` (per-cell `spans.jsonl` event logs + - `cached-result.json` scores + artifacts) plus a `grep`/`cat`-to-diagnose - instruction — so the coding agent reads the actual failures itself rather than - a pre-summary. Unset (default): raw-trace findings whenever the run is durable - (a real `runDir` — that is where the traces live), the distilled failure digest - otherwise; the `memory` surface always defaults to its curation distiller. - `true` forces `rawTraceDistiller()` even for an in-memory run (it emits a loud - warning finding instead of paths); `false` forces the digest distiller even - with a real `runDir`. Ignored when `analyzeGeneration` is set explicitly - (that wins) or is `null` (disabled). - -##### code? - -> `optional` **code?**: [`ImproveCodeOptions`](#improvecodeoptions) - -CODE-surface wiring: name `surface: 'code'`, point at a repo, and the - facade assembles the whole candidate pipeline — an isolated incumbent plus git worktrees - (`gitWorktreeAdapter`) driven by `improvementDriver` with the full agentic - generator (a real coding harness edits each candidate worktree; a `verify` - hook gates candidates before they are ever measured). Ignored when - `opts.generator` is supplied. Required for every code run because a real - repository and base ref are necessary to measure the incumbent. +Feed code candidates paths to prior raw traces instead of a failure digest. +Defaults to true for durable runs and false for in-memory runs. -##### skills? +##### code -> `optional` **skills?**: [`ImproveSkillsOptions`](#improveskillsoptions) +> **code**: [`ImproveCodeOptions`](#improvecodeoptions) -Select the exact inline skill document to optimize. +Isolated repository and candidate generator settings. ##### promotionGate? @@ -11608,23 +12086,81 @@ Custom held-back-exam decision. The string `gate` above controls whether *** -### ProfileDiffProposerContext +### ImproveOptions -> **ProfileDiffProposerContext**\<`TFindings`\> = `ProposeContext`\<`TFindings`\> & `object` +> **ImproveOptions**\<`TScenario`, `TArtifact`\> = [`ImproveMethodOptions`](#improvemethodoptions)\<`TScenario`, `TArtifact`\> \| [`ImproveCodeRunOptions`](#improvecoderunoptions)\<`TScenario`, `TArtifact`\> -Defined in: [src/improvement/profile-diff-proposer.ts:20](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/profile-diff-proposer.ts#L20) +Defined in: [src/improvement/improve.ts:165](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L165) -#### Type Declaration +The canonical improvement API: complete methods for profiles, worktrees for code. -##### profile +#### Type Parameters + +##### TScenario + +`TScenario` *extends* `Scenario` + +##### TArtifact + +`TArtifact` + +*** + +### ImproveResult + +> **ImproveResult**\<`TScenario`, `TArtifact`\> = [`ImproveMethodResult`](#improvemethodresult) \| [`ImproveCodeResult`](#improvecoderesult)\<`TScenario`, `TArtifact`\> + +Defined in: [src/improvement/improve.ts:267](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L267) + +#### Type Parameters + +##### TScenario + +`TScenario` *extends* `Scenario` + +##### TArtifact + +`TArtifact` + +*** + +### OfficialGepaOptions + +> **OfficialGepaOptions**\<`TScenario`, `TArtifact`\> = `Omit`\<`GepaOptimizationMethodConfig`\<`TScenario`, `TArtifact`\>, `"background"`\> & [`OfficialOptimizerContextOptions`](#officialoptimizercontextoptions) + +Defined in: src/improvement/official-optimizers.ts:32 + +Official GEPA configuration plus bounded Runtime findings context. + +#### Type Parameters + +##### TScenario + +`TScenario` *extends* `object` + +##### TArtifact + +`TArtifact` = `unknown` -> **profile**: `AgentProfile` +*** + +### OfficialSkillOptOptions + +> **OfficialSkillOptOptions**\<`TScenario`, `TArtifact`\> = `Omit`\<`SkillOptOptimizationMethodConfig`\<`TScenario`, `TArtifact`\>, `"background"`\> & [`OfficialOptimizerContextOptions`](#officialoptimizercontextoptions) + +Defined in: src/improvement/official-optimizers.ts:39 + +Official SkillOpt configuration plus bounded Runtime findings context. #### Type Parameters -##### TFindings +##### TScenario + +`TScenario` *extends* `object` + +##### TArtifact -`TFindings` = `unknown` +`TArtifact` = `unknown` *** @@ -11676,7 +12212,7 @@ Defined in: [src/knowledge/supervised-update.ts:87](https://github.com/tangle-ne > **DelegatedLoopMode** = *typeof* [`DELEGATED_LOOP_MODES`](#delegated_loop_modes)\[`number`\] -Defined in: [src/loop-runner.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L50) +Defined in: [src/loop-runner.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L44) **`Experimental`** @@ -11686,7 +12222,7 @@ Defined in: [src/loop-runner.ts:50](https://github.com/tangle-network/agent-runt > **DelegatedLoopRunner**\<`T`\> = (`signal`) => `Promise`\<`T`\> -Defined in: [src/loop-runner.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L59) +Defined in: [src/loop-runner.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L53) **`Experimental`** @@ -11715,7 +12251,7 @@ A pre-configured loop for one mode. Returns the mode's raw > **DelegatedLoopRegistry** = `Partial`\<`Record`\<[`DelegatedLoopMode`](#delegatedloopmode), [`DelegatedLoopRunner`](#delegatedlooprunner)\>\> -Defined in: [src/loop-runner.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L63) +Defined in: [src/loop-runner.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L57) **`Experimental`** @@ -12351,7 +12887,7 @@ Hard cap on chained gateway hops; refused beyond this. Default keeps recursion b > `const` **AGENTIC\_PROFILE\_RESOURCE\_ROOT**: `".agent-runtime-profile-resources"` = `'.agent-runtime-profile-resources'` -Defined in: [src/improvement/agentic-generator.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L71) +Defined in: [src/improvement/agentic-generator.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L70) Dedicated ephemeral root for generic author-profile files. Every declared file must live below this root so cleanup cannot alter candidate-owned files. @@ -12420,75 +12956,12 @@ to the strategy contract (author-blind, conserved budget, one module out). > `const` **ROLLOUT\_POLICY\_EXTENSION**: `"structural-rollout"` = `'structural-rollout'` -Defined in: [src/improvement/rollout-policy.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/rollout-policy.ts#L39) +Defined in: [src/improvement/rollout-policy.ts:11](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/rollout-policy.ts#L11) The profile extensions namespace the policy persists under. *** -### ROLLOUT\_POLICY\_BOUNDS - -> `const` **ROLLOUT\_POLICY\_BOUNDS**: `object` - -Defined in: [src/improvement/rollout-policy.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/rollout-policy.ts#L45) - -Proposal bounds per dial. These are the SEARCH bounds (what the proposer may - explore), chosen so every reachable value is a measured-sane recipe: k=1 is the - low-compute preset, testgen=0 disables check authoring, repairRounds caps where - the measured increment flattens (+1–3pp beyond round 2). - -#### Type Declaration - -##### k - -> `readonly` **k**: `object` - -###### k.min - -> `readonly` **min**: `1` = `1` - -###### k.max - -> `readonly` **max**: `10` = `10` - -###### k.step - -> `readonly` **step**: `2` = `2` - -##### repairRounds - -> `readonly` **repairRounds**: `object` - -###### repairRounds.min - -> `readonly` **min**: `0` = `0` - -###### repairRounds.max - -> `readonly` **max**: `3` = `3` - -###### repairRounds.step - -> `readonly` **step**: `1` = `1` - -##### testgen - -> `readonly` **testgen**: `object` - -###### testgen.min - -> `readonly` **min**: `0` = `0` - -###### testgen.max - -> `readonly` **max**: `10` = `10` - -###### testgen.step - -> `readonly` **step**: `3` = `3` - -*** - ### RESEARCH\_SUPERVISOR\_SYSTEM\_PROMPT > `const` **RESEARCH\_SUPERVISOR\_SYSTEM\_PROMPT**: `string` @@ -12503,7 +12976,7 @@ Standing prompt for a supervisor that grows a shared knowledge base through spaw > `const` **DELEGATED\_LOOP\_MODES**: readonly \[`"code"`, `"review"`, `"research"`, `"audit"`, `"self-improve"`\] -Defined in: [src/loop-runner.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L47) +Defined in: [src/loop-runner.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L41) **`Experimental`** @@ -13707,7 +14180,7 @@ Wire integration: > **agenticGenerator**(`opts?`): [`CandidateGenerator`](#candidategenerator) -Defined in: [src/improvement/agentic-generator.ts:214](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L214) +Defined in: [src/improvement/agentic-generator.ts:213](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L213) Full-agentic `CandidateGenerator` (the `shots=N, sandbox=on` setting): run a real coding harness inside the candidate worktree so the agent makes the change in place. @@ -13727,7 +14200,7 @@ Full-agentic `CandidateGenerator` (the `shots=N, sandbox=on` setting): run a rea > **defaultBuildPrompt**(`args`): `string` -Defined in: [src/improvement/agentic-generator.ts:793](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L793) +Defined in: [src/improvement/agentic-generator.ts:792](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L792) Turn the analyst's findings (+ optional report) into a concrete coder task — the senior scientific-method framing shared with the tool/MCP build prompts. @@ -13754,7 +14227,7 @@ Turn the analyst's findings (+ optional report) into a concrete coder task — > **commandVerifier**(`command`, `args?`, `timeoutMs?`): [`Verifier`](#verifier) -Defined in: [src/improvement/agentic-generator.ts:903](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L903) +Defined in: [src/improvement/agentic-generator.ts:902](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L902) A `Verifier` that runs a command in the worktree: exit 0 ⇒ ok, any other exit ⇒ failed with stdout+stderr as feedback. The common case — verify by @@ -13887,7 +14360,7 @@ Returns `[]` for empty/recordless content (a dispatch that never touched Defined in: [src/improvement/campaign-otlp.ts:136](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/campaign-otlp.ts#L136) -Walk `dir` (a campaign run dir, a generation dir, or a whole `selfImprove` +Walk `dir` (a campaign run dir, a generation dir, or a whole code-improvement run root) for `spans.jsonl` files and return their concatenated OTLP-flat JSONL — the exact string the `resolveTraces` contract expects. `''` when no spans exist (the proposers fail loud on empty by design). @@ -14006,73 +14479,104 @@ readonly `unknown`[] ### improve() -> **improve**\<`TScenario`, `TArtifact`\>(`profile`, `findings`, `opts`): `Promise`\<[`ImproveResult`](#improveresult)\<`TScenario`, `TArtifact`\>\> +#### Call Signature -Defined in: [src/improvement/improve.ts:663](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L663) +> **improve**\<`TScenario`, `TArtifact`\>(`profile`, `opts`): `Promise`\<[`ImproveMethodResult`](#improvemethodresult)\> -Run the held-out-gated self-improvement loop on ONE profile surface. +Defined in: [src/improvement/improve.ts:972](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L972) -#### Type Parameters +Optimize one exact profile surface with a complete method, or optimize code +through Runtime's isolated worktree path. The input profile is never changed. -##### TScenario +##### Type Parameters + +###### TScenario `TScenario` *extends* `Scenario` -##### TArtifact +###### TArtifact `TArtifact` -#### Parameters +##### Parameters -##### profile +###### profile `AgentProfile` -##### findings +###### opts -`unknown`[] +[`ImproveMethodOptions`](#improvemethodoptions)\<`TScenario`, `TArtifact`\> -##### opts +##### Returns -[`ImproveOptions`](#improveoptions)\<`TScenario`, `TArtifact`\> +`Promise`\<[`ImproveMethodResult`](#improvemethodresult)\> -#### Returns +#### Call Signature -`Promise`\<[`ImproveResult`](#improveresult)\<`TScenario`, `TArtifact`\>\> +> **improve**\<`TScenario`, `TArtifact`\>(`profile`, `opts`): `Promise`\<[`ImproveCodeResult`](#improvecoderesult)\<`TScenario`, `TArtifact`\>\> -#### Example +Defined in: [src/improvement/improve.ts:976](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L976) -```ts -Optimize the system prompt, default holdout gate: +Optimize one exact profile surface with a complete method, or optimize code +through Runtime's isolated worktree path. The input profile is never changed. - const out = await improve(profile, findings, { - surface: 'prompt', - scenarios, - judge, - agent: (surface, scenario, ctx) => runAgent(surface, scenario, ctx.signal), - }) - if (out.decision === 'ship') console.log(out.candidate) -``` +##### Type Parameters -*** +###### TScenario -### improvementDriver() +`TScenario` *extends* `Scenario` -> **improvementDriver**(`opts`): [`ManagedImprovementDriver`](#managedimprovementdriver) +###### TArtifact -Defined in: [src/improvement/improvement-driver.ts:102](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improvement-driver.ts#L102) +`TArtifact` -The one reflective/agentic improvement proposer (`SurfaceProposer`): owns the candidate worktree lifecycle and delegates HOW a change is produced to a pluggable `CandidateGenerator`. +##### Parameters -#### Parameters +###### profile -##### opts +`AgentProfile` -[`ImprovementDriverOptions`](#improvementdriveroptions) +###### opts -#### Returns +[`ImproveCodeRunOptions`](#improvecoderunoptions)\<`TScenario`, `TArtifact`\> + +##### Returns + +`Promise`\<[`ImproveCodeResult`](#improvecoderesult)\<`TScenario`, `TArtifact`\>\> + +#### Call Signature + +> **improve**\<`TScenario`, `TArtifact`\>(`profile`, `opts`): `Promise`\<[`ImproveResult`](#improveresult)\<`TScenario`, `TArtifact`\>\> + +Defined in: [src/improvement/improve.ts:980](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L980) + +Optimize one exact profile surface with a complete method, or optimize code +through Runtime's isolated worktree path. The input profile is never changed. + +##### Type Parameters + +###### TScenario + +`TScenario` *extends* `Scenario` + +###### TArtifact + +`TArtifact` + +##### Parameters + +###### profile + +`AgentProfile` + +###### opts -[`ManagedImprovementDriver`](#managedimprovementdriver) +[`ImproveOptions`](#improveoptions)\<`TScenario`, `TArtifact`\> + +##### Returns + +`Promise`\<[`ImproveResult`](#improveresult)\<`TScenario`, `TArtifact`\>\> *** @@ -14096,31 +14600,66 @@ Build a `Verifier` that boots a generated MCP server over stdio and checks it ex *** -### profileDiffProposer() +### officialGepa() + +> **officialGepa**\<`TScenario`, `TArtifact`\>(`options`): [`ImproveMethodFactory`](#improvemethodfactory)\<`TScenario`, `TArtifact`\> + +Defined in: src/improvement/official-optimizers.ts:75 + +Build a complete method backed by GEPA's official Optimize Anything API. + +The recipe is passed through unchanged. Use `engine`, `sequential`, +`adaptive-sequential`, `best-of`, `vote`, or `omni` explicitly. + +#### Type Parameters + +##### TScenario + +`TScenario` *extends* `object` + +##### TArtifact + +`TArtifact` = `unknown` + +#### Parameters + +##### options + +[`OfficialGepaOptions`](#officialgepaoptions)\<`TScenario`, `TArtifact`\> + +#### Returns + +[`ImproveMethodFactory`](#improvemethodfactory)\<`TScenario`, `TArtifact`\> + +*** + +### officialSkillOpt() -> **profileDiffProposer**\<`TFindings`\>(`options`): `SurfaceProposer`\<`TFindings`\> +> **officialSkillOpt**\<`TScenario`, `TArtifact`\>(`options`): [`ImproveMethodFactory`](#improvemethodfactory)\<`TScenario`, `TArtifact`\> -Defined in: [src/improvement/profile-diff-proposer.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/profile-diff-proposer.ts#L35) +Defined in: src/improvement/official-optimizers.ts:97 -Turn exact AgentProfileDiffs from any source into full profile candidates for -the shared optimization loop. Research, catalogs, humans, and trace miners -differ only in `proposeDiffs`; measurement and promotion stay identical. +Build a complete method backed by Microsoft's official SkillOpt trainer. #### Type Parameters -##### TFindings +##### TScenario + +`TScenario` *extends* `object` + +##### TArtifact -`TFindings` = `unknown` +`TArtifact` = `unknown` #### Parameters ##### options -[`ProfileDiffProposerOptions`](#profilediffproposeroptions)\<`TFindings`\> +[`OfficialSkillOptOptions`](#officialskilloptoptions)\<`TScenario`, `TArtifact`\> #### Returns -`SurfaceProposer`\<`TFindings`\> +[`ImproveMethodFactory`](#improvemethodfactory)\<`TScenario`, `TArtifact`\> *** @@ -14128,16 +14667,17 @@ differ only in `proposeDiffs`; measurement and promotion stay identical. > **rawTraceDistiller**\<`TScenario`, `TArtifact`\>(`options?`): (`input`) => `Promise`\<`unknown`[]\> -Defined in: [src/improvement/raw-trace-distiller.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/raw-trace-distiller.ts#L88) +Defined in: [src/improvement/raw-trace-distiller.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/raw-trace-distiller.ts#L89) Build an `analyzeGeneration` producer that feeds the proposer RAW-TRACE FILESYSTEM CONTEXT — paths into the prior generation's real run traces plus a grep/cat-to-diagnose instruction — instead of a pre-summarized digest. -Drop-in for `opts.analyzeGeneration` on `improve()` / `selfImprove()`: +Drop-in for `analyzeGeneration` on `improve({ surface: 'code' })`: - await improve(profile, seedFindings, { + await improve(profile, { surface: 'code', + findings: seedFindings, code: { repoRoot }, runDir: '/abs/run', // MUST be a real path — the traces live here analyzeGeneration: rawTraceDistiller(), @@ -14170,7 +14710,7 @@ Drop-in for `opts.analyzeGeneration` on `improve()` / `selfImprove()`: > **reflectiveGenerator**(`opts`): [`CandidateGenerator`](#candidategenerator) -Defined in: [src/improvement/reflective-generator.ts:26](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/reflective-generator.ts#L26) +Defined in: [src/improvement/reflective-generator.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/reflective-generator.ts#L25) Cheap no-sandbox `CandidateGenerator` (the `shots=1` setting): draft surface edits via the improvement adapter and apply them as one coherent candidate. @@ -14190,14 +14730,11 @@ Cheap no-sandbox `CandidateGenerator` (the `shots=1` setting): draft surface edi > **parseRolloutPolicy**(`surface`): [`StructuralRolloutPolicy`](runtime.md#structuralrolloutpolicy) \| `undefined` -Defined in: [src/improvement/rollout-policy.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/rollout-policy.ts#L66) +Defined in: [src/improvement/rollout-policy.ts:19](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/rollout-policy.ts#L19) -Parse a serialized policy surface. Defensive by design — the proposer reads - `ctx.currentSurface`, which the loop types as `string | CodeSurface`. Returns - `undefined` (never throws) for non-strings, malformed JSON, or a shape that - violates the policy's own invariants: the no-op signal. Unknown dials are - dropped; `diverse`/`temperature` ride through untouched (the proposer never - mutates them — `diverse` is a measured paired null). +Parse a serialized policy surface. Returns `undefined` for non-strings, +malformed JSON, or values outside the policy invariants. Unknown fields are +dropped; supported optional fields are preserved. #### Parameters @@ -14215,7 +14752,7 @@ Parse a serialized policy surface. Defensive by design — the proposer reads > **normalizeRolloutPolicy**(`raw`): [`StructuralRolloutPolicy`](runtime.md#structuralrolloutpolicy) \| `undefined` -Defined in: [src/improvement/rollout-policy.ts:82](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/rollout-policy.ts#L82) +Defined in: [src/improvement/rollout-policy.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/rollout-policy.ts#L35) Normalize an untyped policy bag (a parsed surface or a profile extension) into a full `StructuralRolloutPolicy`, defaults merged. Returns `undefined` when any @@ -14239,10 +14776,9 @@ Normalize an untyped policy bag (a parsed surface or a profile extension) into > **serializeRolloutPolicy**(`policy`): `string` -Defined in: [src/improvement/rollout-policy.ts:102](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/rollout-policy.ts#L102) +Defined in: [src/improvement/rollout-policy.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/rollout-policy.ts#L54) -Stable serialization — dial order is fixed so identical policies produce - identical surfaces (the loop dedupes/hashes candidates by surface content). +Stable serialization with fixed field order. #### Parameters @@ -14260,11 +14796,10 @@ Stable serialization — dial order is fixed so identical policies produce > **structuralRolloutPolicyFromProfile**(`profile`): [`StructuralRolloutPolicy`](runtime.md#structuralrolloutpolicy) \| `undefined` -Defined in: [src/improvement/rollout-policy.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/rollout-policy.ts#L115) +Defined in: [src/improvement/rollout-policy.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/rollout-policy.ts#L66) Read the persisted policy off the profile. `undefined` when the profile does - not opt into structural rollout — the improve() surface no-ops then, because - tuning dials nothing consumes would ship dead config. + not opt into structural rollout. #### Parameters @@ -14282,10 +14817,9 @@ Read the persisted policy off the profile. `undefined` when the profile does > **applyRolloutPolicyToProfile**(`profile`, `policy`): `AgentProfile` -Defined in: [src/improvement/rollout-policy.ts:125](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/rollout-policy.ts#L125) +Defined in: [src/improvement/rollout-policy.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/rollout-policy.ts#L75) -Persist a policy into the profile's extensions namespace. Shallow copy; never - mutates the input profile (the applyWinnerToProfile contract). +Persist a detached policy under the profile extension without mutating the input. #### Parameters @@ -14303,50 +14837,6 @@ Persist a policy into the profile's extensions namespace. Shallow copy; never *** -### enumerateNeighborPolicies() - -> **enumerateNeighborPolicies**(`policy`): [`StructuralRolloutPolicy`](runtime.md#structuralrolloutpolicy)[] - -Defined in: [src/improvement/rollout-policy.ts:146](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/rollout-policy.ts#L146) - -All bounded single-dial neighbors of `policy`, in a fixed priority order: k - first (selection breadth carries 85–92% of the measured effect), then - repairRounds, then testgen. Steps clamp to the dial's bounds; clamped-to-no-op - and duplicate policies are dropped. - -#### Parameters - -##### policy - -[`StructuralRolloutPolicy`](runtime.md#structuralrolloutpolicy) - -#### Returns - -[`StructuralRolloutPolicy`](runtime.md#structuralrolloutpolicy)[] - -*** - -### rolloutPolicyProposer() - -> **rolloutPolicyProposer**(): `SurfaceProposer` - -Defined in: [src/improvement/rollout-policy.ts:188](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/rollout-policy.ts#L188) - -The deterministic `SurfaceProposer` for the `'rollout-policy'` surface. - -Each generation: parse the current policy surface, enumerate its bounded -single-dial neighbors, and return at most `min(populationSize, 4)` of them, -rotating the enumeration window by generation so successive generations explore -different neighbors when nothing promoted. Proposes NOTHING when the surface -carries no policy (the profile never opted in) — an empty proposal is the -loop-native no-op, mirroring `improvementDriver`'s no-findings behavior. - -#### Returns - -`SurfaceProposer` - -*** - ### createKnowledgeImprovementActivationExecutor() > **createKnowledgeImprovementActivationExecutor**(`options`): [`KnowledgeImprovementActivationExecutor`](#knowledgeimprovementactivationexecutor) @@ -14515,7 +15005,7 @@ Format the supervisor task with the KB root, readiness requirements, current fin > **isDelegatedLoopMode**(`value`): value is "code" \| "review" \| "research" \| "audit" \| "self-improve" -Defined in: [src/loop-runner.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L53) +Defined in: [src/loop-runner.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L47) **`Experimental`** @@ -14537,7 +15027,7 @@ value is "code" \| "review" \| "research" \| "audit" \| "self-improve" > **runDelegatedLoop**\<`T`\>(`mode`, `registry`, `options?`): `Promise`\<[`DelegatedLoopResult`](#delegatedloopresult)\<`T`\>\> -Defined in: [src/loop-runner.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L91) +Defined in: [src/loop-runner.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L85) **`Experimental`** @@ -14576,7 +15066,7 @@ config bug, not a silent no-op. A runner that throws is captured as > **worktreeLoopRunner**(`options`): [`DelegatedLoopRunner`](#delegatedlooprunner)\<`WorktreeHarnessResult`\> -Defined in: [src/loop-runner.ts:162](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L162) +Defined in: [src/loop-runner.ts:156](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L156) **`Experimental`** @@ -14606,7 +15096,7 @@ patch artifact, or throws when no candidate is delivered (fail loud, never a vac > **researchLoopRunner**(`o`): [`DelegatedLoopRunner`](#delegatedlooprunner)\<[`ResearchLoopResult`](#researchloopresult)\> -Defined in: [src/loop-runner.ts:249](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L249) +Defined in: [src/loop-runner.ts:243](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L243) **`Experimental`** @@ -14629,43 +15119,11 @@ never silently dropped) so the caller audits vs retries. *** -### selfImproveLoopRunner() - -> **selfImproveLoopRunner**\<`TScenario`, `TArtifact`\>(`options`): [`DelegatedLoopRunner`](#delegatedlooprunner)\<`SelfImproveResult`\<`TScenario`, `TArtifact`\>\> - -Defined in: [src/loop-runner.ts:280](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L280) - -**`Experimental`** - -`self-improve` mode — agent-eval's one-call closed improvement loop (held-out gated). - -#### Type Parameters - -##### TScenario - -`TScenario` *extends* `Scenario` - -##### TArtifact - -`TArtifact` - -#### Parameters - -##### options - -`SelfImproveOptions`\<`TScenario`, `TArtifact`\> - -#### Returns - -[`DelegatedLoopRunner`](#delegatedlooprunner)\<`SelfImproveResult`\<`TScenario`, `TArtifact`\>\> - -*** - ### auditLoopRunner() > **auditLoopRunner**\<`TProposal`, `TEdit`\>(`options`): [`DelegatedLoopRunner`](#delegatedlooprunner)\<[`RunAnalystLoopResult`](analyst-loop.md#runanalystloopresult)\<`TProposal`, `TEdit`\>\> -Defined in: [src/loop-runner.ts:291](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L291) +Defined in: [src/loop-runner.ts:274](https://github.com/tangle-network/agent-runtime/blob/main/src/loop-runner.ts#L274) **`Experimental`** diff --git a/docs/api/intelligence.md b/docs/api/intelligence.md index d9386a84..80a5445e 100644 --- a/docs/api/intelligence.md +++ b/docs/api/intelligence.md @@ -4766,7 +4766,7 @@ Analyze, search, then remeasure the resulting exact candidate before proposing i > **createAgentImprovementProposal**(`options`): `AgentImprovementProposal` -Defined in: [src/intelligence/improvement-cycle.ts:417](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L417) +Defined in: [src/intelligence/improvement-cycle.ts:422](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L422) Create the reviewable record only from a complete, recomputable experiment result. @@ -4786,7 +4786,7 @@ Create the reviewable record only from a complete, recomputable experiment resul > **reviewAgentImprovementProposal**(`inputProposal`, `input`): `AgentImprovementReview` -Defined in: [src/intelligence/improvement-cycle.ts:448](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L448) +Defined in: [src/intelligence/improvement-cycle.ts:453](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L453) Persist a human or tenant-policy decision bound to one exact proposal. @@ -4810,7 +4810,7 @@ Persist a human or tenant-policy decision bound to one exact proposal. > **createAgentImprovementActivation**(`inputProposal`, `inputReview`, `options`): `AgentImprovementActivation` -Defined in: [src/intelligence/improvement-cycle.ts:476](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L476) +Defined in: [src/intelligence/improvement-cycle.ts:481](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L481) Authorize product-owned writes only after the exact candidate was measured and approved. @@ -4838,7 +4838,7 @@ Authorize product-owned writes only after the exact candidate was measured and a > **verifyAgentImprovementProposal**(`input`): `AgentImprovementProposal` -Defined in: [src/intelligence/improvement-cycle.ts:518](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L518) +Defined in: [src/intelligence/improvement-cycle.ts:523](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L523) Validate a proposal and recompute every binding to its measured experiment. @@ -4858,7 +4858,7 @@ Validate a proposal and recompute every binding to its measured experiment. > **verifyAgentImprovementReview**(`input`): `AgentImprovementReview` -Defined in: [src/intelligence/improvement-cycle.ts:542](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L542) +Defined in: [src/intelligence/improvement-cycle.ts:547](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L547) Validate the canonical identity and wire shape of an improvement review. @@ -4878,7 +4878,7 @@ Validate the canonical identity and wire shape of an improvement review. > **verifyAgentImprovementActivation**(`input`): `AgentImprovementActivation` -Defined in: [src/intelligence/improvement-cycle.ts:550](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L550) +Defined in: [src/intelligence/improvement-cycle.ts:555](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L555) Validate activation authority against the exact proposal, review, experiment, and base state. @@ -4908,7 +4908,7 @@ Validate activation authority against the exact proposal, review, experiment, an > **verifyCandidateExecutionEvidence**(`input`, `options`): `CandidateExecutionEvidence` -Defined in: [src/intelligence/improvement-cycle.ts:584](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L584) +Defined in: [src/intelligence/improvement-cycle.ts:589](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/improvement-cycle.ts#L589) Recheck one Runtime receipt against its exact signed experiment cell. diff --git a/docs/api/mcp.md b/docs/api/mcp.md index 3cb4f7a5..df3a36aa 100644 --- a/docs/api/mcp.md +++ b/docs/api/mcp.md @@ -3219,7 +3219,7 @@ Present for reproducible Codex runs; generated and checked before model executio ### MemoryItem -Defined in: [src/mcp/memory-server.ts:34](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L34) +Defined in: [src/mcp/memory-server.ts:33](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L33) One row of agent memory: a crisp lesson/fact with provenance. @@ -3229,7 +3229,7 @@ One row of agent memory: a crisp lesson/fact with provenance. > **id**: `string` -Defined in: [src/mcp/memory-server.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L36) +Defined in: [src/mcp/memory-server.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L35) Stable id (content-hash by convention; see `memoryArtifactFromLessons`). @@ -3237,7 +3237,7 @@ Stable id (content-hash by convention; see `memoryArtifactFromLessons`). > **text**: `string` -Defined in: [src/mcp/memory-server.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L38) +Defined in: [src/mcp/memory-server.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L37) The lesson itself — one imperative or observation the agent should recall. @@ -3245,7 +3245,7 @@ The lesson itself — one imperative or observation the agent should recall. > `optional` **tags?**: `string`[] -Defined in: [src/mcp/memory-server.ts:40](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L40) +Defined in: [src/mcp/memory-server.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L39) Optional retrieval tags, matched by `memory_search` alongside the text. @@ -3253,7 +3253,7 @@ Optional retrieval tags, matched by `memory_search` alongside the text. > `optional` **source?**: `string` -Defined in: [src/mcp/memory-server.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L42) +Defined in: [src/mcp/memory-server.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L41) Provenance: the finding / trace / curation pass this row came from. @@ -3261,7 +3261,7 @@ Provenance: the finding / trace / curation pass this row came from. ### AgentMemorySpec -Defined in: [src/mcp/memory-server.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L59) +Defined in: [src/mcp/memory-server.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L58) The `memory` artifact payload — HOW a profile's memory is stored and served: @@ -3282,13 +3282,13 @@ The `memory` artifact payload — HOW a profile's memory is stored and served: > **store**: `"mcp"` \| `"file"` -Defined in: [src/mcp/memory-server.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L60) +Defined in: [src/mcp/memory-server.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L59) ##### path? > `optional` **path?**: `string` -Defined in: [src/mcp/memory-server.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L62) +Defined in: [src/mcp/memory-server.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L61) `store:'file'` — host path to the durable row store (JSON array or JSONL). @@ -3296,7 +3296,7 @@ Defined in: [src/mcp/memory-server.ts:62](https://github.com/tangle-network/agen > `optional` **items?**: [`MemoryItem`](#memoryitem)[] -Defined in: [src/mcp/memory-server.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L64) +Defined in: [src/mcp/memory-server.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L63) Inline seed rows, served alongside (and winning over) `path` rows. @@ -3304,7 +3304,7 @@ Inline seed rows, served alongside (and winning over) `path` rows. > `optional` **server?**: `AgentProfileMcpServer` -Defined in: [src/mcp/memory-server.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L66) +Defined in: [src/mcp/memory-server.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L65) `store:'mcp'` — the external server that already serves memory tools. @@ -3312,7 +3312,7 @@ Defined in: [src/mcp/memory-server.ts:66](https://github.com/tangle-network/agen > `optional` **logPath?**: `string` -Defined in: [src/mcp/memory-server.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L68) +Defined in: [src/mcp/memory-server.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L67) JSONL retrieval log: one row per `memory_search` (ts, query, k, returned). @@ -3320,7 +3320,7 @@ JSONL retrieval log: one row per `memory_search` (ts, query, k, returned). ### CreateMemoryToolServerOptions -Defined in: [src/mcp/memory-server.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L81) +Defined in: [src/mcp/memory-server.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L80) #### Properties @@ -3328,7 +3328,7 @@ Defined in: [src/mcp/memory-server.ts:81](https://github.com/tangle-network/agen > **items**: readonly [`MemoryItem`](#memoryitem)[] -Defined in: [src/mcp/memory-server.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L83) +Defined in: [src/mcp/memory-server.ts:82](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L82) The rows to serve. MUST be non-empty (an empty memory is never served). @@ -3336,7 +3336,7 @@ The rows to serve. MUST be non-empty (an empty memory is never served). > `optional` **serverName?**: `string` -Defined in: [src/mcp/memory-server.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L85) +Defined in: [src/mcp/memory-server.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L84) Server display name surfaced via `initialize`. Default 'agent-memory'. @@ -3344,7 +3344,7 @@ Server display name surfaced via `initialize`. Default 'agent-memory'. > `optional` **serverVersion?**: `string` -Defined in: [src/mcp/memory-server.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L87) +Defined in: [src/mcp/memory-server.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L86) Server version surfaced via `initialize`. Default '0'. @@ -3352,7 +3352,7 @@ Server version surfaced via `initialize`. Default '0'. > `optional` **defaultK?**: `number` -Defined in: [src/mcp/memory-server.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L89) +Defined in: [src/mcp/memory-server.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L88) Default result count for `memory_search`. Default 5. @@ -3360,7 +3360,7 @@ Default result count for `memory_search`. Default 5. > `optional` **logPath?**: `string` -Defined in: [src/mcp/memory-server.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L91) +Defined in: [src/mcp/memory-server.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L90) Append one JSONL row per `memory_search` (the retrieval-holdout seam). @@ -3368,7 +3368,7 @@ Append one JSONL row per `memory_search` (the retrieval-holdout seam). ### ResolvedMemoryEnv -Defined in: [src/mcp/memory-server.ts:261](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L261) +Defined in: [src/mcp/memory-server.ts:260](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L260) What the memory bin resolved from its environment. @@ -3378,19 +3378,19 @@ What the memory bin resolved from its environment. > **items**: [`MemoryItem`](#memoryitem)[] -Defined in: [src/mcp/memory-server.ts:262](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L262) +Defined in: [src/mcp/memory-server.ts:261](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L261) ##### serverName? > `optional` **serverName?**: `string` -Defined in: [src/mcp/memory-server.ts:263](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L263) +Defined in: [src/mcp/memory-server.ts:262](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L262) ##### logPath? > `optional` **logPath?**: `string` -Defined in: [src/mcp/memory-server.ts:264](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L264) +Defined in: [src/mcp/memory-server.ts:263](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L263) *** @@ -6803,7 +6803,7 @@ Default cap on the serialized trace payload per record, in bytes. > `const` **MEMORY\_FILE\_ENV**: `"AGENT_MEMORY_FILE"` = `'AGENT_MEMORY_FILE'` -Defined in: [src/mcp/memory-server.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L73) +Defined in: [src/mcp/memory-server.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L72) Env var naming the durable row store file the memory bin loads (the `memoryMcpServer` ↔ memory-bin contract). @@ -6814,7 +6814,7 @@ Env var naming the durable row store file the memory bin loads (the > `const` **MEMORY\_ITEMS\_ENV**: `"AGENT_MEMORY_ITEMS"` = `'AGENT_MEMORY_ITEMS'` -Defined in: [src/mcp/memory-server.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L75) +Defined in: [src/mcp/memory-server.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L74) Env var carrying inline JSON `MemoryItem` rows (win over file rows on id). @@ -6824,7 +6824,7 @@ Env var carrying inline JSON `MemoryItem` rows (win over file rows on id). > `const` **MEMORY\_LOG\_ENV**: `"AGENT_MEMORY_LOG"` = `'AGENT_MEMORY_LOG'` -Defined in: [src/mcp/memory-server.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L77) +Defined in: [src/mcp/memory-server.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L76) Env var naming the JSONL retrieval log (one row per `memory_search`). @@ -6834,7 +6834,7 @@ Env var naming the JSONL retrieval log (one row per `memory_search`). > `const` **MEMORY\_NAME\_ENV**: `"AGENT_MEMORY_NAME"` = `'AGENT_MEMORY_NAME'` -Defined in: [src/mcp/memory-server.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L79) +Defined in: [src/mcp/memory-server.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L78) Env var overriding the served display name (default 'agent-memory'). @@ -8132,7 +8132,7 @@ Parse and validate the one terminal usage event emitted by `codex exec --json`. > **createMemoryToolServer**(`opts`): [`StdioToolServer`](#stdiotoolserver) -Defined in: [src/mcp/memory-server.ts:98](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L98) +Defined in: [src/mcp/memory-server.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L97) Build the memory MCP server: `memory_search` (lexical top-k over the rows) and `memory_get` (one row by id) on the generic stdio JSON-RPC core. @@ -8153,7 +8153,7 @@ and `memory_get` (one row by id) on the generic stdio JSON-RPC core. > **parseMemoryItems**(`value`, `source`): [`MemoryItem`](#memoryitem)[] -Defined in: [src/mcp/memory-server.ts:213](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L213) +Defined in: [src/mcp/memory-server.ts:212](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L212) Coerce an untrusted JSON array into validated `MemoryItem` rows. @@ -8177,7 +8177,7 @@ Coerce an untrusted JSON array into validated `MemoryItem` rows. > **readMemoryItemsFile**(`path`): [`MemoryItem`](#memoryitem)[] -Defined in: [src/mcp/memory-server.ts:221](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L221) +Defined in: [src/mcp/memory-server.ts:220](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L220) Read a memory store file: a JSON array, or JSONL (one `MemoryItem` per line). @@ -8197,7 +8197,7 @@ Read a memory store file: a JSON array, or JSONL (one `MemoryItem` per line). > **resolveMemoryFromEnv**(`env`): [`ResolvedMemoryEnv`](#resolvedmemoryenv) -Defined in: [src/mcp/memory-server.ts:272](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L272) +Defined in: [src/mcp/memory-server.ts:271](https://github.com/tangle-network/agent-runtime/blob/main/src/mcp/memory-server.ts#L271) Resolve the bin's memory from `AGENT_MEMORY_FILE` (durable store) and/or `AGENT_MEMORY_ITEMS` (inline JSON rows; wins on id collision). Zero rows is diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 56e1a06c..42ea68e6 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.103.0` and `@tangle-network/agent-eval@0.123.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.104.0` and `@tangle-network/agent-eval@0.126.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -15,13 +15,13 @@ Every subpath this package declares in `package.json` `exports`. Reach for these ### Root — task lifecycle, conversation, RSI verbs, observability -Import from `@tangle-network/agent-runtime` — 378 exports. +Import from `@tangle-network/agent-runtime` — 380 exports. | Symbol | Kind | Summary | |---|---|---| | `agenticGenerator` | function | Full-agentic `CandidateGenerator` (the `shots=N, sandbox=on` setting): run a real coding harness inside the candidate worktree so the agent makes the change in place. | | `applyExactAgentProfileDiff` | function | Apply one exact diff and reject any value that cannot be preserved canonically. | -| `applyRolloutPolicyToProfile` | function | Persist a policy into the profile's extensions namespace. Shallow copy; never | +| `applyRolloutPolicyToProfile` | function | Persist a detached policy under the profile extension without mutating the input. | | `applyRunRecordDefaults` | function | Stamp cross-cutting defaults onto adapter-projected RunRecords without | | `assertCandidateProfileBinding` | function | Prove the measured generic profile and sealed candidate profile describe the same behavior. | | `auditLoopRunner` | function | `audit` mode — analyst loop over captured trace/run data. | @@ -31,8 +31,6 @@ Import from `@tangle-network/agent-runtime` — 378 exports. | `buildLoopOtelSpans` | function | Build a nested, real-duration OTLP span tree for ONE loop run from its full | | `buildLoopSpanNodes` | function | Sink-neutral core behind {@link buildLoopOtelSpans}: reconstruct the | | `buildRuntimeEventOtelSpans` | function | Convert normalized runtime events into lossless, redacted child spans. | -| `campaignCellSpansToOtlp` | function | Convert ONE cell's `spans.jsonl` content to OTLP-flat JSONL lines. | -| `campaignTraceResolver` | function | Build the `resolveTraces` function `traceAnalystProposer`/`haloProposer` | | `candidateExecutionClaim` | function | Extract the complete durable claim from a prepared execution. | | `candidateKnowledgeExecutionPaths` | function | Deterministic, signed locations used by every candidate executor. | | `captureAgentCandidateWorkspace` | function | Capture one exact regular-file workspace for immutable candidate execution. | @@ -41,7 +39,6 @@ Import from `@tangle-network/agent-runtime` — 378 exports. | `commandVerifier` | function | A `Verifier` that runs a command in the worktree: exit 0 ⇒ ok, any other | | `composeRuntimeHooks` | function | Merge several {@link RuntimeHooks} into one. Falsy entries are dropped (so you can | | `computeBackoff` | function | Compute the delay before the next attempt. Default: 250ms exponential with jitter. | -| `convertCampaignDirToOtlp` | function | Walk `dir` (a campaign run dir, a generation dir, or a whole `selfImprove` | | `createAgentCandidateWorkspacePort` | function | Create the standard bounded materializer for candidate execution ports. | | `createAgentKnowledgeReadinessCheck` | function | Build the default readiness check backed by `@tangle-network/agent-knowledge` validation and scoring. | | `createConversationBackend` | function | Wrap a `Conversation` so it satisfies `AgentExecutionBackend`. The result is | @@ -62,7 +59,6 @@ Import from `@tangle-network/agent-runtime` — 378 exports. | `deriveExecutionId` | function | Derive a stable executionId from the run identity. The same | | `disposePreparedAgentCandidateExecution` | function | Revoke reservations held by a prepared candidate that will not be executed. | | `driverLoopGenerator` | function | Driver→worker `CandidateGenerator`: an LLM driver on the canonical tool-loop authors, observes, rates, and steers coding-harness sessions in the worktree until the verifier passes or the session budge | -| `enumerateNeighborPolicies` | function | All bounded single-dial neighbors of `policy`, in a fixed priority order: k | | `exactProcessProviderAsCandidateExecutor` | function | Adapt one neutral exact-process provider to Runtime's trusted candidate boundary. | | `executePreparedAgentCandidate` | function | Executes and finalizes one durably claimed candidate without exposing an unproven result. | | `exportEvalRuns` | function | Ship self-improvement eval-run events to Tangle Intelligence. Unlike the | @@ -70,8 +66,7 @@ Import from `@tangle-network/agent-runtime` — 378 exports. | `formatSupervisedKnowledgeTask` | function | Format the supervisor task with the KB root, readiness requirements, current findings, and metadata. | | `getModels` | function | Fetch the model catalog from the router's `/v1/models`. Throws on a non-2xx | | `handleChatTurn` | function | Run one chat turn. Returns immediately with a `ReadableStream` body; | -| `improve` | function | Run the held-out-gated self-improvement loop on ONE profile surface. | -| `improvementDriver` | function | The one reflective/agentic improvement proposer (`SurfaceProposer`): owns the candidate worktree lifecycle and delegates HOW a change is produced to a pluggable `CandidateGenerator`. | +| `improve` | function | Optimize one exact profile surface with a complete method, or optimize code | | `isAnalystFinding` | function | Structural guard for the schema-versioned `AnalystFinding` envelope. | | `isDelegatedLoopMode` | function | Type guard — returns true when `value` is a valid `DelegatedLoopMode` string. | | `isDepthExceeded` | function | Refuse further forwarding when the inbound depth has reached the limit. | @@ -85,13 +80,14 @@ Import from `@tangle-network/agent-runtime` — 378 exports. | `normalizeRolloutPolicy` | function | Normalize an untyped policy bag (a parsed surface or a profile extension) into | | `notifyRuntimeDecisionPoint` | function | Fire `hooks.onDecisionPoint`, swallowing sync throws and surfacing async failures to `onError`. | | `notifyRuntimeHookEvent` | function | Fire `hooks.onEvent`, swallowing sync throws and surfacing async failures to `onError`. | +| `officialGepa` | function | Build a complete method backed by GEPA's official Optimize Anything API. | +| `officialSkillOpt` | function | Build a complete method backed by Microsoft's official SkillOpt trainer. | | `parseExactAgentProfile` | function | Parse a complete profile without silently discarding unsupported fields. | | `parseExactAgentProfileDiff` | function | Parse a profile diff without silently discarding unsupported fields. | | `parseLoopRunnerArgv` | function | Parse `--mode X --config Y` from an argv tail (`process.argv.slice(2)`). | -| `parseRolloutPolicy` | function | Parse a serialized policy surface. Defensive by design — the proposer reads | +| `parseRolloutPolicy` | function | Parse a serialized policy surface. Returns `undefined` for non-strings, | | `persistCandidateOutputArtifact` | function | Persist evaluator evidence, read it back, and bind the returned locator to the exact bytes. | | `prepareAgentCandidateExecution` | function | Materializes a verified candidate into one immutable evaluator-owned execution plan. | -| `profileDiffProposer` | function | Turn exact AgentProfileDiffs from any source into full profile candidates for | | `rawTraceDistiller` | function | Build an `analyzeGeneration` producer that feeds the proposer RAW-TRACE | | `readDepth` | function | Read the depth counter off an inbound request. Missing → 0 (caller is the | | `readinessServerSentEvent` | function | Serialize a `KnowledgeReadinessReport` as a Server-Sent Event string. | @@ -101,7 +97,6 @@ Import from `@tangle-network/agent-runtime` — 378 exports. | `resolveAgentBackend` | function | Resolve the `AgentExecutionBackend` for the chosen `kind`. Reuse this instead | | `resolveChatModel` | function | Resolve a chat model by precedence: the first candidate carrying a | | `resolveRouterBaseUrl` | function | Resolve the router base URL from env, normalised — no trailing `/v1` or `/`. | -| `rolloutPolicyProposer` | function | The deterministic `SurfaceProposer` for the `'rollout-policy'` surface. | | `runAgentTask` | function | Single-shot task lifecycle for adapter-driven tasks: readiness-gated, emits the runtime lifecycle event vocabulary, session-store pluggable. | | `runAgentTaskStream` | function | Streaming task lifecycle: delegates execution to an `AgentExecutionBackend` (model API, sandbox, or custom iterable) and yields lifecycle events as they happen. | | `runConversation` | function | Conversation orchestrator. Drives N participants in turn through their own | @@ -118,8 +113,7 @@ Import from `@tangle-network/agent-runtime` — 378 exports. | `sanitizeKnowledgeReadinessReport` | function | Strip PII and large blobs from a `KnowledgeReadinessReport` for safe telemetry emission. | | `sanitizeRuntimeStreamEvent` | function | Reduce a `RuntimeStreamEvent` to a PII-safe, serializable plain object for telemetry. | | `sealAgentCandidateBundle` | function | Validate and content-address a candidate bundle before it crosses an approval boundary. | -| `selfImproveLoopRunner` | function | `self-improve` mode — agent-eval's one-call closed improvement loop (held-out gated). | -| `serializeRolloutPolicy` | function | Stable serialization — dial order is fixed so identical policies produce | +| `serializeRolloutPolicy` | function | Stable serialization with fixed field order. | | `sleep` | function | Resolve after `ms` milliseconds — used for retry backoff in conversation call policy. | | `slugifySpeaker` | function | Reduce a speaker name to ASCII alphanumerics + dashes. Preserves enough | | `startRuntimeRun` | function | Construct a runtime-run handle. The returned handle is mutable across its | @@ -148,7 +142,6 @@ Import from `@tangle-network/agent-runtime` — 378 exports. | `optimizerMethod` | const | The shared method block every build/author prompt embeds. Domain framing | | `RESEARCH_SUPERVISOR_SYSTEM_PROMPT` | const | Standing prompt for a supervisor that grows a shared knowledge base through spawned researchers. | | `researchDriverNote` | const | The driver's ADOPT-not-build doctrine, appended to `buildDriverSystem` when | -| `ROLLOUT_POLICY_BOUNDS` | const | Proposal bounds per dial. These are the SEARCH bounds (what the proposer may | | `ROLLOUT_POLICY_EXTENSION` | const | The profile extensions namespace the policy persists under. | | `strategyAuthorMethod` | const | The senior authoring process for `authorStrategy` — the same method, shaped | | `AgentEvalError` | class | Base class for every contract error this package throws — carries the stable | @@ -164,6 +157,7 @@ Import from `@tangle-network/agent-runtime` — 378 exports. | `InMemoryRuntimeSessionStore` | class | In-memory `RuntimeSessionStore` for single-process use and tests. | | `JudgeError` | class | A judge call failed in a way that's not retryable: schema parse failure, bad rubric, conflicting dimensions. | | `NotFoundError` | class | A named resource (run, span, rubric, scenario, dataset row, route) does not exist. | +| `OfficialOptimizerUnavailableError` | class | Missing optional Python dependencies for an official optimizer. | | `PlannerError` | class | The dynamic-loop planner returned an unusable topology move — the LLM emitted | | `RuntimeRunStateError` | class | A runtime-run lifecycle method was called in an order the state machine does | | `SqlConversationJournal` | class | SQL-backed ConversationJournal. Two tables — runs (one row per runId, holds | @@ -191,7 +185,6 @@ Import from `@tangle-network/agent-runtime` — 378 exports. | `AgentCandidateWorkspacePort` | interface | Materializes an already-verified workspace archive. | | `BackendErrorDetail` | interface | Typed transport / backend failure detail. Carried on `backend_error` and | | `BuildAgentCandidateBundleInput` | interface | Complete measured surfaces and execution policy compiled into one candidate bundle. | -| `CampaignOtlpOptions` | interface | Campaign `spans.jsonl` → OTLP-flat JSONL — the missing converter between | | `CandidateGenerator` | interface | The byte-producing seam — the ONE thing that differs between the cheap | | `ChatStreamEvent` | interface | The NDJSON line protocol every product chat client already speaks. | | `ChatTurnIdentity` | interface | Identity of a chat turn. `tenantId` is the workspace id for workspace- | @@ -200,9 +193,13 @@ Import from `@tangle-network/agent-runtime` — 378 exports. | `ConversationJournalEntry` | interface | Durable conversation transcript — survives a driver process crash mid-run. | | `D1DatabaseLike` | interface | Structural type matching the surface of `D1Database` we depend on, so the | | `DriverLoopGeneratorOptions` | interface | `driverLoopGenerator` — the driver→worker `CandidateGenerator`: the build | +| `ImproveCost` | interface | Normalized spend reported for one Runtime improvement run. | +| `ImproveLineage` | interface | Optimizer ancestry sealed into downstream candidate experiments. | +| `ImproveProfileComponents` | interface | Caller-owned mapping for optimizing several profile fields as one candidate. | | `LoopSpanNode` | interface | Sink-neutral node in a reconstructed loop span tree. The root node's | | `McpServeSpec` | interface | `mcpServeVerifier` — the intrinsic verifier for a built MCP server: the | | `ModelInfo` | interface | A model entry as returned by the Tangle Router `/v1/models` endpoint. | +| `OfficialOptimizerContextOptions` | interface | Runtime context appended to an official optimizer's own configuration. | | `OpenAIChatTool` | interface | OpenAI Chat Completions tool descriptor. The shape mirrors the | | `OtelExportConfig` | interface | OTEL span exporter — streams LoopTraceEvents to an OTLP/HTTP collector. | | `PreparedAgentCandidateKnowledge` | interface | Exact file-backed knowledge admitted by the candidate bundle. | @@ -233,7 +230,13 @@ Import from `@tangle-network/agent-runtime` — 378 exports. | `AgentEvalErrorCode` | type | Error taxonomy for `@tangle-network/agent-eval`. | | `AgenticGeneratorShotDisposition` | type | Worktree decision emitted before a completed shot is retried, accepted, or | | `AgenticGeneratorShotExecution` | type | Frozen exact harness result for an author shot: full streams, process state, | +| `ImproveCodeRunOptions` | type | Runtime-owned code search in isolated git worktrees. | +| `ImproveMethodFactory` | type | Build a complete method after trace findings are available. | +| `ImproveMethodOptions` | type | Complete-method configuration for every non-code profile surface. | +| `ImproveOptions` | type | The canonical improvement API: complete methods for profiles, worktrees for code. | | `ImproveSurface` | type | The executable agent lever `improve` optimizes. Profile fields remain | +| `OfficialGepaOptions` | type | Official GEPA configuration plus bounded Runtime findings context. | +| `OfficialSkillOptOptions` | type | Official SkillOpt configuration plus bounded Runtime findings context. | | `OpenAIChatResponseFormat` | type | `response_format` parameter for OpenAI-compatible chat endpoints. Use | | `OpenAIChatToolChoice` | type | `tool_choice` parameter for OpenAI-compat chat. Same shape as the OpenAI | | `PersonaDriver` | type | A persona that drives the conversation: either a full driver `AgentProfile` | @@ -247,7 +250,7 @@ Import from `@tangle-network/agent-runtime` — 378 exports. | `VerifiedAgentCandidateTaskOutcome` | type | Branded task outcome that has survived independent evaluator verification. | | `Verifier` | type | Verifies the edited worktree. Sync or async; throws only on a setup fault | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidatePreparationEvidence`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateVerificationPorts`, `AgentCandidateWorkspaceArchiveLimits`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgenticGeneratorShotReceipt`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `BackendCallPolicy`, `CampaignTraceResolverOptions`, `CanonicalCandidateDocument`, `CaptureAgentCandidateWorkspaceOptions`, `CapturedAgentCandidateWorkspace`, `ChatTurnHooks`, `ChatTurnResult`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `CreateAgentCandidateWorkspacePortOptions`, `CreateKnowledgeImprovementActivationExecutorOptions`, `CreateProtectedAgentCandidateModelPortOptions`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `DisposePreparedAgentCandidateOptions`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `ExactProcessCandidateExecutorOptions`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `HaltContext`, `HaltSignal`, `ImproveCodeOptions`, `ImprovementCandidate`, `ImprovementDriverOptions`, `ImproveResult`, `ImproveSkillsOptions`, `KnowledgeImprovementActivationExecutor`, `KnowledgeImprovementCandidatePair`, `KnowledgeImprovementExperimentBundles`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `ManagedImprovementDriver`, `OtelAttribute`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `ProfileDiffProposerOptions`, `RecoverExpiredAgentCandidateOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolveAgentBackendOptions`, `ResolvedAgentCandidateContainer`, `ResolvedChatModel`, `RunChatTurnInput`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeEventOtelOptions`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunHandle`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeTelemetryOptions`, `RunToolLoopOptions`, `SanitizedKnowledgeReadinessReport`, `StreamToolLoopOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `ToAnalystFindingsOptions`, `ToolLoopResult`, `VerifiedAgentCandidate`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ControlDecision`, `ConversationStreamEvent`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `ImproveOptions`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `ProfileDiffProposerContext`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeStreamEvent`, `StreamToolLoopYield`, `SupervisedKnowledgeUpdater`, `ToolLoopEvent`, `TurnOrder`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidatePreparationEvidence`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateVerificationPorts`, `AgentCandidateWorkspaceArchiveLimits`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgenticGeneratorShotReceipt`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `BackendCallPolicy`, `CanonicalCandidateDocument`, `CaptureAgentCandidateWorkspaceOptions`, `CapturedAgentCandidateWorkspace`, `ChatTurnHooks`, `ChatTurnResult`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `CreateAgentCandidateWorkspacePortOptions`, `CreateKnowledgeImprovementActivationExecutorOptions`, `CreateProtectedAgentCandidateModelPortOptions`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `DisposePreparedAgentCandidateOptions`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `ExactProcessCandidateExecutorOptions`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `HaltContext`, `HaltSignal`, `ImproveCodeOptions`, `ImproveCodeResult`, `ImprovementCandidate`, `ImproveMethodContext`, `ImproveMethodResult`, `ImproveSkillsOptions`, `KnowledgeImprovementActivationExecutor`, `KnowledgeImprovementCandidatePair`, `KnowledgeImprovementExperimentBundles`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `OtelAttribute`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `RecoverExpiredAgentCandidateOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolveAgentBackendOptions`, `ResolvedAgentCandidateContainer`, `ResolvedChatModel`, `RunChatTurnInput`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeEventOtelOptions`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunHandle`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeTelemetryOptions`, `RunToolLoopOptions`, `SanitizedKnowledgeReadinessReport`, `StreamToolLoopOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `ToAnalystFindingsOptions`, `ToolLoopResult`, `VerifiedAgentCandidate`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ControlDecision`, `ConversationStreamEvent`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `ImproveMethodSource`, `ImproveProfileSurface`, `ImproveResult`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeStreamEvent`, `StreamToolLoopYield`, `SupervisedKnowledgeUpdater`, `ToolLoopEvent`, `TurnOrder`. ### Vertical agent — manifest + surface proposal source @@ -1155,11 +1158,10 @@ The scoring/measurement/judge substrate. **Do NOT re-implement a judge, an authe ### JUDGE — LLM-as-judge, panels, calibration -Import from `@tangle-network/agent-eval` — 32 exports. +Import from `@tangle-network/agent-eval` — 31 exports. | Symbol | Kind | Summary | |---|---|---| -| `buildAgreementJudge` | function | Build a `JudgeConfig` that scores a produced student artifact against the | | `cachedJudge` | function | Wrap a `JudgeConfig` so repeat judgments of the same artifact are served | | `calibrateJudge` | function | Measure judge quality against human gold labels: computes Cohen's κ, Pearson correlation, and MAE over matched item ids. | | `compilerJudge` | function | Build a `SandboxJudgeSpec` that scores whether the harness compiles without errors. | @@ -1270,24 +1272,20 @@ Import from `@tangle-network/agent-eval` — 51 exports. ### CAMPAIGN — profile matrix, gates, improvement loop -Import from `@tangle-network/agent-eval/campaign` — 398 exports. +Import from `@tangle-network/agent-eval/campaign` — 320 exports. | Symbol | Kind | Summary | |---|---|---| -| `aceProposer` | function | Append-only context engineering proposer: grows a skill playbook by appending generation-tagged lessons without merging or overwriting prior entries. | | `acquireSingleRunLock` | function | Acquire the lock or throw naming the live holder. A stale lock (holder pid | | `analyzeCrossSurfaceInteractions` | function | Build the complete cross-surface evidence matrix and derive all three frozen | -| `applySkillPatch` | function | Apply a SkillOpt patch to a text surface. Ops apply in array order against | | `assertCampaignDesign` | function | Reject campaign designs whose denominator cannot be identified exactly. | | `assertCampaignSplitIdentity` | function | Refuse a campaign whose retained task identities contradict its split digest. | | `assertCodeSurfaceIdentity` | function | Validate the immutable identity shape; the owning executor verifies the Git objects and patch. | -| `assertPolicyEditAuthorContextBudget` | function | Serialize once and fail before dispatch when author context exceeds its budget. | +| `assertComponentSurface` | function | _(no summary — add a TSDoc line at the declaration)_ | | `buildAnalystSurfaceDispatch` | function | Build the `dispatchWithSurface(surface, scenario, ctx)` the improvement loop | | `buildEvidenceVector` | function | The Evidence Bus. For each objective, pair candidate vs baseline by full | | `buildLoopProvenanceRecord` | function | Build the durable provenance record from a completed loop result. | -| `callbackGovernor` | function | The LLM-supervisor slot: a governor whose `decide` defers to a caller-supplied | | `campaignBreakdown` | function | Per-candidate evidence a reflective/patch proposer grounds its next proposal | -| `campaignLineageStore` | function | Store a lineage through CampaignStorage. Appends use its compare-and-append | | `campaignMeanComposite` | function | Mean composite across a campaign: per cell, the mean of its finite, | | `campaignMeasurementDigest` | function | Digest the exact campaign fields that can affect a measured comparison. | | `campaignScenarioIdentity` | function | Redacted but independently verifiable identity of one complete scenario. | @@ -1296,10 +1294,10 @@ Import from `@tangle-network/agent-eval/campaign` — 398 exports. | `canonicalDigest` | function | _(no summary — add a TSDoc line at the declaration)_ | | `classifyUngroundedLiterals` | function | Scan revised artifact text for single-quoted single-word literals (the | | `codeSurfaceIdentityMaterial` | function | Canonical, location-independent identity of a finalized code candidate. | -| `compareProposers` | function | Run a head-to-head lift benchmark across surface proposers on a shared holdout, returning per-proposer lift CIs and pairwise "who wins" verdicts. | +| `compareOptimizationMethods` | function | Compare complete optimization methods on disjoint train, selection, and final test data. | +| `componentSurfaceIdentityMaterial` | function | _(no summary — add a TSDoc line at the declaration)_ | | `composeGate` | function | Compose gates — all must `ship` for the composite to `ship`. First | -| `compositeProposer` | function | Fan the population budget across N proposers and merge their candidates into | -| `countSentenceEdits` | function | Sentence-level edit distance — count distinct add/remove ops between | +| `costFromLedgerSummary` | function | Keep the cost fields a custom optimization method must report. | | `createReferenceEquivalenceJudge` | function | Build the campaign-native expected-answer judge. | | `createRunCostLedger` | function | Open the durable spend account stored beside a logical run. | | `defaultProductionGate` | function | Opinionated production gate composing held-out significance, red-team, reward-hacking, and canary checks into a single `Gate.decide` decision. | @@ -1307,50 +1305,33 @@ Import from `@tangle-network/agent-eval/campaign` — 398 exports. | `dimensionRegressions` | function | Per-critical-dimension regression guard. For each dimension, pair the | | `discoverEvalFixtures` | function | Walk `evalsDir` and return the relative name of every fixture directory (one containing an exact-case `PROMPT.md`). | | `emitLoopProvenance` | function | Build the provenance record + OTel spans and persist them durably under the | -| `evolutionaryProposer` | function | Wrap a stateless `Mutator` (GEPA, AxGEPA, reflective-mutation) as a `SurfaceProposer` that mutates the current best surface into N candidates each generation. | -| `extractFapoAttributionSignals` | function | Scan a findings array and extract FAPO attribution signals — per-level counts and failure clusters used to decide which optimization level to escalate to next. | -| `extractH2Sections` | function | Extract H2 headings (`## Foo`) from a markdown surface. Exported for | +| `externalTextOptimizationMethod` | function | Adapt a third-party text optimizer without reimplementing its search. | | `failureModeRecallJudge` | function | Deterministic, ground-truth judge for analyst findings. Composite = | -| `fapoEscalationEntry` | function | Build a `ProposerEntry` that runs the full FAPO escalation policy (prompt → parameter → structural) as a single comparable optimizer entry. | -| `fapoProposer` | function | Build a FAPO policy proposer from level-specific candidate generators. | | `fsCampaignStorage` | function | Node-filesystem storage — the default. Lazily requires `node:fs` so the | -| `fsLineageStore` | function | Filesystem convenience over the conflict-safe CampaignStorage implementation. | -| `gepaParetoEntry` | function | GEPA with the Pareto frontier + combine-complementary-lessons. | -| `gepaProposer` | function | GEPA reflective proposer: each generation reflects on the weakest scenarios and dimensions to produce targeted prompt rewrites, optionally combining Pareto-frontier parents. | -| `gepaReflectionEntry` | function | GEPA, reflection-only (single-parent, no Pareto combine). | +| `gepaOptimizationMethod` | function | Turn an optional GEPA installation into an `OptimizationMethod`. | | `gitWorktreeAdapter` | function | Git-backed `WorktreeAdapter`: creates isolated worktrees on fresh branches, commits agent changes, and discards losers. | -| `haloProposer` | function | Wrap the real halo-engine CLI as a SurfaceProposer (prompt-tier). | | `heldOutGate` | function | Composable held-out gate: ships only when the PAIRED bootstrap CI lower bound | | `heldoutSignificance` | function | Significance of the held-out composite lift: ship only when the paired | -| `heuristicGovernor` | function | The reference deterministic policy an agent {@link Governor} can replace. | | `inMemoryCampaignStorage` | function | In-memory storage for filesystem-less runtimes. Artifacts + trace spans | | `isProposedCandidate` | function | Type guard: a proposal carrying its rationale vs a bare | | `isTransientTransportFailure` | function | True when the error text describes an infrastructure hiccup that should be | | `labelTrustRank` | function | Ordinal rank for a label-trust tier; absent ⇒ `unverified` (rank 0). | -| `lineageNodeId` | function | Deterministic node id: a hash of the node's lineage + content + proposer. | | `llmJudge` | function | Build a campaign-shaped `JudgeConfig` whose `score()` makes ONE LLM call | -| `llmPolicyEditProposer` | function | LLM-backed PolicyEdit author. It reads only the current JSON surface, | | `loadEvalFixture` | function | Load ONE fixture by name: reads `PROMPT.md` (plus `EVAL.ts`/`EVAL.tsx` and `package.json` under | | `loadEvalFixtureScenarios` | function | Load fixtures (all discovered, or just `names`) as campaign `Scenario`s tagged `eval-fixture`. | | `loopProvenanceArgsFromResult` | function | One translation from a completed improvement loop into durable evidence. | | `loopProvenanceSpans` | function | Build the loop's OTLP-ingestable spans from a provenance record. One root | | `makePlaybackDispatch` | function | Adapt a `PlaybackDriver` into a `runProfileMatrix` dispatch. The artifact the | -| `memLineageStore` | function | In-memory store (default; for tests and ephemeral runs). | -| `memoryCurationProposer` | function | Build the CURATOR proposer. | | `neutralizationGate` | function | Composable placebo gate: ships only when the candidate's held-out lift is NOT | | `neutralizeText` | function | Blank every non-whitespace character to a 1-byte filler while preserving all | | `openAutoPr` | function | Open a GitHub PR for a gate-approved surface promotion, attaching the manifest hash, gate verdict, and diff as the PR body. | | `openSearchLedger` | function | Open a durable filesystem search ledger. Construction performs no I/O; the | +| `optimizationTokenUsageFromSummary` | function | Preserve every optimizer token class while keeping total input and output explicit. | | `pairHoldout` | function | Pair candidate vs baseline holdout observations by FULL cellId. `select` | -| `parameterSweepProposer` | function | Config/parameter-level proposer for FAPO's middle escalation level. | | `paretoSignificanceGate` | function | Wrap the bus + a policy as a `Gate`. Plugs into the existing | -| `parseSkillPatchResponse` | function | Parse a SkillOpt LLM response into validated `SkillPatch` objects, throwing `SkillPatchParseError` on malformed JSON and silently dropping ops that violate the edit budget. | -| `patchEditCount` | function | Total ops in a patch — the edit-budget axis (SkillOpt's "textual learning | | `planCampaignRun` | function | Plan a campaign WITHOUT dispatching: computes the manifest hash and the per-cell | | `planEvalFixtureRun` | function | Dry-run planner for a fixture campaign: loads the scenarios, delegates to `planCampaignRun`, | -| `policyEditProposer` | function | `SurfaceProposer` that admission-checks typed analyst `PolicyEdit`s and applies each | | `powerPreflight` | function | Estimate the minimum detectable lift a paired-holdout improvement run can | -| `projectPolicyEditHistory` | function | Projects scored history into the only fields a policy author may consume. | | `provenanceRecordPath` | function | Canonical durable paths under the run dir. | | `provenanceSpansPath` | function | Canonical path for the durable OTLP spans JSONL file under a loop run directory. | | `renderScoreboardMarkdown` | function | Render the scoreboard as a launch-readiness Markdown document — the literal | @@ -1361,45 +1342,32 @@ Import from `@tangle-network/agent-eval/campaign` — 398 exports. | `runCampaign` | function | Core campaign orchestrator: fan scenarios through dispatch, score with judges, aggregate bootstrap CIs, and persist reproducible `CampaignResult` records. | | `runEval` | function | Simplest evaluation preset: run scenarios through dispatch, score with judges, and return a `CampaignResult` — no optimizer, no gate, no PR. | | `runImprovementLoop` | function | Gated-promotion shell over `runOptimization`: scores the winner against the baseline on a holdout set, runs the release gate, and optionally opens a PR. | -| `runLineage` | function | Drive a multi-track improvement DAG under an agent-managed governor. Seeds each | -| `runLineageLoop` | function | Wire the {@link runLineage} DAG's `step`/`merge` seams to a real | | `runOptimization` | function | Improvement loop body: N generations of propose → campaign → rank, maintaining a Pareto frontier and one global incumbent across generations. | | `runProfileMatrix` | function | Profile × scenario matrix runner: fan N agent profiles across M scenarios, project each cell to a validated `RunRecord` with real token usage, and enforce the backend-integrity guard before returning. | -| `runSkillOpt` | function | SkillOpt sequential hill-climb: each epoch reflects on train-scenario weaknesses, proposes bounded patches, accepts the first patch that strictly improves the held-out composite, and anneals the edit | | `scoreboardSummary` | function | Roll the per-requirement rows up into the launch headline counts. | | `scoreDiscrimination` | function | Rank scenarios by how well they DISCRIMINATE candidates. | | `scoreUserStory` | function | Score one story's produced state against its requirements. Thin wrapper over | | `selectDiscriminative` | function | Select the top-`k` most discriminative scenario ids for a holdout, EXCLUDING | -| `selectPolicyEditAuthorRows` | function | Select a bounded, deterministic evidence slice for a PolicyEdit author. | | `sequentialDecide` | function | `SurfaceProposer.decide` adapter — stops the optimization loop the moment | | `sequentialPairedGate` | function | Anytime-valid sequential paired gate. Conforms to the existing `Gate` | -| `skillOptEntry` | function | SkillOpt patch-mode hill-climb. Runs findings-BLIND: `runSkillOpt` owns its | -| `skillOptProposer` | function | SkillOpt proposer: proposes bounded, anchored patch operations (add/delete/replace) on a skill document, conforming to both the patch-native `SkillOptProposer` and the generic `SurfaceProposer` interf | +| `skillOptOptimizationMethod` | function | Run Microsoft's SkillOpt trainer as a complete optimization method. | | `surfaceContentHash` | function | Full SHA-256 content identity for a prompt or finalized code surface. | | `surfaceHash` | function | Short loop key derived from the same content identity as provenance. | | `tangleTracesRoot` | function | The shared, out-of-repo root for campaign/benchmark run bundles. Keeping run | -| `traceAnalystProposer` | function | Wrap agent-eval's trace-analyst registry as a SurfaceProposer (prompt-tier). | | `userStoryScoreboard` | function | Flatten story verdicts into the per-requirement scoreboard — the literal | -| `validatePolicyEditCandidateRecord` | function | _(no summary — add a TSDoc line at the declaration)_ | | `validateSearchLedgerEvent` | function | Validate and return a canonical copy. Arrays whose order is not semantic are | | `verifyCodeSurface` | function | Verify a finalized code surface against its current checkout. This rejects | | `verifyLoopProvenanceRecord` | function | Recompute and validate the self-addressed durable record. | -| `DEFAULT_POLICY_EDIT_HISTORY_LIMITS` | const | _(no summary — add a TSDoc line at the declaration)_ | | `paretoPolicy` | const | The default strategy: symmetric multi-objective Pareto significance. Ship iff | -| `POLICY_EDIT_CANDIDATE_RECORD_SCHEMA` | const | _(no summary — add a TSDoc line at the declaration)_ | | `SEARCH_LEDGER_SCHEMA` | const | Durable append-only audit log for improvement searches. | | `FileSearchLedger` | class | _(no summary — add a TSDoc line at the declaration)_ | | `FsLabeledScenarioStore` | class | Filesystem `LabeledScenarioStore`: appends one JSONL file per source with provenance and | | `LabeledScenarioStoreError` | class | Typed rejection from a labeled-scenario store (bad provenance, rate limit, invalid sample args) — carries a stable string `code`. | -| `Lineage` | class | Deterministic improvement-candidate graph with mutation, merge, frontier, and persistence helpers. | -| `LineageStoreConflictError` | class | _(no summary — add a TSDoc line at the declaration)_ | | `ProfileMatrixError` | class | Thrown when the matrix is misconfigured (no profiles, a profile whose model | | `SearchLedgerConflictError` | class | _(no summary — add a TSDoc line at the declaration)_ | | `SearchLedgerError` | class | _(no summary — add a TSDoc line at the declaration)_ | | `SearchLedgerIntegrityError` | class | _(no summary — add a TSDoc line at the declaration)_ | -| `SkillPatchParseError` | class | Parse + validate the patch response. Throws `SkillPatchParseError` when the | | `WorktreeAdapterError` | class | Typed failure from a `WorktreeAdapter` operation (create/finalize/discard) — wraps the underlying git error as `cause`. | -| `AceProposerOptions` | interface | `aceProposer` — Agentic Context Engineering: an APPEND-MOSTLY curator, the | | `AnalystArtifact` | interface | The analyst's output for one scenario — the artifact the judge scores. | | `AnalystScenario` | interface | A labeled trace scenario: a FIXED trace corpus plus the failure modes a | | `CampaignArtifactWriter` | interface | Scoped artifact writer — `write(path, content)` lands under | @@ -1408,7 +1376,8 @@ Import from `@tangle-network/agent-eval/campaign` — 398 exports. | `CampaignStorage` | interface | `CampaignStorage` — the filesystem seam `runCampaign` writes through | | `CampaignTraceWriter` | interface | Scoped trace writer handed to each dispatch — every span | | `CodeSurface` | interface | A tier-4 code surface — a finalized candidate change to the agent's | -| `CompositeProposerOptions` | interface | `compositeProposer` — run N proposers TOGETHER on the same surface. | +| `ComparisonCost` | interface | Cost reported by a method or by final test scoring. | +| `ComponentSurface` | interface | Named text components optimized together as one candidate. | | `CrossSurfaceCandidate` | interface | Immutable identity for a single candidate or a materialized composition. | | `CrossSurfaceComponent` | interface | One independently proposed change on one caller-defined surface. | | `CrossSurfaceComponentEvidence` | interface | Per-component trace evidence captured during one task attempt. | @@ -1417,44 +1386,34 @@ Import from `@tangle-network/agent-eval/campaign` — 398 exports. | `CrossSurfaceTaskRow` | interface | Canonical per-task input row. Consumers may extend this interface with | | `DefaultProductionGateOptions` | interface | `defaultProductionGate` — composes the substrate's existing safety | | `DispatchContext` | interface | Context handed to every dispatch invocation. Scoped — every | -| `EvolutionaryProposerOptions` | interface | `evolutionaryProposer` — adapts a stateless `Mutator` (population mutation: | -| `FapoEntryConfig` | interface | FAPO reviewed-escalation policy. This is an orchestration layer over | +| `ExternalTextOptimizationMethodConfig` | interface | Configuration for adapting another text optimizer. | | `FsLabeledScenarioStoreOptions` | interface | Filesystem `LabeledScenarioStore` adapter. The default capture sink for | | `Gate` | interface | Composable promotion gate. | | `GenerationCandidate` | interface | One scored candidate surface in a generation. `dimensions` + `scenarios` | -| `GepaProposerConstraints` | interface | `gepaProposer` — a reflective `SurfaceProposer` for prompt-tier surfaces. | -| `HaloProposerOptions` | interface | `haloProposer` — wraps the REAL halo-engine (Inference.net's hierarchical | +| `GepaEngineOptions` | interface | Shared settings for one bounded GEPA engine invocation. | +| `GepaEngineRun` | interface | One independently budgeted GEPA engine invocation. | | `JudgeConfig` | interface | Pluggable dimensional scorer. `score` is the contract: | | `JudgeScore` | interface | The canonical judge verdict shape — one declaration, shared by campaign | | `LabeledScenarioWrite` | interface | Required-provenance write. The store rejects writes that | -| `LineageNode` | interface | Lineage DAG — a git-graph of improvement candidates. | | `LoopProvenanceCandidate` | interface | Loop provenance — the durable, queryable record of WHAT a self-improvement | | `LoopProvenanceRecord` | interface | The durable provenance record. Aligns to the hosted `EvalRunEvent` path but | -| `MemoryCurationProposerOptions` | interface | `memoryCurationProposer` — a CURATOR `SurfaceProposer`, the complement to the | -| `Mutator` | interface | Stateless surface mutation — given findings + current | +| `OpenAICompatibleOptimizerModel` | interface | One metered OpenAI-compatible model connection shared by official optimizers. | | `OpenAutoPrOptions` | interface | `openAutoPr` — thin shell-out helper for the `runImprovementLoop` preset's | -| `OptimizerEntryConfig` | interface | Shared corpus + transport for the three built-in optimizer entries. | +| `OptimizationMethod` | interface | A complete optimization method, including candidate generation and selection. | +| `OptimizationMethodInput` | interface | Shared inputs for one optimization method. Final test data is absent. | | `PairedHoldout` | interface | Statistical held-out promotion machinery — the trustworthy core the | | `ParetoParent` | interface | A non-dominated parent on the GEPA Pareto frontier — a | | `PlaybackContext` | interface | Dispatch context plus the profile under test (which cheap model, etc.). | | `PlaybackDriver` | interface | Drives the real product through a story and returns the runtime event stream | | `PlaybackStep` | interface | One step of a user story — what the user does. The driver interprets | -| `PolicyEditAuthorScenarioRow` | interface | One measured scenario row eligible for PolicyEdit author context. | -| `PolicyEditCandidateRecord` | interface | JSON-safe attribution carried with a measured candidate and its scores. | -| `PolicyEditFindingInput` | interface | Trace-derived findings must name the measured profile that produced them. | -| `PolicyEditProposerOptions` | interface | `policyEditProposer` turns typed analyst policy edits into measured candidate | | `PowerPreflightOptions` | interface | Power preflight — "can this budget detect the effect you are hunting?" | -| `PremeasuredOptimizationBaseline` | interface | `runOptimization` — the improvement loop body. Runs N generations: the | +| `PremeasuredOptimizationBaseline` | interface | `runOptimization` runs a caller-owned candidate generator for a bounded | | `ProposalTrackContext` | interface | The lineage track that requested a proposal. | | `ProposeContext` | interface | Everything a proposer may read to plan the next | | `ProposedCandidate` | interface | A proposer output carrying the surface AND the WHY behind | -| `ProposerEntry` | interface | What an optimizer produced: the surface it promoted + what it cost to get | -| `RejectedEdit` | interface | A patch that was tried and not accepted — fed back to the model so it does | | `RolloutCall` | interface | One tool/action call observed in a rollout: a name plus its arguments. | | `RunCampaignOptions` | interface | `runCampaign` — Pass A substrate primitive. ONE function that orchestrates | | `RunEvalOptions` | interface | `runEval` — the simplest preset over `runCampaign`. No optimizer, no | -| `RunLineageLoopSeed` | interface | A seed track: the initial surface + track identity. Unlike | -| `RunSkillOptOptions` | interface | `runSkillOpt` — the SkillOpt epoch hill-climb (Microsoft, arXiv:2605.23904). | | `Scenario` | interface | Stable identifier + kind tag for any scenario. Consumers | | `ScenarioSignal` | interface | Per-scenario observation: the composite scores each candidate earned on it. | | `ScoreboardRow` | interface | One row of the launch scoreboard — story × requirement → PASS/FAIL. | @@ -1466,10 +1425,7 @@ Import from `@tangle-network/agent-eval/campaign` — 398 exports. | `SearchSurfaceEvidence` | interface | Per-attempt proof that a declared candidate surface was or was not active, | | `SessionScript` | interface | One session within a multi-session journey. Dispatch is | | `SingleRunLockOptions` | interface | Single-run lock for evaluations that share one mutable environment. | -| `SkillOptEvidence` | interface | Evidence the optimizer reflects on: where the current surface is weakest. | -| `SkillPatch` | interface | A named, attributable bundle of ops the optimizer proposes as one edit. | | `SurfaceProposer` | interface | A surface-improvement strategy. Given the current best | -| `SurfaceScore` | interface | The measured fitness of one surface — the value recorded on a DAG node. | | `TransientFailureOptions` | interface | Transient-transport-failure classification for dispatch retry policies. | | `UserStory` | interface | A user story = a runnable product journey plus the requirements that define | | `UserStoryVerdict` | interface | A scored user story — the completion verdict plus its human title. | @@ -1478,23 +1434,23 @@ Import from `@tangle-network/agent-eval/campaign` — 398 exports. | `CostLedgerHandle` | type | Public callback surface for a shared cost ledger. | | `CrossSurfaceAttemptCompleteness` | type | Whether one candidate attempt produced a usable executable outcome. | | `DispatchFn` | type | One function: scenario + ctx → artifact. Dispatcher chooses | -| `FapoOptimizationLevel` | type | FAPO (Fully Autonomous Prompt Optimization) is an orchestration policy, not | | `GateDecision` | type | Five-valued verdict taxonomy (MOSS-paper alignment). | +| `GepaAdaptiveEngineRun` | type | An engine in an adaptive run. All engines share the recipe evaluation limit. | +| `GepaOptimizationRecipe` | type | A direct mapping to a GEPA optimization recipe. | +| `GepaRunnerCommand` | type | The command that runs the Python GEPA bridge. | | `LabeledScenarioSource` | type | Source tag — required on every store write. Used by the | | `LabelTrust` | type | How much a label can be trusted to evaluate against — the gold-admission | -| `LineageNodeInput` | type | Input to {@link Lineage.addNode}: everything but the derived `id`/`seq` and the | | `LlmJudgeDimension` | type | A rubric dimension as a bare key or the full `{ key, description }` shape. A | | `MutableSurface` | type | The mutable surface a proposer changes. Tiers (see | | `ObjectiveSource` | type | Where an objective's per-cell scalar comes from. `composite` reads the | +| `OptimizationMethodRunOptions` | type | Per-method campaign settings. Each method receives its own spend account. | | `OptimizationProposer` | type | Optional vocabulary alias. The loop is the optimizer; this object is the | | `ProfileDispatchFn` | type | Dispatch for one cell: render `profile` against `scenario`, returning the | | `PromotionPolicy` | type | A promotion strategy: a pure function from the evidence vector to a verdict. | -| `RunImprovementLoopOptions` | type | `runImprovementLoop` — the gated-promotion shell around the improvement | +| `RunImprovementLoopOptions` | type | Run a caller-owned candidate generator, compare its winner with the starting | | `SequentialDecision` | type | Anytime-valid sequential promotion gate — an e-process (betting | -| `SkillPatchOp` | type | A single bounded edit against a skill surface. | -| `TraceAnalystPriorFindings` | type | `traceAnalystProposer` — wraps agent-eval's OWN trace-analyst engine | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AcceptedEdit`, `AnalyzeCrossSurfaceInteractionsInput`, `ApplySkillPatchResult`, `AxisEvidence`, `BuildAnalystSurfaceDispatchOptions`, `BuildEvidenceVectorOptions`, `BuildLoopProvenanceArgs`, `CampaignAggregates`, `CampaignBreakdown`, `CampaignCellResult`, `CampaignResult`, `CampaignRunPlan`, `CampaignRunPlanCell`, `CodeSurfaceVerification`, `CompareProposersOptions`, `CrossSurfaceAdditionDecision`, `CrossSurfaceBestSingleSelection`, `CrossSurfaceBootstrapPolicy`, `CrossSurfaceCandidateComparison`, `CrossSurfaceCandidateEvidence`, `CrossSurfaceCandidateOutcome`, `CrossSurfaceCandidateSummary`, `CrossSurfaceCompositionStep`, `CrossSurfaceDistribution`, `CrossSurfaceEligibility`, `CrossSurfaceEvidenceBreakdown`, `CrossSurfaceInteractionAwareSelection`, `CrossSurfaceInteractionEffect`, `CrossSurfaceInteractionReport`, `CrossSurfaceInteractionTask`, `CrossSurfaceNaiveStackSelection`, `CrossSurfacePairCompatibility`, `CrossSurfacePairEvidence`, `CrossSurfacePairwiseEntry`, `CrossSurfaceRankedSingle`, `CrossSurfaceRelativeCost`, `CrossSurfaceSelections`, `DimensionRegression`, `DiscriminationScore`, `EmitLoopProvenanceArgs`, `EmitLoopProvenanceResult`, `EvalFixture`, `EvalFixtureFile`, `EvalFixtureLoadOptions`, `EvalFixtureScenario`, `EvidenceVector`, `FailureModeRecallJudgeOptions`, `FapoAttributionSignals`, `FapoFailureCluster`, `FapoProposerOptions`, `FapoReviewInput`, `FapoReviewIssue`, `FapoReviewResult`, `FapoScopeContract`, `GateContext`, `GateResult`, `GenerationRecord`, `GepaProposerOptions`, `GitWorktreeAdapterOptions`, `Governor`, `GovernorContext`, `HeldOutGateOptions`, `HeldoutSignificance`, `HeldoutSignificanceOptions`, `HeuristicGovernorOptions`, `JudgeAggregate`, `JudgeDimension`, `LabeledScenarioRecord`, `LabeledScenarioSampleArgs`, `LabeledScenarioStore`, `LineageEdge`, `LineageGraph`, `LineageStore`, `LlmJudgeOptions`, `LlmPolicyEditProposerOptions`, `LoadEvalFixtureScenariosOptions`, `LoopProvenanceArgsFromResult`, `LoopProvenanceBackend`, `LoopProvenanceEvidence`, `NeutralizationGateOptions`, `OpenAutoPrResult`, `OpenSearchLedgerOptions`, `OptimizerConfig`, `ParameterCandidate`, `ParameterChange`, `ParameterSweepProposerOptions`, `ParetoSignificanceGateOptions`, `PendingCostCallView`, `PlanCampaignRunOptions`, `PlanEvalFixtureRunOptions`, `PolicyEditCandidateSummary`, `PolicyEditHistoryCandidateContext`, `PolicyEditHistoryGenerationContext`, `PolicyEditHistoryProjectionOptions`, `PolicyEditObjective`, `PolicyEditOutcomeContext`, `PowerPreflight`, `ProfileSummary`, `PromotionObjective`, `ProposePatchesArgs`, `ProposerComparison`, `ProposerPairwise`, `ProposerScore`, `ReferenceEquivalenceJudgeOptions`, `ReferenceEquivalenceScenario`, `RolloutArgumentDiff`, `RolloutArgumentDiffOptions`, `RunImprovementLoopResult`, `RunLineageLoopOptions`, `RunLineageLoopResult`, `RunLineageOptions`, `RunLineageResult`, `RunLineageSeed`, `RunLineageStepResult`, `RunOptimizationResult`, `RunProfileMatrixOptions`, `RunProfileMatrixResult`, `RunSkillOptResult`, `ScenarioAggregate`, `ScenarioRollup`, `ScoreboardRenderOptions`, `SearchAttemptAccounting`, `SearchCandidateDecidedEvent`, `SearchCandidateLineage`, `SearchCandidateRegisteredEvent`, `SearchCandidateSlot`, `SearchCandidateSlotClosedEvent`, `SearchCandidateSurface`, `SearchCompletedEvent`, `SearchFailureReason`, `SearchLedger`, `SearchLedgerAppendResult`, `SearchLedgerEntry`, `SearchLedgerReplay`, `SearchModelIdentity`, `SearchOperationRecordedEvent`, `SearchPlan`, `SearchPlannedEvent`, `SearchPlannedOperation`, `SearchPlannedTask`, `SearchTaskAttemptedEvent`, `SelectPolicyEditAuthorRowsOptions`, `SequentialDecideFn`, `SequentialDecideOptions`, `SequentialObservation`, `SequentialPairedGate`, `SequentialPairedGateOptions`, `SerializedJsonBudget`, `SingleRunLock`, `SkillOptEpochRecord`, `SkillOptProposer`, `SkillOptProposerOptions`, `SkillPatchRejection`, `TraceAnalystProposerOptions`, `TraceSpan`, `UngroundedLiteralReport`, `Worktree`, `WorktreeAdapter`, `CrossSurfaceAdditionRejectionReason`, `CrossSurfaceIneligibilityReason`, `CrossSurfacePairIncompatibilityReason`, `EvalFixtureRunPlan`, `EvalFixtureValidationMode`, `GovernorOp`, `JsonPolicyEditTargetSurface`, `JsonPrimitive`, `JsonValue`, `PolicyEditFindingSource`, `RedactionStatus`, `RunOptimizationOptions`, `SearchAccountingAudit`, `SearchCostAccounting`, `SearchLedgerEvent`, `SearchLedgerHash`, `SearchOperationKind`, `SearchSurfaceEffect`, `SearchSurfaceKind`, `SearchTaskOutcome`, `SearchTokenAccounting`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AnalyzeCrossSurfaceInteractionsInput`, `AxisEvidence`, `BuildAnalystSurfaceDispatchOptions`, `BuildEvidenceVectorOptions`, `BuildLoopProvenanceArgs`, `CampaignAggregates`, `CampaignBreakdown`, `CampaignCellResult`, `CampaignResult`, `CampaignRunPlan`, `CampaignRunPlanCell`, `CodeSurfaceVerification`, `CompareOptimizationMethodsOptions`, `CrossSurfaceAdditionDecision`, `CrossSurfaceBestSingleSelection`, `CrossSurfaceBootstrapPolicy`, `CrossSurfaceCandidateComparison`, `CrossSurfaceCandidateEvidence`, `CrossSurfaceCandidateOutcome`, `CrossSurfaceCandidateSummary`, `CrossSurfaceCompositionStep`, `CrossSurfaceDistribution`, `CrossSurfaceEligibility`, `CrossSurfaceEvidenceBreakdown`, `CrossSurfaceInteractionAwareSelection`, `CrossSurfaceInteractionEffect`, `CrossSurfaceInteractionReport`, `CrossSurfaceInteractionTask`, `CrossSurfaceNaiveStackSelection`, `CrossSurfacePairCompatibility`, `CrossSurfacePairEvidence`, `CrossSurfacePairwiseEntry`, `CrossSurfaceRankedSingle`, `CrossSurfaceRelativeCost`, `CrossSurfaceSelections`, `DimensionRegression`, `DiscriminationScore`, `EmitLoopProvenanceArgs`, `EmitLoopProvenanceResult`, `EvalFixture`, `EvalFixtureFile`, `EvalFixtureLoadOptions`, `EvalFixtureScenario`, `EvidenceVector`, `ExternalOptimizationExample`, `ExternalTextEvaluationResponse`, `ExternalTextOptimizerContext`, `ExternalTextOptimizerResult`, `FailureModeRecallJudgeOptions`, `GateContext`, `GateResult`, `GenerationRecord`, `GepaOptimizationMethodConfig`, `GitWorktreeAdapterOptions`, `HeldOutGateOptions`, `HeldoutSignificance`, `HeldoutSignificanceOptions`, `JudgeAggregate`, `JudgeDimension`, `LabeledScenarioRecord`, `LabeledScenarioSampleArgs`, `LabeledScenarioStore`, `LlmJudgeOptions`, `LoadEvalFixtureScenariosOptions`, `LoopProvenanceArgsFromResult`, `LoopProvenanceBackend`, `LoopProvenanceEvidence`, `LoopProvenanceOptimizationMethod`, `NeutralizationGateOptions`, `OpenAutoPrResult`, `OpenSearchLedgerOptions`, `OptimizationMethodComparison`, `OptimizationMethodPairwise`, `OptimizationMethodProvenance`, `OptimizationMethodResult`, `OptimizationMethodScore`, `OptimizationPackageSource`, `OptimizationTokenUsage`, `OptimizerConfig`, `ParetoSignificanceGateOptions`, `PendingCostCallView`, `PlanCampaignRunOptions`, `PlanEvalFixtureRunOptions`, `PowerPreflight`, `ProfileSummary`, `PromotionObjective`, `ReferenceEquivalenceJudgeOptions`, `ReferenceEquivalenceScenario`, `RolloutArgumentDiff`, `RolloutArgumentDiffOptions`, `RunImprovementLoopResult`, `RunOptimizationResult`, `RunProfileMatrixOptions`, `RunProfileMatrixResult`, `ScenarioAggregate`, `ScenarioRollup`, `ScoreboardRenderOptions`, `SearchAttemptAccounting`, `SearchCandidateDecidedEvent`, `SearchCandidateLineage`, `SearchCandidateRegisteredEvent`, `SearchCandidateSlot`, `SearchCandidateSlotClosedEvent`, `SearchCandidateSurface`, `SearchCompletedEvent`, `SearchFailureReason`, `SearchLedger`, `SearchLedgerAppendResult`, `SearchLedgerEntry`, `SearchLedgerReplay`, `SearchModelIdentity`, `SearchOperationRecordedEvent`, `SearchPlan`, `SearchPlannedEvent`, `SearchPlannedOperation`, `SearchPlannedTask`, `SearchTaskAttemptedEvent`, `SequentialDecideFn`, `SequentialDecideOptions`, `SequentialObservation`, `SequentialPairedGate`, `SequentialPairedGateOptions`, `SingleRunLock`, `SkillOptOptimizationMethodConfig`, `SkillOptTrainerConfig`, `TraceSpan`, `UngroundedLiteralReport`, `Worktree`, `WorktreeAdapter`, `CrossSurfaceAdditionRejectionReason`, `CrossSurfaceIneligibilityReason`, `CrossSurfacePairIncompatibilityReason`, `EvalFixtureRunPlan`, `EvalFixtureValidationMode`, `OptimizerModelBudget`, `RedactionStatus`, `RunOptimizationOptions`, `SearchAccountingAudit`, `SearchCostAccounting`, `SearchLedgerEvent`, `SearchLedgerHash`, `SearchOperationKind`, `SearchSurfaceEffect`, `SearchSurfaceKind`, `SearchTaskOutcome`, `SearchTokenAccounting`, `SkillOptRunnerCommand`. ### TOKEN / USAGE — usage extraction + run-record usage types diff --git a/docs/architecture-interpretations.md b/docs/architecture-interpretations.md index 8f5fdd9b..8bc27c0f 100644 --- a/docs/architecture-interpretations.md +++ b/docs/architecture-interpretations.md @@ -143,7 +143,7 @@ Breaks: the load-bearing assumption — a **calibrated** gap signal — is absen ### 3.4 Two-timescale / recursive self-improvement -Inner fast loop drives an answer now; outer slow loop (GEPA / `runImprovementLoop`) rewrites policy from accumulated traces + judge scores, measures the exact candidate on held-back tasks, and requires an explicit activation. The recursion is real *in shape* — the optimiser is an atom editing an atom's policy — but cross-benchmark transfer remains unproven. The frame's value is its sharp corpus-vs-policy split: **wiki growth is an input to inference; only prompt/tool/policy rewrites are RSI.** The research-acquisition loop is RSI only if findings about *which acquisition move paid off* rewrite the driver's acquisition policy and the resulting profile wins on fresh tasks. +Inner fast loop drives an answer now; outer slow loop (`improve()` with an official GEPA or SkillOpt method) rewrites policy from accumulated traces + judge scores, measures the exact candidate on final-test tasks hidden from the method, and requires an explicit activation. The recursion is real *in shape* — the optimiser is an atom editing an atom's policy — but cross-benchmark transfer remains unproven. The frame's value is its sharp corpus-vs-policy split: **wiki growth is an input to inference; only prompt/tool/policy rewrites are RSI.** The research-acquisition loop is RSI only if findings about *which acquisition move paid off* rewrite the driver's acquisition policy and the resulting profile wins on fresh tasks. ### 3.5 Skeptic / Occam (adversarial) @@ -221,7 +221,7 @@ Then run the §5 gate. If a findings-fed driver beats random@k at equal k under `authorStrategy`: the program space where the Gate-A strategies run. - `src/analyst-loop/` — `runAnalystLoop`; the trace observer feeding the canonical loop is `observe()` (`src/runtime/observe.ts`), consumed by the agent-driver. -- Prompt-space optimization lives in agent-eval (`selfImprove`); the analyst-prompt +- Prompt-space optimization lives in an agent-eval `OptimizationMethod`, invoked through Runtime's `improve()`; the analyst-prompt coordinate has shown no significant lift on held-back problems in controlled runs to date — see `.evolve/current.json` and the memory ledger for the current evidence state. - `bench/src/selector.ts` + `bench/src/corpus-replay.mts --selector` — the deployable selector and its offline replay harness. diff --git a/docs/architecture.md b/docs/architecture.md index 9b78abba..6ac7dfe8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -130,7 +130,7 @@ The same `Agent` loop runs at two timescales. | Steer output | ephemeral next-shot context | a persisted candidate surface | | Anchored by | the judge scores the answer | `heldOutGate` on a holdout set → PR | | `act → Program` is | a steer over the worker's next shot | a candidate generator (worktree) | -| Where it lives today | the agent-driver over the `Scope`/`Supervisor` (`createCoordinationTools`) + `runAgentic`/`defineStrategy`; the `runLoop` kernel is one leaf backend | `runOptimization`/`runImprovementLoop` + `propose()` (**this is built**) | +| Where it lives today | the agent-driver over the `Scope`/`Supervisor` (`createCoordinationTools`) + `runAgentic`/`defineStrategy`; the `runLoop` kernel is one leaf backend | `improve()` + a complete agent-eval `OptimizationMethod`; code candidates use Runtime-owned worktrees | Both are *"a loop whose step contains a loop"* — `driver↔worker + analyze + propose`. The recursive `Agent` makes them the **same node** at different @@ -188,8 +188,9 @@ the enforcing code, in §13.3 (`assertTraceDerivedFindings`). ## 5. GEPA at every level The optimizer `O` improves any `Agent`'s `context`+prompt and the program shape, -from the shared corpus, **held-out gated** (train ∩ holdout = ∅, enforced in -`runImprovementLoop`). This is the **outer flywheel**: the controller is learned, +from the shared corpus, **held-out gated** (train ∩ selection ∩ final test = ∅, enforced by +`compareOptimizationMethods`). Official GEPA runs through `officialGepa(...)` with an explicit +recipe and evaluation ID. This is the **outer flywheel**: the controller is learned, not hand-written. Optimize against the **multi-objective vector** (§0.5.2) — *correct, fast, secure, cheap* — Pareto, **not** a pre-collapsed scalar; each component is graded by its own deployable checker (tests · clock · scanner · cost meter), with the external diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 9313c3f2..59344e69 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -2,7 +2,7 @@ -> **Version 0.103.0.** The export inventory + per-symbol signatures live in the generated `docs/api/` reference: **`docs/api/primitive-catalog.md`** is the never-stale, grouped list of every primitive to reuse (own surface + the agent-eval judge / authenticity / verification / statistics / campaign / token-usage surfaces), with each one's import path and one-line summary read live from source; the per-module pages hold the full signatures. The pinned substrate is agent-eval `>=0.122.8 <0.124.0`; the sandbox substrate that materializes profiles into harness shapes is `@tangle-network/sandbox` (peer `>=0.11.1 <1.0.0`). The neutral contract types (`AgentProfile`, `AgentProfileMcpServer`, `HarnessType`, `ReasoningEffort`, `Part`/`ToolPart`/`ToolState`, plus environment-provider types) are owned by **`@tangle-network/agent-interface`** (peer `>=0.32.0 <0.33.0`) — the single source of truth. Substrate primitives are re-exported through `@tangle-network/agent-eval/contract` (or `/campaign`), not local to this package — the catalog's §2 shows exactly which subpath each lives under. +> **Version 0.103.0.** The export inventory + per-symbol signatures live in the generated `docs/api/` reference: **`docs/api/primitive-catalog.md`** is the never-stale, grouped list of every primitive to reuse (own surface + the agent-eval judge / authenticity / verification / statistics / campaign / token-usage surfaces), with each one's import path and one-line summary read live from source; the per-module pages hold the full signatures. The pinned substrate is agent-eval `>=0.124.0 <0.125.0`; the sandbox substrate that materializes profiles into harness shapes is `@tangle-network/sandbox` (peer `>=0.11.1 <1.0.0`). The neutral contract types (`AgentProfile`, `AgentProfileMcpServer`, `HarnessType`, `ReasoningEffort`, `Part`/`ToolPart`/`ToolState`, plus environment-provider types) are owned by **`@tangle-network/agent-interface`** (peer `>=0.32.0 <0.33.0`) — the single source of truth. Substrate primitives are re-exported through `@tangle-network/agent-eval/contract` (or `/campaign`), not local to this package — the catalog's §2 shows exactly which subpath each lives under. > > **`./loops` is the runtime barrel** — `package.json` maps it to `src/runtime/index.ts`. Everything below labelled `/loops` is the recursive-atom + loop-kernel surface. > @@ -22,7 +22,7 @@ The system is four steps, each with a named entry point: 1. **Describe the agent as data.** A **profile** is the whole agent: `systemPrompt + skills + tools + mcp + knowledge + memory + rag`, one combined surface. 2. **Run it.** A driver steers workers over rounds — `runPersonified` composes a combinator (`loopUntil`, `fanout`, …) over the `Supervisor`, spending K rounds against one persistent, journaled artifact from a *conserved budget pool*, so any two topologies you compare cost the same by construction. 3. **Score it on a benchmark.** Either the `ADAPTERS` registry driven by `runGate` over the Supervisor, or an `AgenticSurface` driven by `runBenchmark`/`runAgentic`. -4. **Improve it under a gate.** `selfImprove`/`runImprovementLoop` (with `gepaProposer`) or the multi-generation `runStrategyEvolution` mutate the profile, and a gate (`defaultProductionGate`/`heldOutGate`/`promotionGate`) certifies a win **on a frozen holdout** — never on the tasks the optimizer trained against. +4. **Improve it on three partitions.** `improve(profile, { method, trainScenarios, selectionScenarios, testScenarios })` runs a complete agent-eval `OptimizationMethod`. The method receives train and selection cases only. Runtime scores its selected candidate on the untouched final test and returns `ship` only when the paired interval clears the required lift. Two standing rules: the model that picks the best attempt is never the model that grades it, and observation attaches to the *loop* via `RuntimeHooks`, never to the portable profile. One known limit: the current `Supervisor` records completed settlements but does not resume a live tree after coordinator restart. @@ -94,10 +94,11 @@ A general "loop" primitive is the single most common modelling error in this rep | Pick the **execution transport a driven loop runs on** (`sandbox` box / cli-bridge / router) from a product flag | `resolveSandboxClient({ backend })` — `/loops` | a per-product `if (backend === 'router') …` branch re-wiring `createExecutor` + `inlineSandboxClient` | | Pick the **chat backend an in-process turn runs on** (`router`/`tcloud`/`cli-bridge`/`sandbox`) from a product flag | `resolveAgentBackend({ backend })` — root `.` | the copy-pasted `backend-name → createOpenAICompatibleBackend` branch every eval product hand-rolled (the copies drift) | | Pick / register a leaf backend, or bring your own agent | `createExecutor({ backend })` / `createExecutorRegistry()` / implement `Executor` — `/loops` | a per-vendor adapter or closed `inline\|sandbox\|cli` switch (won't report through the `UsageEvent` channel) | -| Evolve a **prompt/string** surface | `gepaProposer({ llm, model, target })` (default inside `selfImprove`; the skill-surface twin is `skillOptProposer`, same source) — `agent-eval/campaign` | a hand-rolled prompt-mutation reflection loop with its own Pareto bookkeeping | -| Self-improve an agent (one pluggable verb) — START HERE (self-improvement) | `improve(profile, findings, { surface, gate })` — root `.`; prompt, skill-document, and curated-memory surfaces have built-in generators, tools/MCP/hooks/subagents/whole-profile take an explicit generator, and code gets isolated incumbent/candidate worktrees. Workflow and rollout-policy files are code/config changes; use the code surface plus agent-eval's `parameterSweepProposer` for JSON sweeps. | a bespoke optimize loop, calling `selfImprove`/a skill or memory optimizer directly for the common case, storing orchestration in opaque profile extensions, or comparing code against an empty/string stand-in | -| Run the self-improvement loop with full substrate control | `selfImprove({ agent, scenarios, judge, baselineSurface })` — `agent-eval/contract` | a bespoke optimize loop or a parallel skill-optimizer | -| Run the gated loop with full control | `runImprovementLoop({ baselineSurface, dispatchWithSurface, driver, holdoutScenarios, gate })` — `agent-eval/contract` | your own propose→campaign→rank→re-score-on-holdout→gate→PR loop | +| Optimize text or named components with upstream GEPA | `officialGepa({ evaluationId, recipe, ... })`, passed as `improve(...).method` from root `.` | a local GEPA approximation, prompt mutation loop, or silent fallback when Python is unavailable | +| Optimize one text surface with Microsoft SkillOpt | `officialSkillOpt({ evaluationId, trainer, optimizer, ... })`, passed as `improve(...).method` from root `.` | Runtime-owned SkillOpt search or a silent local fallback | +| Improve one profile coordinate | `improve(profile, { surface, method, trainScenarios, selectionScenarios, testScenarios, judges, agent })` from root `.` | an implicit per-surface optimizer, a method that sees final-test cases, or a profile mutation during search | +| Compare complete optimization methods directly | `compareOptimizationMethods(...)` from `agent-eval/campaign` | comparing one method's training score to another method's final score | +| Improve repository code | `improve(profile, { surface: 'code', code, scenarios, judge, agent, budget })` from root `.` | passing code through a text optimizer or managing candidate worktrees in product code | | Decide ship/hold on a candidate (campaign context) | `defaultProductionGate({ holdoutScenarios, deltaThreshold })`; compose with `heldOutGate` / `composeGate` — `agent-eval/contract` | a raw `h1>h0` point comparison on the training set | | Decide ship/hold from a **`BenchmarkReport`** (per-task cells) | `promotionGate({ report, incumbent, candidate })` — `/loops` | comparing two strategies' mean scores directly; re-deriving the bootstrap | | Run the full multi-generation strategy flywheel + certify | `runStrategyEvolution(config)` — `/loops` | a bespoke gen0→author→gen1→holdout loop with hand-rolled champion selection | diff --git a/docs/design.md b/docs/design.md index 9b7f84b5..65ac38d6 100644 --- a/docs/design.md +++ b/docs/design.md @@ -24,7 +24,7 @@ Competing topologies draw from one shared compute budget, so "smarter coordinati The internal one-sentence statement of the whole system, preserved from the API reference (each bolded term is defined there in plain words): -> A **genome** (an `AgentProfile`: `systemPrompt + skills + tools + mcp + knowledge + memory + rag` — one combined surface) is run as a **driver⟷worker conversation** (`runPersonified` composing a combinator like `loopUntil`/`fanout` over the keystone `Supervisor` — K rounds spent against one persistent, journaled artifact on a *conserved budget pool* so equal-compute holds by construction) over a **benchmark** (the `ADAPTERS` registry driven by `runGate` over the Supervisor, or an `AgenticSurface` driven by `runBenchmark`/`runAgentic`), then **optimized by a gated loop** (`selfImprove`/`runImprovementLoop` + `gepaProposer`, certified by `defaultProductionGate`/`heldOutGate`/`promotionGate`, or the full multi-generation `runStrategyEvolution`) that evolves the genome and **certifies wins on a frozen holdout** — never on the training composite. +> An `AgentProfile` defines the agent. Runtime executes it. Agent-eval measures it and runs a complete optimization method on train and selection cases. Runtime then compares the selected candidate with the baseline on an untouched final test. Repository code follows the same rule but uses isolated worktrees. ## The internal design and research docs diff --git a/docs/design/structural-rollout-integration.md b/docs/design/structural-rollout-integration.md index 52fc6946..4efa0ee8 100644 --- a/docs/design/structural-rollout-integration.md +++ b/docs/design/structural-rollout-integration.md @@ -35,7 +35,7 @@ New module `src/runtime/structural-rollout.ts`: - `structuralRollout({ policy, checkSource, checkRunner }) → Strategy` — a fourth member of the sample/refine family via `defineStrategy`; argmax by weighted visible score, ≤`repairRounds` repair shots steered by `failureOutput`, keep-best-by-score. Emits `SelectionReceipt`s. - `StructuralRolloutPolicy { k, repairRounds, testgen, diverse?, temperature? }` — promoted from the rig env vars; later an optimizable surface for `improve()`. -Placement rule: this is an INFERENCE-TIME capability (wraps the model call). It does not go into `improve()`/`selfImprove` (training-time); `improve()` may later tune the policy knobs. +Placement rule: this is an INFERENCE-TIME capability (wraps the model call). It does not run inside optimization-time `improve()`; a complete method may later tune the policy knobs through that API. Extend-don't-fork list: strategy family, verifier-environment, selectWinner/receipts, agent-eval field routing + judges, the rigs' `generateTests`/`extractRepairCode` as default impls. diff --git a/docs/intelligence-sdk.md b/docs/intelligence-sdk.md index fcc43b3d..15f80fe5 100644 --- a/docs/intelligence-sdk.md +++ b/docs/intelligence-sdk.md @@ -189,7 +189,7 @@ The SDK is a thin layer over shipped primitives: | production chat envelope | `handleChatTurn` | | manifest and mutable surfaces | `defineAgent` | | trace-to-finding loop | `runAnalystLoop` | -| code/tool/MCP candidate generation | `improvementDriver`, `agenticGenerator`, verifiers | +| code/tool/MCP candidate generation | `improve({ surface: 'code' })`, `CandidateGenerator`, `agenticGenerator`, verifiers | | loop execution | `runLoop` (kernel), `runAgentic` / `defineStrategy` (Supervisor), `createCoordinationTools` (agent-driver) | | promotion | `promotionGate`, held-out gates in `@tangle-network/agent-eval` | @@ -226,6 +226,6 @@ defineImprovementSurface({ /* declarative mutable-surface registration */ }) What exists underneath each verb today: - **flushRecommendations** — the substrate is `client.recordTrace(...)` → trace analysts → `improve()`, proven offline in `examples/intelligence-recommend`. Missing: the hosted verb that runs it as a service. -- **runImprovementCycle** — the readiness half ships as `IntelligenceClient.doctor()` (checks + surfaces + repo); the loop half ships as `improve()`, `improvementDriver`, `agenticGenerator`, and `promotionGate`. Missing: the client verb plus the repo plumbing (isolated branch/worktree, check runner, PR open). +- **runImprovementCycle** — the readiness half ships as `IntelligenceClient.doctor()` (checks + surfaces + repo); the loop half ships as `improve()`, complete methods for profile fields, Runtime's code worktree path, and `promotionGate`. Missing: the client verb plus the repo plumbing (isolated branch/worktree, check runner, PR open). - **configureLoop** — the loop grammar ships as `runLoop`, `runAgentic` / `defineStrategy`, and the held-out gates in `@tangle-network/agent-eval`. Missing: the config-object verb that maps a declarative gate/matrix onto them. - **defineImprovementSurface** — mutable surfaces are declared today as the `surfaces` field of `IntelligenceConfig` (a readiness input only). Missing: a first-class registration a PR mode consumes. diff --git a/docs/learning-flywheel.md b/docs/learning-flywheel.md index e7d93546..16022878 100644 --- a/docs/learning-flywheel.md +++ b/docs/learning-flywheel.md @@ -228,8 +228,8 @@ steer-detector and `J` measure a correlated property, optimizing the observable reported contrast is **Benjamini-Hochberg corrected within its family** (`corpus-report.mts`), and a result counts only if it clears the family FDR — never on its own CI. Separate a reusable **exploration** set (rank candidates freely, BH-corrected) from a **frozen confirmation holdout** - spent once per *locked* candidate; this is what `runImprovementLoop` enforces by refusing - train ∩ holdout overlap (memorization read as generalization is the default failure otherwise). + spent once per *locked* candidate; this is what `compareOptimizationMethods` enforces by keeping + the final-test partition out of the optimization method (memorization read as generalization is the default failure otherwise). - **"Validates the concept" ≠ "validates the product."** A hand-rolled refine loop proves refinement helps, NOT that `runLoop`/the controller does. Route through the real kernel. - **Eval economics is the moonshot bottleneck, not controller cleverness.** Build the offline @@ -332,5 +332,6 @@ steer-detector and `J` measure a correlated property, optimizing the observable `terminal-compare.ts`, `corpus-report.mts`). The gen0 → `authorStrategy` → gen1 → rotating-disjoint-holdout runner (the minimal single-objective Gate-B form) over `authorStrategy` (`src/runtime/strategy-author.ts`) + the seeded `promotionGate` is open work. -- Substrate optimizer/corpus primitives: `@tangle-network/agent-eval` (`selfImprove`, - `runImprovementLoop`, `heldoutSignificance`, `RunRecord`/trace-store, `./rl`). +- Substrate optimizer/corpus primitives: `@tangle-network/agent-eval` (`OptimizationMethod`, + `compareOptimizationMethods`, `gepaOptimizationMethod`, `skillOptOptimizationMethod`, + `heldoutSignificance`, `RunRecord`/trace-store, `./rl`). diff --git a/docs/research/simplification-plan.md b/docs/research/simplification-plan.md index a0275db0..80e291b3 100644 --- a/docs/research/simplification-plan.md +++ b/docs/research/simplification-plan.md @@ -1,6 +1,11 @@ -# agent-runtime simplification — MASTER TRACKER +# agent-runtime simplification plan -> **Living doc. State: SHIPPING on `feat/usability-overhaul` (clean-merges origin/main, all gates green).** Done: **WS1a+1b** (`supervisorAgent(profile, deps)` resolves the brain from `profile.harness` — `null`→in-process router tool-loop, a coding-CLI harness→a sandboxed harness driving the verbs; `DriverChat` deleted, `routerBrain` now internal; both arms proven offline), **WS3** (runtime barrel 355→277, subpaths 13→6), **WS5** (canonical-api 984→76, 26→17 docs +5 archived, CLASS 6 prose-symbol gate so docs can't lie). Next: WS2, WS4, WS6, WS7-9. The bar is what a world-class staff-eng org ships: **SIMPLE**. A competent engineer derives the call path in *seconds*. **THINK BIG (§0) · THINK SIMPLE (§1).** +> Historical record only. +> This plan predates the complete `OptimizationMethod` API and the agent-managed-compute roadmap. +> Do not use its code samples as current API guidance. +> See [canonical-api.md](../canonical-api.md) and [agent-managed-compute/roadmap.md](../agent-managed-compute/roadmap.md). + +The entries below preserve earlier decisions and rejected designs. --- @@ -15,7 +20,7 @@ - **The atom: `AgentProfile`** = `{ prompt, tools, model|harness, skills, mcp }`. Worker/driver/supervisor are **not types** — a *driver* is a profile whose tools spawn; a *supervisor* **authors its children's profiles** ("recursive profile authoring" — supervisor-lab's core primitive). Everything spawned is an AgentProfile. No `role` flags, no `{name, systemPrompt}` shims. - **FOUR honest public verbs** (red-team-corrected: four honest beat three lying; each wraps a REAL existing function — TARGET, not yet built): - `run(profile, against, scope?) → Result(against)` — the one driven loop; recurses when the worker spawns. **Counterparty axis:** `against` = a **TASK** (benchmark; oracle = `score()`) | a **DRIVER profile** (user-sim; oracle = persona DONE) | a **HUMAN** (product; oracle = the turn boundary + persisted sandbox session). **In-flight STEER lives HERE** — it's loop control (a per-shot string, `strategy.ts pendingSteer`), never a profile change. *Caveat: the return type leaks the counterparty (BenchmarkReport vs transcript vs turn) → a tagged union narrowed on `against`, never `any`.* Wraps `runAgentTask`/`runBenchmark`/`runAgentic`/`runPersonaConversation`/`handleChatTurn`. The per-worker settlement oracle `gateOnDeliverable` lives INSIDE `against` — it is NOT a verb. - - `improve(profile, findings, { generator?, surface?, gate? }) → profile'` — **ONE verb, PLUGGABLE** (not "one engine" — the optimizers are real and already built). The SHAPE is uniform (findings → candidate profile → optionally gate); the ALGORITHM is a pluggable **`CandidateGenerator`** (`improvement-driver.ts:35`, 3 fields): **GEPA** (`gepaDriver`, prompt) · **skillOpt** (`skillOptDriver`, skills) · **autoresearch** (`createResearchExecutor`) · **reflective**/**agentic** (code) · BYO. The **`surface`** is a param: `prompt | skills | tools | mcp | hooks | code` — the §1.5 whole profile, including *building* tools/mcp/hooks. `gate: 'holdout' | 'none'`. The win is ONE clean plug-in shape for all your optimizers, not erasing them. + - Current replacement: `improve(profile, { method, trainScenarios, selectionScenarios, testScenarios, judges, agent })`. Complete methods own search and selection. Runtime owns the untouched final comparison. Code alone keeps Runtime's worktree candidate path. - `certify(profile, { baseline?, holdout? }) → verdict` — POST-run statistical cert = `promotionGate` (paired bootstrap + min-tasks floor + CI-low). The real "never fakes a win." - `refuse(task) → verdict` — PRE-run static readiness = `decideKnowledgeReadiness`. SEPARATE from certify on purpose (different input/timing/question); fusing them under one `gate` name is the over-MERGE failure. - **Invariants:** `analyze` is **internal** (auto-on-settle; findings UP the bus — not a verb). Backend is **DATA**. **Durability:** turn/leaf resume = sandbox session + journal (real); a half-finished **multi-generation `improve` run is NOT resumable today** (port `loops`' disk-observable phase state — NOT a new event log). **Compute:** the `ComputeGovernor` (8/8) lives in `loops/`, **NOT this repo yet** — migrating it is a real, unstarted task. **Skills = policy.** @@ -140,7 +145,7 @@ - **WS4 — Naming taxonomy.** depth/breadth→Strategy; improvementDriver→improve; supervisorInstructions→supervisorInstructions; DriverChat deleted; `AgentRunSpec`→`SandboxIterationSpec`. Ship deprecation aliases one release + fleet sweep. **Done:** grep "Driver" = one concept. - **WS5 — Docs truth + gate.** Separate packages (runtime doc = runtime exports only); extend the gate to EVERY backticked symbol; execute §3. **Done:** a doc symbol that doesn't resolve = red build; canonical-api shrunk/deleted. - **WS6 — Examples.** §5. **Done:** offline-first + CI all. -- **WS7 — RSI is one verb, PLUGGABLE.** `improve(profile, findings, {generator?, surface?, gate?})` — one SHAPE, pluggable `CandidateGenerator` (GEPA/skillOpt/autoresearch/reflective/agentic/BYO) over a `surface` (prompt/skills/tools/mcp/hooks/code); in-flight steer moved to `run`. `certify`=`promotionGate`, `refuse`=`decideKnowledgeReadiness` kept separate. **Done:** one `improve` entrypoint exposes every existing agent-eval optimizer cleanly; products improve the WHOLE profile, not just a string. +- **WS7, superseded.** The shipped replacement accepts a complete agent-eval `OptimizationMethod` for profile fields and uses isolated Runtime worktrees for code. Official GEPA recipes and Microsoft SkillOpt are explicit method factories with no local fallback. - **WS8 — Product primitives.** `evaluateWithUser(product, persona)` (rename `runPersonaConversation` + weld the DONE oracle); the counterparty axis on `run` (tagged-union return); route the harness human-tool to the product via the bus; profile-as-worker-behind-an-Environment + route benchmark to the governed fleet. **Done:** the 4 product shapes (product/benchmark/user-sim/builder) use the same primitive. - **WS9 — Long-horizon plan-driver (§7).** Build the **milestone primitive + completion-oracle rollup** (worker `gateOnDeliverable` → milestone acceptance → plan 100%, reusing `CompletionAnalyst`); **vendor the 8 oh-my-codex skills** into supervisor-lab; **port `loops`' disk-observable phase state** for multi-generation `improve`-run resume; **ticket the `ComputeGovernor` migration** (loops→agent-runtime). **Done:** a supervisor finishes a multi-milestone parallel plan with **zero** human steers. @@ -149,10 +154,10 @@ The naming work converged on these calls. Each is settled — do not re-propose; where a rename is DEFERRED the reason is the blast radius, not indecision. 1. **`AgentRunSpec` → `SandboxIterationSpec` — name DECIDED, rename DEFERRED.** The target name is right (it is the per-iteration sandbox spec, not a generic "agent run"), but the symbol is PUBLIC at `src/runtime/index.ts` with a ~28-file blast radius across the fleet. A rename needs a major bump + consumer migration PRs (the §9 non-goal: no public rename without that), so it is OUT of a usability PR. Ship the rename in its own breaking release with deprecation aliases one cycle. -2. **`improvementDriver` KEEPS its name.** `improve()` is now the public RSI verb (the WS7 facade); `improvementDriver` is the internal code-surface driver behind it, not a public `Driver` export. The "reserve `Driver` for orchestration" goal is met because `improvementDriver` is not surfaced as an orchestration driver — it is the generator seam `improve()`/`selfImprove` consume. +2. **`improvementDriver` KEEPS its internal name.** `improve()` is the public improvement verb; `improvementDriver` is the private code-surface implementation behind it. Profile fields use complete agent-eval methods and never receive this proposer. 3. **`runLoop` KEEPS its name AND stays public.** It is a published `/loops` primitive (the round-synchronous leaf kernel the sandbox benches drive); internalizing or renaming it would break consumers for no taxonomy gain. The Scope/Supervisor core is the PREFERRED path for new recursive work, but `runLoop` is not deprecated. 4. **`runAgentic` and `runPersonified` BOTH stay.** They are distinct contracts (one-shot agentic run vs persona-driven cross-run combinator), not duplicates — a merge would conflate two real shapes. WS2's "one documented run path" is satisfied by `supervise()` as the one-call entry; the lower-level combinators remain for power use. -5. **`improve()` surface ∈ {`tools`, `mcp`, `hooks`, `code`} accepts only a BYO `generator` — fail-loud otherwise (designed boundary).** Only `prompt` (→ `gepaDriver`) and `skills` (→ `skillOptDriver`) have a zero-config default driver, because both are derivable from `opts.llm`. A code/config driver needs caller-supplied wiring (a worktree repo root, a candidate generator, a serializer) the facade cannot invent; inventing one would be exactly the silent fallback the house rules forbid, so the facade throws a `ConfigError` when `opts.generator` is absent for those surfaces. +5. **Superseded.** Every profile field now requires a complete method. Runtime supplies no zero-config prompt, skill, memory, or profile optimizer. Code requires explicit worktree configuration. --- diff --git a/examples/README.md b/examples/README.md index 33f7f027..21b532ab 100644 --- a/examples/README.md +++ b/examples/README.md @@ -81,10 +81,10 @@ repeat. A failing validator prunes a bad candidate so the loop can't keep it. | # | Example | What it shows | |---|---|---| | 16 | [`strategy-evolution/`](./strategy-evolution/) | Full policy search with a safety gate: write new tactics from past losses, promote a champion only if a statistical test says the win isn't luck. Needs `TANGLE_API_KEY`. | -| 17 | [`improve/`](./improve/) | The one self-improvement verb: `improve(profile, findings)` rewrites a prompt and returns a detached winner plus a held-out decision. Offline. | +| 17 | [`improve/`](./improve/) | Run a complete optimization method on explicit train, selection, and final-test partitions. Offline. | | 17b | [`self-improving-coder/`](./self-improving-coder/) | The flywheel on a contamination-proof coding task: an agent writes strategies from its training losses, graded by real pytest, promoted only if a fresh holdout confirms the gain. `CALIBRATE=1` is a $0 no-key check. | | 18 | [`self-improving-loop/`](./self-improving-loop/) | #17 unrolled step by step: v0 → judge → analyst → mutation → v1 → gate, showing which part owns each phase. Offline. | -| 19 | [`intelligence-recommend/`](./intelligence-recommend/) | The improvement loop offline end to end: read a run's trace → derive findings → `improve()` → a gated candidate. | +| 19 | [`intelligence-recommend/`](./intelligence-recommend/) | Read a run trace, derive findings, pass them to a complete method, and final-test its candidate. | | 20 | [`intelligence-drop-in/`](./intelligence-drop-in/) | Wrap any agent with `withIntelligence` to send one RunRecord per call — best-effort, and a proof that "off" is a zero-cost passthrough. | | 20b | [`intelligence-webcode/`](./intelligence-webcode/) | The full observability SDK (billing boundary, effort tiers, per-tool cost breakdown, OTLP export) instrumented over every cell of the WebCode benchmark. Needs a sandbox key. | | 21 | [`agents-of-all-shapes/`](./agents-of-all-shapes/) | Proof that any framework's traces converge on one open telemetry contract and produce one insight report. CI-tested. Offline. | diff --git a/examples/ablation-suite/ablation.ts b/examples/ablation-suite/ablation.ts index 46693f67..da1234ac 100644 --- a/examples/ablation-suite/ablation.ts +++ b/examples/ablation-suite/ablation.ts @@ -60,9 +60,8 @@ export interface AblationKnobs { * each settled round → a `finding` the driver pulls and composes the next prompt from. (NOT the * refine analyst-steerer — that's the degenerate inline version; this is a driver brain in the loop.) */ driverSteer?: boolean // supervise(driverProfile,{backend,analyzeOnSettle}) + steer_agent - /** GEPA-optimize the DRIVER's compose-next-prompt system prompt on TRAIN (executable-graded via the - * surface score), frozen, then run — selfImprove() with an executable JudgeConfig (NOT improve(): the - * steerer prompt is not a profile field). */ + /** Optimize the driver's standing prompt with official GEPA, explicit train, + * selection, and final-test partitions, and an executable judge. */ optimize?: 'off' | 'gepa' halo?: boolean // HALO analyst option persistentArtifact?: boolean // multi-round persistent artifact (openSandboxRun resume) @@ -188,7 +187,7 @@ export async function runAblation(opts: { driverPrompt = opt.systemPrompt gepaUsd = opt.usd // the TRAIN-side GEPA cost, counted into this arm's $ (the fair-cost invariant) console.log( - `ablation: arm "${arm.name}" GEPA driver-prompt ${opt.shipped ? 'SHIPPED' : 'kept-baseline'} (train lift ${opt.lift === undefined ? 'deferred' : `${(100 * opt.lift).toFixed(0)}pp`})`, + `ablation: arm "${arm.name}" GEPA driver-prompt ${opt.shipped ? 'SHIPPED' : 'kept-baseline'} (final-test lift ${opt.lift === undefined ? 'deferred' : `${(100 * opt.lift).toFixed(0)}pp`})`, ) } diff --git a/examples/improve/README.md b/examples/improve/README.md index 56f0816b..5823a255 100644 --- a/examples/improve/README.md +++ b/examples/improve/README.md @@ -1,9 +1,9 @@ -# An agent that rewrites its own prompt — and can't fool itself +# Improve one agent profile field -`improve()` takes an agent, a list of what went wrong, and produces a *better version of that agent* — -by rewriting one part of it (its system prompt, its tool list, its skills) and testing whether the new -version actually beats the old one. The catch that makes it trustworthy: the improved version is -only reviewable if it wins on **held-out examples it wasn't tuned on**. It never changes live state. +`improve()` runs a complete optimization method against one profile field. +The method receives train and selection cases. +Runtime keeps the final-test cases private, compares the selected candidate with the baseline, and returns a detached candidate. +It never changes the input profile. ```bash pnpm tsx examples/improve/improve.ts @@ -11,54 +11,41 @@ pnpm tsx examples/improve/improve.ts Runs offline, no credentials. -## Why it matters - -"Self-improving AI" usually means an agent that tweaks itself and declares victory. That's how you get -a change that looks great on the examples it was optimizing against and falls apart everywhere else. -`improve()` guards against exactly that: it splits your test cases, optimizes on one half, and only -accepts the change if it still wins on the other half it never saw. So improvement is *measured*, not -asserted, and a change that does not genuinely help is held. - ## What it does, step by step -1. You hand it an agent profile, a set of **findings** (a plain read of what went wrong — e.g. "the - agent under-specifies its answer format"), and which **surface** to improve (`'prompt'`, `'tools'`, - `'skills'`, ...). -2. It proposes candidate rewrites of that surface, reflecting on the findings. -3. It scores each candidate on your test scenarios. -4. It runs the winner against a **held-out** slice of scenarios that weren't used to pick it. -5. It returns the frozen winner, decision, and lift. Approval and activation are separate operations. +1. Runtime extracts the exact profile field selected by `surface`. +2. The supplied `OptimizationMethod` generates and selects a candidate using train and selection cases. +3. Runtime scores the baseline and candidate on the untouched final-test cases. +4. Runtime returns `ship` only when the paired confidence interval clears the required lift. +5. Approval and activation remain separate operations. ## What you'll see ``` -improve() proposed a detached prompt candidate and measured it on held-out scenarios … +improve() proposed a detached prompt candidate and measured it on final-test scenarios decision: ship lift: 1.000 candidate prompt: PROMOTED live prompt unchanged: BASELINE ``` -The starting prompt was `BASELINE`; the improved one is `PROMOTED`. `lift: 1.000` is how much better it -scored; `decision: ship` means the held-out check made the candidate eligible for review. +The starting prompt is `BASELINE`; the candidate is `PROMOTED`. +The final-test lift is `1.000`. ## How it stays offline -To run with no key, the moving parts are stubbed but the *machinery is real*: a scripted proposer -always suggests the winning rewrite, a deterministic judge scores it (the literal string `PROMOTED` -= 1.0, anything else = 0.0), and the "agent" echoes the candidate back while reporting token usage so -the integrity check sees a genuine run, not an empty stub. The gate, the held-out split, and the -promotion logic are the same code that runs live. +The example supplies a deterministic complete method, agent, and judge. +The method returns `PROMOTED`; the judge scores that literal string as `1`. +The partition firewall, final comparison, cost receipts, and confidence interval are production code. ## Going live -Drop the scripted proposer and pass `llm: { apiKey, baseUrl, model }` — `improve()` then uses a real -model to reflect on the findings and propose rewrites. Keep the default held-out gate (don't set -`gate: 'none'`) so real evidence decides which candidates are eligible for review. +Replace `scriptedWinner` with `officialGepa(...)`, `officialSkillOpt(...)`, or another complete method from `@tangle-network/agent-eval`. +The root README documents the optional Python installation for official GEPA. ## Files | file | what it is | |---|---| -| `improve.ts` | the profile, the findings, the offline judge/proposer/agent, and the run | +| `improve.ts` | The profile, complete method, three partitions, agent, judge, and result | -Same path is exercised by `tests/improve.test.ts` (part of `pnpm test`). +The same path is covered by `src/improvement/improve.test.ts`. diff --git a/examples/improve/improve.ts b/examples/improve/improve.ts index d7f09b44..b5ca8dec 100644 --- a/examples/improve/improve.ts +++ b/examples/improve/improve.ts @@ -1,43 +1,27 @@ -/** - * `improve()` — the one pluggable RSI verb, offline. - * - * `improve(profile, findings, opts)` optimizes ONE surface of an agent's profile (here the system - * prompt) and returns the frozen winner only if it clears the held-out gate. It is a facade over agent-eval's - * `selfImprove`: you name a `surface` and it picks the matching default mutator, extracts the baseline - * from the profile, and runs the loop. A ship verdict still requires a separate measured approval and - * product-owned activation; this function never mutates the live profile. - * - * The REQUIRED positional `findings` (an `AnalystFinding[]`) is what the loop reflects on: the trace - * analysts' read of what went wrong. Here it is a single hand-written finding. - * - * This example runs OFFLINE with no credentials: a scripted `SurfaceProposer` proposes a fixed - * winning candidate, a deterministic judge scores it, and the "agent" returns the surface verbatim - * while reporting token usage (so agent-eval's backend-integrity guard sees a real backend). Mirrors - * `tests/improve.test.ts`. - * - * Run: pnpm tsx examples/improve/improve.ts - */ +/** Complete-method prompt improvement, offline and deterministic. */ import { makeFinding } from '@tangle-network/agent-eval' +import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' import type { DispatchContext, JudgeConfig, MutableSurface, Scenario, - SurfaceProposer, } from '@tangle-network/agent-eval/contract' import type { AgentProfile } from '@tangle-network/agent-interface' -import { improve } from '@tangle-network/agent-runtime' +import { type ImproveMethodFactory, improve } from '@tangle-network/agent-runtime' export interface DemoScenario extends Scenario { kind: 'demo' } -// 12 trivial scenarios — enough for the held-out gate's minimum-evidence floor. export const scenarios: DemoScenario[] = Array.from({ length: 12 }, (_, i) => ({ id: `s${i}`, kind: 'demo' as const, })) +export const trainScenarios = scenarios.slice(0, 4) +export const selectionScenarios = scenarios.slice(4, 8) +export const testScenarios = scenarios.slice(8) // The agent returns the surface verbatim as the artifact AND reports usage, so the backend-integrity // guard sees a real backend rather than a stub-zero cell. No LLM. @@ -73,14 +57,17 @@ export const judge: JudgeConfig = { }, } -// A scripted SurfaceProposer that always proposes the winning surface — the offline stand-in for -// `gepaProposer`, no router call. -export const scriptedWinner: SurfaceProposer = { - kind: 'scripted-winner', - async propose() { - return [{ surface: 'PROMOTED', label: 'win', rationale: 'from findings' }] +// A complete deterministic method for the offline example. Production callers +// pass `officialGepa(...)`, `officialSkillOpt(...)`, or another OptimizationMethod. +export const scriptedWinner: ImproveMethodFactory = (context) => ({ + name: 'scripted-complete-method', + async optimize() { + return { + winnerSurface: context.findings.length > 0 ? 'PROMOTED' : context.baselineSurface, + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + } }, -} +}) // What the loop reflects on: the trace analysts' read of what went wrong. `makeFinding` stamps the // schema-version / finding-id / timestamp the full `AnalystFinding` shape requires. @@ -98,19 +85,24 @@ const findings = [ export const profile: AgentProfile = { name: 'demo', prompt: { systemPrompt: 'BASELINE' } } async function main(): Promise { - const out = await improve(profile, findings, { + const out = await improve(profile, { surface: 'prompt', - generator: scriptedWinner, - scenarios, - judge, + method: scriptedWinner, + findings, + trainScenarios, + selectionScenarios, + testScenarios, + judges: [judge], agent, - // A perfect +1.0 lift at this n/reps clears the default held-out gate. - budget: { generations: 1, populationSize: 2, reps: 3, holdoutFraction: 0.5 }, + runDir: 'mem://improve-example', + storage: inMemoryCampaignStorage(), + resamples: 40, + confidence: 0.95, }) console.log( - 'improve() proposed a detached prompt candidate and measured it on held-out scenarios (offline: scripted proposer, deterministic judge):', + 'improve() proposed a detached prompt candidate and measured it on final-test scenarios (offline complete method, deterministic judge):', ) - console.log(`decision: ${out.decision} lift: ${out.lift?.toFixed(3) ?? 'deferred'}`) + console.log(`decision: ${out.decision} lift: ${out.lift.toFixed(3)}`) console.log(`candidate prompt: ${out.candidate.profile?.prompt?.systemPrompt}`) console.log(`live prompt unchanged: ${profile.prompt?.systemPrompt}`) } diff --git a/examples/intelligence-recommend/README.md b/examples/intelligence-recommend/README.md index a9e8ff0f..e704a343 100644 --- a/examples/intelligence-recommend/README.md +++ b/examples/intelligence-recommend/README.md @@ -2,36 +2,34 @@ An agent finishes a run, a log of that run gets turned into concrete complaints ("the agent under-specifies its answer format"), and those complaints drive a prompt candidate that must beat -the old prompt on held-back test cases. This example runs that whole chain end to -end on your laptop with **no API key and no network** — a scripted stand-in plays every part that -would normally call a model, so you can see the shape of the loop before you wire real components in. +the old prompt on held-back test cases. +This example runs that whole chain end to end on your laptop with **no API key and no network**. +A deterministic method replaces the optimizer call so you can inspect the flow before wiring a live method. ## Why it matters Most "self-improving agent" demos hand-write the feedback, which is the interesting part. This one proves the missing link: feedback that is **derived from an actual recorded run** and carries a -pointer back to the exact trace it came from, so every proposed change is traceable to evidence. And -the rewrite is **gated** — it is compared against the current prompt on a set of scenarios held out -from the ones used to generate it, so a change that only looks good on paper is held. +pointer back to the exact trace it came from. +The selected rewrite is compared against the current prompt on untouched final-test scenarios. ## How it works Three plain steps, all in `intelligence-recommend.ts`: -1. **Observe** — a run emits a stream of events (loop started, loop ended, cost, iteration count). +1. **Observe**: a run emits a stream of events (loop started, loop ended, cost, iteration count). `recordTrace(events)` bundles that stream into one trace with an id. Exporting the trace to a collector is best-effort, so with no collector configured it does nothing and the demo stays offline. -2. **Analyze** — turn the trace into `findings`: short claims about what went wrong, each tagged with +2. **Analyze**: turn the trace into `findings`, short claims about what went wrong, each tagged with a severity and an `evidence_refs` pointer to the trace it came from. In production a "trace analyst" agent writes these by reading the run; here two are hand-written so the demo needs no model. -3. **Improve** — `improve(profile, findings, ...)` reads the findings and proposes a new system - prompt, then checks it on held-out scenarios and reports a decision. It does not change the live - profile. Here the proposer - and judge are scripted and deterministic. +3. **Improve**: `improve(profile, { method, findings, ... })` gives those findings to a complete method, + then checks its selected prompt on final-test scenarios. + It does not change the live profile. -## See it work — no API key needed +## See it work ```bash pnpm tsx examples/intelligence-recommend/intelligence-recommend.ts @@ -56,7 +54,4 @@ live prompt unchanged: BASELINE ## Going live -To run this for real: give `createIntelligenceClient` a tenant `apiKey` (and, if not the prod plane, a -`baseUrl` / `TANGLE_INTELLIGENCE_URL`) so traces actually export; replace the two hand-written findings with a -real trace-analyst agent's output; and drop the scripted proposer for a model-backed one (pass an -`llm` instead of a `generator`) so the rewrite is reflected from the findings rather than pre-canned. +For a live run, configure `createIntelligenceClient` so traces export, replace the two deterministic findings with trace-analyst output, and replace `scriptedWinner` with `officialGepa(...)`, `officialSkillOpt(...)`, or another complete method. diff --git a/examples/intelligence-recommend/intelligence-recommend.ts b/examples/intelligence-recommend/intelligence-recommend.ts index bc296583..6f8809cd 100644 --- a/examples/intelligence-recommend/intelligence-recommend.ts +++ b/examples/intelligence-recommend/intelligence-recommend.ts @@ -4,7 +4,7 @@ * * This is `improve()` (see examples/improve/) with ONE thing changed: the findings the loop * reflects on are no longer hand-written — they are DERIVED from a recorded run trace. The agent - * harness (scenarios, agent, judge, the scripted proposer, the baseline profile) is imported from + * harness (scenarios, agent, judge, the complete method, the baseline profile) is imported from * improve.ts verbatim, so this file shows only the NEW seam a Recommend mode runs in production: * * 1. OBSERVE — record a run's `LoopTraceEvent` stream as one trace (best-effort export; with no @@ -17,10 +17,19 @@ */ import { makeFinding } from '@tangle-network/agent-eval' +import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' import { improve } from '@tangle-network/agent-runtime' import { createIntelligenceClient } from '@tangle-network/agent-runtime/intelligence' import type { LoopTraceEvent } from '@tangle-network/agent-runtime/loops' -import { agent, judge, profile, scenarios, scriptedWinner } from '../improve/improve' +import { + agent, + judge, + profile, + scriptedWinner, + selectionScenarios, + testScenarios, + trainScenarios, +} from '../improve/improve' // ── 1. OBSERVE — record a run's loop topology as one trace ────────────────── // A real run emits this `LoopTraceEvent` stream from the kernel; here a tiny two-event trace stands @@ -68,13 +77,19 @@ const findings = [ // ── 3. IMPROVE — feed the derived findings to the gated RSI verb (offline) ── async function main(): Promise { - const out = await improve(profile, findings, { + const out = await improve(profile, { surface: 'prompt', - generator: scriptedWinner, - scenarios, - judge, + method: scriptedWinner, + findings, + trainScenarios, + selectionScenarios, + testScenarios, + judges: [judge], agent, - budget: { generations: 1, populationSize: 2, reps: 3, holdoutFraction: 0.5 }, + runDir: 'mem://intelligence-recommend', + storage: inMemoryCampaignStorage(), + resamples: 40, + confidence: 0.95, }) console.log(`trace recorded: ${traceId}`) console.log(`findings derived: ${findings.length}`) diff --git a/examples/self-improving-coder/README.md b/examples/self-improving-coder/README.md index e70a7843..0887e7fa 100644 --- a/examples/self-improving-coder/README.md +++ b/examples/self-improving-coder/README.md @@ -78,6 +78,6 @@ To see a real promotion, give the loop a task with a middle band — some attemp ## Related examples -- [`../improve`](../improve) — the one-call `improve(profile, findings)` shortcut over this same loop. +- [`../improve`](../improve) shows the complete-method `improve(profile, options)` path. - [`../self-improving-loop`](../self-improving-loop) — the same held-out gate on a text/prompt task, fully offline. - [`../strategy-evolution`](../strategy-evolution) — the multi-generation search in isolation, without the coding task. diff --git a/examples/self-improving-loop/self-improving-loop.ts b/examples/self-improving-loop/self-improving-loop.ts index 36ceb8dc..12e8fd4f 100644 --- a/examples/self-improving-loop/self-improving-loop.ts +++ b/examples/self-improving-loop/self-improving-loop.ts @@ -16,8 +16,8 @@ // // The finding type and the gate statistic are the real substrate primitives (not a local one-off); // the analyst body, the proposer, and the LLM are scripted ONLY so the demo runs offline and -// deterministically. The production path is `improve()` over `selfImprove` (see examples/improve/) -// or `runStrategyEvolution` + `promotionGate`. See README.md for the conceptual map. +// deterministically. The production profile path is `improve()` with a complete +// OptimizationMethod and explicit train, selection, and final-test partitions. import { type AnalystFinding, makeFinding, pairedBootstrap } from '@tangle-network/agent-eval' import { @@ -119,10 +119,11 @@ const conversationJudge: JudgeConfig<{ transcript: MultishotMessage[]; persona: } // ── 5. Analyst — reads v0 transcripts + scores, emits a canonical AnalystFinding ── -// The reflective step a production analyst-loop runs (@tangle-network/agent-runtime/analyst-loop / -// improvementDriver). The finding type is the canonical `AnalystFinding` from agent-eval — stamped via +// The reflective step a production analyst loop runs before a complete optimization method. +// The finding type is the canonical `AnalystFinding` from agent-eval — stamped via // `makeFinding` (schema-version / finding-id / timestamp) — NOT a local one-off interface; that is the -// exact shape `improve(profile, findings, opts)` reflects on. The proposed mutation rides +// exact finding shape a method factory receives through `improve(profile, { findings, ... })`. +// The proposed mutation rides // `recommended_action`. Offline we derive it deterministically so the demo stays reproducible. async function runAnalyst( @@ -153,9 +154,8 @@ function applyMutation(base: AgentProfile, mutation: string): AgentProfile { } // ── 6. Gate — promote v1 only if the PAIRED-bootstrap CI lower bound clears 0 ── -// This shows the STATISTICAL CORE of the production held-out gate (`HeldOutGate` / `improve()` over -// `selfImprove`, `agent-eval/contract`): pair v1 against v0 per scenario, bootstrap a CI on the median -// paired delta, and ship only if the CI's lower bound beats 0 — i.e. the lift is unlikely to be luck. +// This shows the statistical core of `improve()` final-test scoring: pair v1 against v0 per scenario, +// bootstrap a CI on the median paired delta, and ship only if the CI's lower bound beats 0. // `pairedBootstrap` (the same statistics primitive the real gate is built on) does exactly this; the // `seed` keeps it deterministic offline. // diff --git a/package.json b/package.json index b863772e..b874bec5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.103.0", + "version": "0.104.0", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { @@ -122,7 +122,7 @@ }, "devDependencies": { "@biomejs/biome": "^2.4.15", - "@tangle-network/agent-eval": "0.123.0", + "@tangle-network/agent-eval": "0.126.0", "@tangle-network/agent-interface": "0.32.0", "@tangle-network/sandbox": "^0.11.1", "@types/node": "^25.9.3", @@ -154,7 +154,7 @@ "license": "MIT", "packageManager": "pnpm@10.28.0", "peerDependencies": { - "@tangle-network/agent-eval": ">=0.122.8 <0.124.0", + "@tangle-network/agent-eval": ">=0.126.0 <0.127.0", "@tangle-network/agent-interface": ">=0.32.0 <0.33.0", "@tangle-network/sandbox": ">=0.11.1 <1.0.0", "playwright": "^1.40.0" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b6005e86..db9e0db7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -22,8 +22,8 @@ importers: specifier: ^2.4.15 version: 2.4.15 '@tangle-network/agent-eval': - specifier: 0.123.0 - version: 0.123.0(typescript@5.9.3) + specifier: 0.126.0 + version: 0.126.0(typescript@5.9.3) '@tangle-network/agent-interface': specifier: 0.32.0 version: 0.32.0 @@ -61,8 +61,8 @@ importers: bench: dependencies: '@tangle-network/agent-eval': - specifier: 0.123.5 - version: 0.123.5(typescript@5.9.3) + specifier: 0.126.0 + version: 0.126.0(typescript@5.9.3) '@tangle-network/agent-interface': specifier: 0.32.0 version: 0.32.0 @@ -683,18 +683,16 @@ packages: '@tangle-network/agent-core@0.4.11': resolution: {integrity: sha512-5B1IjrJ8xDR7w8Hv/MSk2ixul6NEJQ5Ftzo+z6l7ipeFpc0yaf1+Kkml4HDUEmGW/rqdrNpBf9/MpT7i/SY0PA==} + '@tangle-network/agent-core@0.4.20': + resolution: {integrity: sha512-gJzZh5PqPJtWW6kMEfm3IP7CGibvHdVXM4uqCAuCy+Kvg9TlVVkzXqZPRt9gU44mjdw5hDGoVzZotFsPoHAlFw==} + '@tangle-network/agent-eval@0.122.8': resolution: {integrity: sha512-7v0us+6zpcR+VKqt9olG7C7j4KlTctvwhlHgm2qHU+FBorEJFcJs6JDqWMw+3h7t9ba/07muIb0AfuvSsmJJ8w==} engines: {node: '>=20'} hasBin: true - '@tangle-network/agent-eval@0.123.0': - resolution: {integrity: sha512-Mwx28FpEny3It3gljb1EW15M+JHDw6vxd8xbUnQ43IlB0LiTl+Kz18qnJpLFMXeHau1bXEWPGvB8c9fH4Ml8Aw==} - engines: {node: '>=20'} - hasBin: true - - '@tangle-network/agent-eval@0.123.5': - resolution: {integrity: sha512-3pRITnTek+sTfQ6zfOORvWK1gCLmi8FUpwzZjYAYOvB5ge7QivRtkAR7dsKYaE8bXV749IWt9+Jd1xGfIzM6Zw==} + '@tangle-network/agent-eval@0.126.0': + resolution: {integrity: sha512-IY2GJ+DmZcTzWLYj+87sz+agf3TZ/pQyYD38NUZifCIMh7pgO7le/rebPJFII9mNHMek+ojCH8edUm9q90TydA==} engines: {node: '>=20'} hasBin: true @@ -1778,26 +1776,12 @@ snapshots: '@tangle-network/agent-interface': 0.26.0 zod: 4.4.3 - '@tangle-network/agent-eval@0.122.8(typescript@5.9.3)': + '@tangle-network/agent-core@0.4.20': dependencies: - '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) - '@ax-llm/ax': 23.0.1(zod@4.4.3) - '@hono/node-server': 2.0.8(hono@4.12.30) - '@tangle-network/agent-core': 0.4.11 - '@tangle-network/agent-interface': 0.31.0 - '@tangle-network/tcloud': 0.4.14(typescript@5.9.3)(zod@4.4.3) - hono: 4.12.30 + '@tangle-network/agent-interface': 0.32.0 zod: 4.4.3 - transitivePeerDependencies: - - '@mastra/core' - - '@modelcontextprotocol/sdk' - - ai - - bufferutil - - openai - - typescript - - utf-8-validate - '@tangle-network/agent-eval@0.123.0(typescript@5.9.3)': + '@tangle-network/agent-eval@0.122.8(typescript@5.9.3)': dependencies: '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) '@ax-llm/ax': 23.0.1(zod@4.4.3) @@ -1816,13 +1800,13 @@ snapshots: - typescript - utf-8-validate - '@tangle-network/agent-eval@0.123.5(typescript@5.9.3)': + '@tangle-network/agent-eval@0.126.0(typescript@5.9.3)': dependencies: '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) '@ax-llm/ax': 23.0.1(zod@4.4.3) '@hono/node-server': 2.0.8(hono@4.12.30) - '@tangle-network/agent-core': 0.4.11 - '@tangle-network/agent-interface': 0.31.0 + '@tangle-network/agent-core': 0.4.20 + '@tangle-network/agent-interface': 0.32.0 '@tangle-network/tcloud': 0.4.14(typescript@5.9.3)(zod@4.4.3) hono: 4.12.30 zod: 4.4.3 diff --git a/skills/build-with-agent-runtime/SKILL.md b/skills/build-with-agent-runtime/SKILL.md index bf1bee39..81b2ad8d 100644 --- a/skills/build-with-agent-runtime/SKILL.md +++ b/skills/build-with-agent-runtime/SKILL.md @@ -50,11 +50,16 @@ Do not move product storage transactions into a provider-neutral package. ## Improvement flow -`improve(profile, findings, options)` searches one surface and returns a detached winner. +`improve(profile, options)` searches one surface and returns a detached winner. It never changes a profile, document, repository, memory store, or knowledge base. -Supported profile surfaces are prompt, one named inline skill, tools, MCP, hooks, subagents, whole profile, and curated memory in `profile.resources.instructions`. -Code uses isolated worktrees and returns a sealed patch candidate. +For a profile field, pass one complete agent-eval `OptimizationMethod`, explicit train, selection, and final-test partitions, judges, and the candidate execution function. +Use `officialGepa(...)` with an explicit recipe when upstream GEPA should own search. +Use `officialSkillOpt(...)` when Microsoft's SkillOpt should own search. +Both require `evaluationId`; change it whenever dispatch, judges, models, or scoring behavior changes. +Resumable runs accept `never`, `if-compatible`, or `required` and reuse state only when agent-eval derives the same run identity. +Runtime has no local prompt, skill, memory, or profile optimizer fallback. +Code uses Runtime's isolated worktrees and returns a sealed patch candidate. Knowledge uses `runKnowledgeImprovementJob(...)` and returns paired snapshots. Use `proposeAgentImprovement(...)` for a production proposal. @@ -80,12 +85,12 @@ Never treat a lost response as a failed write without reconciling it. ## Surface rules -- Prompt changes `profile.prompt` only. -- Skill optimization selects one inline skill by `skills.resourceName`; profile resources must fail closed. +- Prompt changes `profile.prompt` only and requires a complete method. +- Skill optimization selects one inline skill by `skills.resourceName`, requires a complete method, and requires profile resources to fail closed. - Curated memory changes `profile.resources.instructions`; retrieval stores and memory databases belong in the knowledge flow. -- Tools, MCP, hooks, subagents, and whole-profile changes require an explicit proposer because Runtime cannot invent domain capabilities safely. +- Tools, MCP, hooks, subagents, curated memory, rollout policy, and whole-profile changes require a complete method. - Code candidates must come from the Runtime worktree path so patch identity and cleanup stay intact. -- Workflow and policy files are code surfaces; parameter sweeps come from agent-eval. +- Workflow files are code surfaces. Parameter sweeps belong in a complete agent-eval method. - Knowledge candidates remain detached until the shared activation path applies or restores their frozen snapshots. ## Product integration @@ -104,7 +109,8 @@ The product must not recreate candidate hashing, paired comparison, confidence i ## Do not duplicate - Do not write a provider-specific profile wrapper; extend `AgentProfile` and its materializer. -- Do not write a second optimizer loop; compose Eval proposers through `improve(...)`. +- Do not write a second optimizer loop; pass a complete agent-eval method to `improve(...)`. +- Do not use Runtime's code generator to approximate GEPA, SkillOpt, or another upstream profile optimizer. - Do not write a second candidate catalog; persist the immutable proposal records. - Do not let an analyst or adapter commit, push, open a pull request, or edit a live store. - Do not hand-roll SSE parsing, usage totals, profile matrices, bootstrap statistics, sandbox acquisition, or worktree cleanup. diff --git a/src/improvement/agentic-generator.ts b/src/improvement/agentic-generator.ts index 21bcc398..83c402db 100644 --- a/src/improvement/agentic-generator.ts +++ b/src/improvement/agentic-generator.ts @@ -1,7 +1,6 @@ /** * - * `agenticGenerator` — the full-agentic `CandidateGenerator`: the - * `shots=N, sandbox=on` setting of the one `improvementDriver`. It runs a real + * `agenticGenerator` — the full-agentic `CandidateGenerator`. It runs a real * coding harness (claude / codex / opencode) inside the candidate worktree the * driver already created, letting the agent read the codebase + the research * report and make the change in place. The driver then commits the worktree @@ -237,7 +236,7 @@ export function agenticGenerator(opts: AgenticGeneratorOptions = {}): CandidateG kind: `agentic:${harness}`, // The seed repo + (in rawTraceContext mode) the raw-trace filesystem context // are the change signal — an agentic coder proposes from them even when the - // distiller yielded zero findings. Without this, the improvementDriver's + // distiller yielded zero findings. Without this, the code candidate driver's // empty-findings guard short-circuits and generates ZERO candidates on the // first (and, for a single-generation run, only) proposal round. proposesWithoutFindings: true, diff --git a/src/improvement/campaign-otlp.test.ts b/src/improvement/campaign-otlp.test.ts deleted file mode 100644 index 86060ef6..00000000 --- a/src/improvement/campaign-otlp.test.ts +++ /dev/null @@ -1,231 +0,0 @@ -/** - * Campaign spans→OTLP converter + resolver proof, at the REAL seams: - * - * 1. The published `runCampaign` records `spans.jsonl` per cell (nothing - * hand-crafted); `campaignTraceResolver` converts that recording to - * OTLP-flat JSONL the published `OtlpFileTraceStore` indexes — the exact - * store the trace-analyst registry reads. - * 2. The published `traceAnalystProposer` wired with the resolver produces a - * candidate from those real recorded traces (analyze + apply seams - * injected — offline, no LLM). - * - * The regression this guards: the campaign trace format - * (`{name, cellId, startMs, durationMs, ...attrs}`) is NOT the OTLP shape the - * analysts parse, so without the converter every trace-native proposer threw - * "resolveTraces returned no OTLP traces" on the traces the loop itself wrote. - */ - -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { makeFinding } from '@tangle-network/agent-eval' -import { - runCampaign, - type Scenario, - traceAnalystProposer, -} from '@tangle-network/agent-eval/campaign' -import type { DispatchContext } from '@tangle-network/agent-eval/contract' -import { OtlpFileTraceStore } from '@tangle-network/agent-eval/traces' -import { afterAll, describe, expect, it } from 'vitest' -import { - campaignCellSpansToOtlp, - campaignTraceResolver, - convertCampaignDirToOtlp, -} from './campaign-otlp' - -const scenarios: Scenario[] = [ - { id: 'alpha', kind: 'fixture' }, - { id: 'beta', kind: 'fixture' }, -] - -/** Dispatch that reports usage AND records a tool-ish span — the two write - * paths (`ctx.cost.observe` → `cost.` spans, `ctx.trace.span`) the - * real substrate uses to populate `spans.jsonl`. */ -async function tracedDispatch(scenario: Scenario, ctx: DispatchContext): Promise<{ text: string }> { - const paid = await ctx.cost.runPaidCall({ - channel: 'agent', - actor: 'stub', - model: 'deterministic-test', - maximumCharge: { externallyEnforcedMaximumUsd: 0.0002 }, - execute: async () => ({ text: `artifact-${scenario.id}` }), - receipt: () => ({ - model: 'deterministic-test', - inputTokens: 3, - outputTokens: 5, - actualCostUsd: 0.0002, - }), - }) - if (!paid.succeeded) throw paid.error - const span = ctx.trace.span('worker.turn', { 'tool.name': 'grep' }) - span.end({ outcome: 'ok' }) - return paid.value -} - -const root = mkdtempSync(join(tmpdir(), 'campaign-otlp-')) -afterAll(() => rmSync(root, { recursive: true, force: true })) - -/** One real campaign per loop slot, into the run-optimization layout. */ -async function recordLayout(): Promise { - for (const dir of [join(root, 'baseline'), join(root, 'gen-0', 'candidate-0')]) { - await runCampaign({ - scenarios, - dispatch: tracedDispatch, - runDir: dir, - resumable: false, - }) - } -} - -describe('campaignTraceResolver — real runCampaign recording → OTLP the analysts index', () => { - it('converts the recorded spans.jsonl and OtlpFileTraceStore indexes them', async () => { - await recordLayout() - - // Proposing generation 0 reads the BASELINE campaign's traces. - const resolver = campaignTraceResolver({ runDir: root }) - const otlp = resolver({ generation: 0 }) - expect(otlp.trim()).not.toBe('') - - const lines = otlp.trim().split('\n') - const parsed = lines.map((l) => JSON.parse(l) as Record) - // Every line carries the OTLP identity fields the analysts key on. - for (const p of parsed) { - expect(typeof p.trace_id).toBe('string') - expect(typeof p.span_id).toBe('string') - } - // One trace per cell: 2 scenarios × 1 rep. - expect(new Set(parsed.map((p) => p.trace_id)).size).toBe(2) - const names = parsed.map((p) => p.name as string) - expect(names.some((n) => n.startsWith('cell.'))).toBe(true) - expect(names).toContain('cost.stub') - expect(names).toContain('worker.turn') - // Only baseline traces — gen-0 stays out of generation 0's view. - for (const p of parsed) { - const res = p.resource as { attributes: Record } - expect(String(res.attributes['campaign.cell_dir'])).toContain(join(root, 'baseline')) - } - - // The consumer contract: the published analyst store indexes the output. - const tracePath = join(root, 'converted.jsonl') - writeFileSync(tracePath, otlp) - const store = new OtlpFileTraceStore({ path: tracePath }) - const overview = await store.getOverview() - expect(overview.total_traces).toBe(2) - expect(overview.tool_names).toContain('grep') - expect(overview.errors.span_count).toBe(0) - - // Generation 1 reads gen-0; a generation with no recorded predecessor - // falls back to every trace under the run root. - const gen1 = resolver({ generation: 1 }) - expect(gen1).toContain(join(root, 'gen-0')) - expect(gen1).not.toContain('baseline') - const gen5 = resolver({ generation: 5 }) - const gen5Ids = new Set( - gen5 - .trim() - .split('\n') - .map((l) => (JSON.parse(l) as { trace_id: string }).trace_id), - ) - expect(gen5Ids.size).toBe(4) - }) - - it('traceAnalystProposer consumes the resolver end-to-end and returns a candidate', async () => { - let analyzedTraces = 0 - const proposer = traceAnalystProposer({ - baseUrl: 'https://stub.local/v1', - apiKey: 'stub-key', - model: 'stub-model', - resolveTraces: campaignTraceResolver({ runDir: root }), - // Analyst seam: assert the materialized trace file is REAL converted - // content (indexable by the same store the registry uses), then return - // one finding for the apply step. - analyze: async (tracePath) => { - const store = new OtlpFileTraceStore({ path: tracePath }) - const overview = await store.getOverview() - analyzedTraces = overview.total_traces - return [ - makeFinding({ - analyst_id: 'test-analyst', - severity: 'high', - area: 'tool-use', - confidence: 1, - claim: 'worker.turn spans show grep-only exploration', - recommended_action: 'instruct the agent to read files before editing', - evidence_refs: [], - }), - ] - }, - // Apply seam: the one LLM edit, stubbed offline. - fetchImpl: async () => - new Response( - JSON.stringify({ - choices: [{ message: { content: 'REVISED PROMPT: read before editing' } }], - usage: { prompt_tokens: 1, completion_tokens: 1 }, - model: 'stub-model', - }), - { status: 200, headers: { 'content-type': 'application/json' } }, - ), - }) - - const candidates = await proposer.propose({ - currentSurface: 'baseline prompt', - history: [], - findings: [], - populationSize: 1, - generation: 0, - signal: new AbortController().signal, - }) - - expect(analyzedTraces).toBe(2) - expect(candidates.length).toBeGreaterThanOrEqual(1) - const first = candidates[0] as { surface: unknown } - expect(String(first.surface)).toContain('REVISED PROMPT') - }) -}) - -describe('campaignCellSpansToOtlp — wire-shape unit checks', () => { - it('maps error records to STATUS_CODE_ERROR and skips torn lines', () => { - const content = [ - JSON.stringify({ name: 'ok.step', cellId: 'c', startMs: 1000, durationMs: 5 }), - '{"torn": ', - JSON.stringify({ name: 'bad.step', cellId: 'c', startMs: 2000, error: 'boom' }), - ].join('\n') - const lines = campaignCellSpansToOtlp(content, { cellId: 'c', cellKey: '/tmp/x/c' }) - // Root anchor + 2 records (torn line skipped). - expect(lines).toHaveLength(3) - const spans = lines.map((l) => JSON.parse(l) as Record) - const anchor = spans[0] as { status: { code: string }; parent_span_id: string } - expect(anchor.parent_span_id).toBe('') - expect(anchor.status.code).toBe('STATUS_CODE_ERROR') - const bad = spans.find((s) => s.name === 'bad.step') as { - status: { code: string; message: string } - } - expect(bad.status).toEqual({ code: 'STATUS_CODE_ERROR', message: 'boom' }) - const ok = spans.find((s) => s.name === 'ok.step') as { - status: { code: string } - start_time: string - end_time: string - } - expect(ok.status.code).toBe('STATUS_CODE_OK') - expect(Date.parse(ok.end_time) - Date.parse(ok.start_time)).toBe(5) - }) - - it('distinct cell paths with the same cellId yield distinct traces', () => { - const content = JSON.stringify({ name: 's', cellId: 'cell_a', startMs: 1 }) - const a = JSON.parse( - campaignCellSpansToOtlp(content, { cellId: 'cell_a', cellKey: '/run/baseline/cell_a' })[0]!, - ) as { trace_id: string } - const b = JSON.parse( - campaignCellSpansToOtlp(content, { - cellId: 'cell_a', - cellKey: '/run/gen-0/candidate-0/cell_a', - })[0]!, - ) as { trace_id: string } - expect(a.trace_id).not.toBe(b.trace_id) - expect(a.trace_id).toMatch(/^[0-9a-f]{32}$/) - }) - - it('empty content and an absent dir resolve to no traces, not a throw', () => { - expect(campaignCellSpansToOtlp('', { cellId: 'c' })).toEqual([]) - expect(convertCampaignDirToOtlp('/nonexistent/definitely-not-here')).toBe('') - }) -}) diff --git a/src/improvement/campaign-otlp.ts b/src/improvement/campaign-otlp.ts deleted file mode 100644 index 15ff3952..00000000 --- a/src/improvement/campaign-otlp.ts +++ /dev/null @@ -1,265 +0,0 @@ -/** - * Campaign `spans.jsonl` → OTLP-flat JSONL — the missing converter between - * what the substrate RECORDS and what its trace analysts READ. - * - * agent-eval's `runCampaign` durably records one `spans.jsonl` per cell - * (`defaultBuildTraceWriter`): flat records - * `{ name, cellId, startMs, durationMs?, ...attributes }` with no trace/span - * ids. Its trace consumers (`OtlpFileTraceStore`, the trace-analyst registry, - * `haloProposer`/`traceAnalystProposer` via `resolveTraces`) read OTLP-flat - * JSONL (`trace_id`/`span_id`/ISO times/status/attributes — the shape - * `projectOtlpFlatLine` parses). Nothing shipped converts between the two, so - * the traces real optimization runs write could never reach the trace-native - * proposers. This module is that wire adapter: - * - * - {@link campaignCellSpansToOtlp} — one cell's `spans.jsonl` content → - * OTLP lines (a per-cell root AGENT anchor span + one child per record). - * - {@link convertCampaignDirToOtlp} — walk any campaign run/generation dir - * for `spans.jsonl` files and concatenate their OTLP lines. - * - {@link campaignTraceResolver} — the `resolveTraces` implementation for - * `traceAnalystProposer`/`haloProposer`: proposing generation g reads the - * traces the loop just recorded (`gen-`, or `baseline` for g = 0) - * under the same `runDir` handed to `improve()`/`selfImprove()`. - * - * Trace identity: one trace per CELL, keyed on the cell's on-disk path — the - * same sanitized `cellId` recurs across the baseline and every candidate - * campaign, so folding the id alone would merge distinct runs into one trace. - * Ids are deterministic FNV-1a folds to OTLP's 32/16-hex width, so re-converts - * are stable and byte-identical. - */ - -import { type Dirent, readdirSync, readFileSync } from 'node:fs' -import { basename, dirname, join, resolve } from 'node:path' -import type { ProposeContext } from '@tangle-network/agent-eval/campaign' -import { OPENINFERENCE_SPAN_KIND } from '@tangle-network/agent-eval/traces' - -/** How deep {@link convertCampaignDirToOtlp} recurses looking for - * `spans.jsonl`. The deepest shipped layout is - * `/gen-/candidate-//spans.jsonl` (3 levels); one spare - * level tolerates a wrapper dir without letting a mis-pointed root scan a - * whole filesystem. */ -const MAX_WALK_DEPTH = 4 - -export interface CampaignOtlpOptions { - /** OTLP `service.name` on every emitted span. Default `'campaign'`. */ - serviceName?: string -} - -interface CampaignSpanRecord { - name: string - cellId: string - startMs: number - durationMs?: number - attributes: Record - error?: string -} - -/** - * Convert ONE cell's `spans.jsonl` content to OTLP-flat JSONL lines. - * `cellKey` is the identity the trace id folds from — pass the cell's on-disk - * path (unique per campaign); `cellId` is the display/attribute label. - * Returns `[]` for empty/recordless content (a dispatch that never touched - * `ctx.trace`/`ctx.cost` writes an empty file — that is data, not an error). - */ -export function campaignCellSpansToOtlp( - content: string, - cell: { cellId: string; cellKey?: string }, - opts: CampaignOtlpOptions = {}, -): string[] { - const records = parseCampaignSpans(content, cell.cellId) - if (records.length === 0) return [] - - const serviceName = opts.serviceName ?? 'campaign' - const key = cell.cellKey ?? cell.cellId - const traceId = foldTo32Hex(key) - const rootSpanId = foldTo16Hex(`${key}::root`) - - const startMs = Math.min(...records.map((r) => r.startMs)) - const endMs = Math.max(...records.map((r) => r.startMs + (r.durationMs ?? 0))) - const anyError = records.some((r) => r.error !== undefined) - - const resource = { - attributes: { - 'service.name': serviceName, - 'campaign.cell_id': cell.cellId, - ...(cell.cellKey ? { 'campaign.cell_dir': cell.cellKey } : {}), - }, - } - - const lines: string[] = [ - JSON.stringify({ - trace_id: traceId, - span_id: rootSpanId, - parent_span_id: '', - name: `cell.${cell.cellId}`, - start_time: msToIso(startMs), - end_time: msToIso(endMs), - status: anyError - ? { code: 'STATUS_CODE_ERROR', message: 'one or more spans errored' } - : { code: 'STATUS_CODE_OK', message: '' }, - resource, - attributes: { - [OPENINFERENCE_SPAN_KIND]: 'AGENT', - 'agent.name': cell.cellId, - }, - }), - ] - - for (let i = 0; i < records.length; i++) { - const r = records[i]! - lines.push( - JSON.stringify({ - trace_id: traceId, - span_id: foldTo16Hex(`${key}::${i}`), - parent_span_id: rootSpanId, - name: r.name, - start_time: msToIso(r.startMs), - end_time: msToIso(r.startMs + (r.durationMs ?? 0)), - status: - r.error === undefined - ? { code: 'STATUS_CODE_OK', message: '' } - : { code: 'STATUS_CODE_ERROR', message: r.error }, - resource, - attributes: r.attributes, - }), - ) - } - return lines -} - -/** - * Walk `dir` (a campaign run dir, a generation dir, or a whole `selfImprove` - * run root) for `spans.jsonl` files and return their concatenated OTLP-flat - * JSONL — the exact string the `resolveTraces` contract expects. `''` when no - * spans exist (the proposers fail loud on empty by design). - */ -export function convertCampaignDirToOtlp(dir: string, opts: CampaignOtlpOptions = {}): string { - const root = resolve(dir) - const files = findSpansFiles(root, MAX_WALK_DEPTH) - const lines: string[] = [] - for (const file of files) { - let content: string - try { - content = readFileSync(file, 'utf8') - } catch { - continue - } - const cellDir = dirname(file) - lines.push( - ...campaignCellSpansToOtlp(content, { cellId: basename(cellDir), cellKey: cellDir }, opts), - ) - } - return lines.length > 0 ? `${lines.join('\n')}\n` : '' -} - -export interface CampaignTraceResolverOptions extends CampaignOtlpOptions { - /** The `selfImprove`/`improve()` run root — the SAME `runDir` the loop - * records under (`/baseline/...`, `/gen-/candidate-/...`). - * Must be a real path; a `mem://` run records nothing to resolve. */ - runDir: string -} - -/** - * Build the `resolveTraces` function `traceAnalystProposer`/`haloProposer` - * take: proposing generation g reads the traces of the campaigns the loop just - * scored — `gen-` (or `baseline` when g = 0), falling back to every trace - * under the run root when that directory has none (e.g. a caller pointing at a - * single campaign dir rather than a loop root). - * - * traceAnalystProposer({ ..., resolveTraces: campaignTraceResolver({ runDir }) }) - */ -export function campaignTraceResolver( - opts: CampaignTraceResolverOptions, -): (ctx: Pick) => string { - return (ctx) => { - const priorDir = - ctx.generation <= 0 - ? join(opts.runDir, 'baseline') - : join(opts.runDir, `gen-${ctx.generation - 1}`) - const prior = convertCampaignDirToOtlp(priorDir, opts) - if (prior) return prior - return convertCampaignDirToOtlp(opts.runDir, opts) - } -} - -/** Parse `spans.jsonl` content into records, splitting the trace writer's - * reserved fields from the free-form attributes. Malformed lines are skipped - * (one torn write must not censor the rest of the cell). */ -function parseCampaignSpans(content: string, cellId: string): CampaignSpanRecord[] { - const out: CampaignSpanRecord[] = [] - for (const line of content.split('\n')) { - const trimmed = line.trim() - if (!trimmed) continue - let raw: unknown - try { - raw = JSON.parse(trimmed) - } catch { - continue - } - if (!raw || typeof raw !== 'object' || Array.isArray(raw)) continue - const o = raw as Record - if (typeof o.name !== 'string' || typeof o.startMs !== 'number') continue - - const attributes: Record = { 'campaign.cell_id': cellId } - for (const [k, v] of Object.entries(o)) { - if (k === 'name' || k === 'cellId' || k === 'startMs' || k === 'durationMs') continue - attributes[k] = v - } - out.push({ - name: o.name, - cellId, - startMs: o.startMs, - ...(typeof o.durationMs === 'number' ? { durationMs: o.durationMs } : {}), - attributes, - ...(typeof o.error === 'string' ? { error: o.error } : {}), - }) - } - return out -} - -/** All `spans.jsonl` files under `dir`, depth-bounded, symlink-dirs skipped - * (a cycle under a run dir must not hang the proposal round). Sorted for a - * deterministic concatenation order. */ -function findSpansFiles(dir: string, depth: number): string[] { - const out: string[] = [] - let entries: Dirent[] - try { - entries = readdirSync(dir, { withFileTypes: true }) - } catch { - return out - } - for (const entry of entries) { - const full = join(dir, entry.name) - if (entry.isFile() && entry.name === 'spans.jsonl') { - out.push(full) - } else if (depth > 0 && entry.isDirectory() && !entry.isSymbolicLink()) { - out.push(...findSpansFiles(full, depth - 1)) - } - } - return out.sort() -} - -function msToIso(ms: number): string { - return Number.isFinite(ms) && ms > 0 ? new Date(ms).toISOString() : new Date(0).toISOString() -} - -/** Deterministic FNV-1a fold to OTLP's 16-hex span-id width. */ -function foldTo16Hex(s: string): string { - const a = fnv1a(s) - const b = fnv1a(`${s}::salt`) - return a + b -} - -/** Deterministic FNV-1a fold to OTLP's 32-hex trace-id width. */ -function foldTo32Hex(s: string): string { - return foldTo16Hex(s) + foldTo16Hex(`${s}::trace`) -} - -function fnv1a(s: string): string { - let h = 0x811c9dc5 - for (let i = 0; i < s.length; i++) { - h ^= s.charCodeAt(i) - h = Math.imul(h, 0x01000193) >>> 0 - } - return h.toString(16).padStart(8, '0') -} diff --git a/src/improvement/improve.test.ts b/src/improvement/improve.test.ts index 4c9a6a73..06694aa5 100644 --- a/src/improvement/improve.test.ts +++ b/src/improvement/improve.test.ts @@ -1,78 +1,64 @@ -/** - * `improve()` default-proposer resolution proof. - * - * The regression this guards: `improve()` maps each surface to a default - * `SurfaceProposer` — `prompt → gepaProposer`, `skills → skillOptProposer`, - * `memory → memoryCurationProposer`. - * If either import resolves to `undefined` (a substrate export drift), the facade - * does not fail at module load — it fails at CALL time, the first time a caller - * names that surface. So a green typecheck is not enough; this test drives the - * REAL `improve()` far enough to construct the default proposer and run the - * baseline-only loop to a gate decision. - * - * It is deterministic and offline: `gate: 'none'` forces `generations = 0`, so - * `selfImprove` runs the baseline cells only and never calls the reflection LLM - * the proposer wraps. The stub agent reports a token-bearing cost through - * `ctx.cost` so the substrate's backend-integrity guard (default `'assert'`) - * sees a real backend rather than a silent-zero stub. - */ - +import { execFileSync } from 'node:child_process' +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import { gitWorktreeAdapter, + inMemoryCampaignStorage, + type OptimizationMethod, + type OptimizationMethodInput, type Worktree, type WorktreeAdapter, } from '@tangle-network/agent-eval/campaign' import type { - CodeSurface, DispatchContext, JudgeConfig, + MutableSurface, Scenario, - SurfaceProposer, } from '@tangle-network/agent-eval/contract' import type { AgentProfile } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { ConfigError } from '../errors' -import { type ImproveSurface, improve } from './improve' +import { improve } from './improve' -// Four scenarios so the train/holdout split is non-empty at the default 0.25 -// holdout fraction (a single scenario yields an empty train split). -const scenarios: Scenario[] = [ - { id: 'a', kind: 'fixture' }, - { id: 'b', kind: 'fixture' }, - { id: 'c', kind: 'fixture' }, - { id: 'd', kind: 'fixture' }, -] +interface TestScenario extends Scenario { + kind: 'fixture' +} -// A deterministic judge — every artifact scores the same. The POINT is the -// proposer wiring, not a score gradient. -const judge: JudgeConfig<{ text: string }, Scenario> = { - name: 'stub-judge', - dimensions: [{ key: 'q', description: 'fixture quality' }], - score: () => ({ dimensions: { q: 0.5 }, composite: 0.5, notes: '' }), +interface TextArtifact { + text: string } -const improvementJudge: JudgeConfig<{ text: string }, Scenario> = { - name: 'improvement-judge', - dimensions: [{ key: 'q', description: 'contains the measured improvement marker' }], +const trainScenarios: TestScenario[] = [{ id: 'train', kind: 'fixture' }] +const selectionScenarios: TestScenario[] = [{ id: 'selection', kind: 'fixture' }] +const testScenarios: TestScenario[] = [ + { id: 'test-a', kind: 'fixture' }, + { id: 'test-b', kind: 'fixture' }, +] +const allScenarios = [...trainScenarios, ...selectionScenarios, ...testScenarios] + +const improvementJudge: JudgeConfig = { + name: 'improvement', + dimensions: [{ key: 'quality', description: 'candidate contains the improvement marker' }], score: ({ artifact }) => { - const score = artifact.text.includes('improved') ? 1 : 0 - return { dimensions: { q: score }, composite: score, notes: '' } + const quality = artifact.text.includes('improved') ? 1 : 0 + return { dimensions: { quality }, composite: quality, notes: '' } }, } -async function paidStubCall( +async function paidText( + surface: MutableSurface, + _scenario: TestScenario, ctx: DispatchContext, - actor: string, - execute: () => T | Promise, -): Promise { +): Promise { const paid = await ctx.cost.runPaidCall({ channel: 'agent', - actor, - model: 'stub-model', + actor: 'test-agent', + model: 'deterministic-test', maximumCharge: { externallyEnforcedMaximumUsd: 0.0001 }, - execute: async () => execute(), + execute: async () => ({ text: String(surface) }), receipt: () => ({ - model: 'stub-model', + model: 'deterministic-test', inputTokens: 1, outputTokens: 1, actualCostUsd: 0.0001, @@ -82,1008 +68,547 @@ async function paidStubCall( return paid.value } -// The fake dispatch enters the same paid-call path as a real backend so the -// backend-integrity check sees its explicit token and cost receipt. -async function stubAgent( - surface: unknown, - _scenario: Scenario, - ctx: DispatchContext, -): Promise<{ text: string }> { - return paidStubCall(ctx, 'stub-agent', () => ({ text: String(surface) })) +function fixedMethod( + winnerSurface: MutableSurface, + inspect?: (input: OptimizationMethodInput) => void, +): OptimizationMethod { + return { + name: 'fixed-method', + async optimize(input) { + inspect?.(input) + return { + winnerSurface, + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + durationMs: 1, + } + }, + } } -const promptProfile = (): AgentProfile => ({ - name: 'fixture-agent', - prompt: { systemPrompt: 'be careful' }, -}) - -const skillDocument = '# Fixture skill\n\nCheck the result.\n' - -const skillProfile = (): AgentProfile => ({ - name: 'fixture-agent', - resources: { - failOnError: true, - skills: [{ kind: 'inline', name: 'fixture-skill', content: skillDocument }], - }, -}) +function methodOptions(method: OptimizationMethod): { + method: OptimizationMethod + trainScenarios: TestScenario[] + selectionScenarios: TestScenario[] + testScenarios: TestScenario[] + judges: JudgeConfig[] + agent: typeof paidText + runDir: string + storage: ReturnType + resamples: number + confidence: number +} { + return { + method, + trainScenarios, + selectionScenarios, + testScenarios, + judges: [improvementJudge], + agent: paidText, + runDir: `mem://improve-method-${Math.random()}`, + storage: inMemoryCampaignStorage(), + resamples: 40, + confidence: 0.95, + } +} -const memoryProfile = (document = '# Durable memory\n'): AgentProfile => ({ +const promptProfile = (): AgentProfile => ({ name: 'fixture-agent', - resources: { - failOnError: true, - instructions: { kind: 'inline', name: 'durable-memory', content: document }, - }, + prompt: { systemPrompt: 'baseline' }, }) -describe('improve() — default proposer resolution (substrate export drift guard)', () => { - it("surface 'prompt' resolves gepaProposer and runs the baseline loop without crashing", async () => { - const result = await improve(promptProfile(), [], { +describe('improve method execution', () => { + it('runs a complete method without exposing final-test cases and materializes its prompt', async () => { + let observed: OptimizationMethodInput | undefined + const profile = promptProfile() + const result = await improve(profile, { + ...methodOptions(fixedMethod('improved prompt', (input) => (observed = input))), surface: 'prompt', - gate: 'none', - scenarios, - judge, - agent: stubAgent, }) - // The default gepaProposer was constructed (not undefined) and selfImprove - // ran to a gate decision; a baseline-only run holds. - expect(result.decision).toBe('hold') - expect(result.candidate).toMatchObject({ - surface: 'prompt', - value: 'be careful', - profile: { prompt: { systemPrompt: 'be careful' } }, - }) + expect(observed?.trainScenarios.map((scenario) => scenario.id)).toEqual(['train']) + expect(observed?.selectionScenarios.map((scenario) => scenario.id)).toEqual(['selection']) + expect(JSON.stringify(observed)).not.toContain('test-a') + expect(result.mode).toBe('method') + expect(result.method).toBe('fixed-method') + expect(result.decision).toBe('ship') + expect(result.lift).toBe(1) + expect(result.liftInterval.low).toBeGreaterThan(0) + expect(result.candidate.profile?.prompt?.systemPrompt).toBe('improved prompt') + expect(profile.prompt?.systemPrompt).toBe('baseline') + expect(Object.isFrozen(result.candidate)).toBe(true) }) - it("surface 'skills' resolves skillOptProposer and runs the baseline loop without crashing", async () => { - const result = await improve(skillProfile(), [], { - surface: 'skills', - gate: 'none', - scenarios, - judge, - agent: stubAgent, - skills: { resourceName: 'fixture-skill' }, - }) - - expect(result.decision).toBe('hold') - expect(result.candidate).toMatchObject({ - surface: 'skills', - value: skillDocument, - profile: { resources: { skills: [{ content: skillDocument }] } }, + it('passes current findings to a method factory', async () => { + const findings = [{ claim: 'answers omit citations' }] + let observedFindings: readonly unknown[] | undefined + const result = await improve(promptProfile(), { + ...methodOptions(fixedMethod('unused')), + findings, + method: (context) => { + observedFindings = context.findings + expect(context.surface).toBe('prompt') + expect(context.baselineSurface).toBe('baseline') + return fixedMethod('improved prompt') + }, }) - }) - it("surface 'skills' fails loud without an exact inline resource identity", async () => { - await expect( - improve(skillProfile(), [], { - surface: 'skills', - gate: 'none', - scenarios, - judge, - agent: stubAgent, - }), - ).rejects.toThrow(/requires opts\.skills\.resourceName/) + expect(observedFindings).toEqual(findings) + expect(result.decision).toBe('ship') }) - it("surface 'memory' resolves memoryCurationProposer from exact profile instructions", async () => { - const result = await improve(memoryProfile(), [], { - surface: 'memory', - gate: 'none', - scenarios, - judge, - agent: stubAgent, + it('holds when the final-test interval does not clear the requested lift', async () => { + const result = await improve(promptProfile(), { + ...methodOptions(fixedMethod('improved prompt')), + minimumLift: 1, }) + expect(result.lift).toBe(1) expect(result.decision).toBe('hold') - expect(result.candidate.value).toBe('# Durable memory\n') - expect(result.candidate.profile?.resources?.instructions).toEqual({ - kind: 'inline', - name: 'durable-memory', - content: '# Durable memory\n', - }) - await expect( - improve(promptProfile(), [], { - surface: 'memory', - gate: 'none', - scenarios, - judge, - agent: stubAgent, - }), - ).rejects.toThrow(/requires profile\.resources\.failOnError/) }) - it("surface 'memory' returns a frozen profile without changing the baseline", async () => { - const baseline = '# Durable memory\n' - const profile = memoryProfile(baseline) - const result = await improve(profile, [{ claim: 'improved verification lesson' }], { - surface: 'memory', - scenarios, - judge: improvementJudge, - agent: stubAgent, - promotionGate: { - name: 'test-ship', - decide: async () => ({ - decision: 'ship', - reasons: ['test candidate'], - contributingGates: [], - }), - }, - budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, - }) - - expect(result.decision).toBe('ship') - expect(result.candidate.value).toContain('improved verification lesson') - expect(result.candidate.profile?.resources?.instructions).toMatchObject({ - content: expect.stringContaining('improved verification lesson'), + it('distinguishes train and selection boundaries in optimizer lineage', async () => { + const extra: TestScenario = { id: 'extra', kind: 'fixture' } + const first = await improve(promptProfile(), { + ...methodOptions(fixedMethod('improved prompt')), + trainScenarios: [trainScenarios[0]!, extra], + selectionScenarios, }) - expect(Object.isFrozen(result.candidate)).toBe(true) - expect(profile.resources?.instructions).toEqual({ - kind: 'inline', - name: 'durable-memory', - content: baseline, + const second = await improve(promptProfile(), { + ...methodOptions(fixedMethod('improved prompt')), + trainScenarios, + selectionScenarios: [extra, selectionScenarios[0]!], }) - }) - it("surface 'memory' rejects a non-text candidate", async () => { - const nonTextSurface: CodeSurface = { - kind: 'code', - worktreeRef: 'not-a-memory-document', - baseRef: 'main', - baseCommit: 'a'.repeat(40), - baseTree: 'b'.repeat(40), - candidateCommit: 'c'.repeat(40), - candidateTree: 'd'.repeat(40), - patch: { - format: 'git-diff-binary', - sha256: `sha256:${'e'.repeat(64)}`, - byteLength: 1, - }, - } - const surfaceKindJudge: JudgeConfig<{ text: string }, Scenario> = { - name: 'surface-kind-judge', - dimensions: [{ key: 'q', description: 'candidate is code-tier' }], - score: ({ artifact }) => { - const score = artifact.text === 'code' ? 1 : 0 - return { dimensions: { q: score }, composite: score, notes: '' } - }, - } - const malformedMemory: SurfaceProposer = { - kind: 'malformed-memory', - propose: async () => [ - { - surface: nonTextSurface, - label: 'wrong-tier', - rationale: 'test malformed winner', - }, - ], - } - await expect( - improve(memoryProfile(), [], { - surface: 'memory', - scenarios, - judge: surfaceKindJudge, - agent: async (surface, _scenario, ctx) => { - return paidStubCall(ctx, 'stub-agent', () => ({ - text: typeof surface === 'string' ? 'text' : surface.kind, - })) - }, - generator: malformedMemory, - promotionGate: { - name: 'test-ship', - decide: async () => ({ - decision: 'ship', - reasons: ['test candidate'], - contributingGates: [], - }), - }, - budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, - }), - ).rejects.toThrow(/incompatible surface value/) + expect(first.lineage.developmentSplitDigest).not.toBe(second.lineage.developmentSplitDigest) }) - it("surface 'memory' returns a held candidate without side effects", async () => { - const result = await improve(memoryProfile(), [], { - surface: 'memory', - gate: 'none', - scenarios, - judge, - agent: stubAgent, - }) - - expect(result.decision).toBe('hold') - expect(result.candidate.value).toBe('# Durable memory\n') - }) - - it('a real runDir makes the loop durable: provenance lands on the filesystem', async () => { - const { mkdtempSync, existsSync, rmSync } = await import('node:fs') - const { tmpdir } = await import('node:os') - const { join } = await import('node:path') - const runDir = mkdtempSync(join(tmpdir(), 'improve-rundir-')) - try { - const result = await improve(promptProfile(), [], { - surface: 'prompt', - gate: 'none', - scenarios, - judge, - agent: stubAgent, - runDir, - }) - expect(result.decision).toBe('hold') - // The durable-storage default keys off a real (non-mem://) runDir: the loop - // provenance record must survive the call on disk — this is what a 5-hour - // search recovers from after a process death. - expect(existsSync(join(runDir, 'loop-provenance.json'))).toBe(true) - } finally { - rmSync(runDir, { recursive: true, force: true }) - } - }) - - it("surface 'skills' returns an exact profile without changing the baseline", async () => { - // Baseline document; a scenario whose judge rewards the presence of a rule the - // skillOpt proposer will add. A deterministic stub proposer stands in for the LLM. - const baselineDoc = '# OR skills\n- always run a solver\n' - const stubProposer = { - kind: 'stub-skillopt', - async propose(ctx: { currentSurface: unknown }) { - // Prove the baseline surface is the DOCUMENT, not a refs array. - expect(ctx.currentSurface).toBe(baselineDoc) - return [ - { - surface: `${baselineDoc}- recompute the objective before writing\n`, - label: 'add-recompute-rule', - rationale: 'stub', - }, - ] - }, - } - // Judge: reward the document that contains the added rule. - const docJudge: JudgeConfig<{ doc: string }, Scenario> = { - name: 'doc-judge', - dimensions: [{ key: 'q', description: 'has recompute rule' }], - score: ({ artifact }) => { - const has = artifact.doc.includes('recompute the objective') - return { dimensions: { q: has ? 1 : 0 }, composite: has ? 1 : 0, notes: '' } - }, - } - const skillProfileWithRef = (): AgentProfile => ({ + it('materializes one exact inline skill', async () => { + const profile: AgentProfile = { name: 'fixture-agent', resources: { failOnError: true, - skills: [{ kind: 'inline', name: 'or-skills', content: baselineDoc }], + skills: [{ kind: 'inline', name: 'review', content: 'baseline skill' }], }, - }) - - const profile = skillProfileWithRef() - const result = await improve(profile, [], { + } + const result = await improve(profile, { + ...methodOptions(fixedMethod('improved skill')), surface: 'skills', - scenarios, - judge: docJudge, - agent: async (surface, _s, ctx) => { - return paidStubCall(ctx, 'stub', () => ({ doc: String(surface) })) - }, - generator: stubProposer as never, - skills: { resourceName: 'or-skills' }, - budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, + skills: { resourceName: 'review' }, }) - expect(typeof result.decision).toBe('string') - expect(result.candidate.value).toContain('recompute the objective') - expect(result.candidate.profile?.resources?.skills?.[0]).toMatchObject({ - content: expect.stringContaining('recompute the objective'), - }) + expect(result.candidate.profile?.resources?.skills).toEqual([ + { kind: 'inline', name: 'review', content: 'improved skill' }, + ]) expect(profile.resources?.skills).toEqual([ - { kind: 'inline', name: 'or-skills', content: baselineDoc }, + { kind: 'inline', name: 'review', content: 'baseline skill' }, ]) }) it.each([ + { + surface: 'tools' as const, + profile: { name: 'fixture-agent', tools: { Bash: true } }, + winner: '{"Read":true}', + expected: { tools: { Read: true } }, + }, + { + surface: 'mcp' as const, + profile: { name: 'fixture-agent', mcp: { old: { command: 'old-server' } } }, + winner: '{"search":{"command":"new-server"}}', + expected: { mcp: { search: { command: 'new-server' } } }, + }, + { + surface: 'hooks' as const, + profile: { name: 'fixture-agent', hooks: { Stop: [{ command: 'echo old' }] } }, + winner: '{"Stop":[{"command":"echo new"}]}', + expected: { hooks: { Stop: [{ command: 'echo new' }] } }, + }, { surface: 'subagents' as const, - profile: { name: 'fixture-agent', subagents: {} }, - winner: JSON.stringify({ reviewer: { prompt: 'improved review instructions' } }), - read: (profile: AgentProfile) => profile.subagents?.reviewer?.prompt, - expected: 'improved review instructions', + profile: { name: 'fixture-agent', subagents: { old: { prompt: 'Old prompt' } } }, + winner: '{"reviewer":{"prompt":"Review the result"}}', + expected: { subagents: { reviewer: { prompt: 'Review the result' } } }, }, { - surface: 'agent-profile' as const, - profile: { name: 'fixture-agent', prompt: { systemPrompt: 'baseline' } }, - winner: JSON.stringify({ + surface: 'memory' as const, + profile: { name: 'fixture-agent', - prompt: { systemPrompt: 'improved whole profile' }, - }), - read: (profile: AgentProfile) => profile.prompt?.systemPrompt, - expected: 'improved whole profile', - }, - ])('materializes a valid $surface profile candidate', async (fixture) => { - const result = await improve(fixture.profile, [{ finding: 'surface needs improvement' }], { - surface: fixture.surface, - scenarios, - judge: improvementJudge, - agent: stubAgent, - generator: { - kind: `stub-${fixture.surface}`, - propose: async () => [ - { surface: fixture.winner, label: 'candidate', rationale: 'test candidate' }, - ], + resources: { failOnError: true as const, instructions: 'baseline memory' }, }, - promotionGate: { - name: 'test-ship', - decide: async () => ({ - decision: 'ship', - reasons: ['test candidate'], - contributingGates: [], - }), + winner: 'improved memory', + expected: { + resources: { failOnError: true, instructions: 'improved memory' }, }, - budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, + }, + ])('materializes an exact $surface profile coordinate', async (testCase) => { + const result = await improve(testCase.profile, { + ...methodOptions(fixedMethod(testCase.winner)), + surface: testCase.surface, }) - expect(result.decision).toBe('ship') - if (!result.candidate.profile) throw new Error('expected a profile candidate') - expect(fixture.read(result.candidate.profile)).toEqual(fixture.expected) + expect(result.candidate.profile).toMatchObject(testCase.expected) + expect(testCase.profile).not.toMatchObject(testCase.expected) }) - it('rejects a config candidate that cannot form a valid AgentProfile', async () => { - await expect( - improve( - { name: 'fixture-agent', subagents: {} }, - [{ finding: 'surface needs improvement' }], - { - surface: 'subagents', - scenarios, - judge: improvementJudge, - agent: stubAgent, - generator: { - kind: 'stub-invalid-subagent', - propose: async () => [ - { - surface: JSON.stringify({ reviewer: { prompt: 'improved', maxSteps: 'many' } }), - label: 'candidate', - rationale: 'invalid candidate', - }, - ], - }, - promotionGate: { - name: 'test-ship', - decide: async () => ({ - decision: 'ship', - reasons: ['exercise validation'], - contributingGates: [], - }), - }, - budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, - }, - ), - ).rejects.toThrow(/valid AgentProfile/) + it('materializes a complete profile candidate', async () => { + const winner = JSON.stringify({ + name: 'fixture-agent', + prompt: { systemPrompt: 'improved whole profile' }, + }) + const result = await improve(promptProfile(), { + ...methodOptions(fixedMethod(winner)), + surface: 'agent-profile', + }) + + expect(result.candidate.profile).toEqual({ + name: 'fixture-agent', + prompt: { systemPrompt: 'improved whole profile' }, + }) }) - it('forwards evaluation controls to the shared self-improvement loop', async () => { - const progressKinds: string[] = [] - let provenanceCalls = 0 - let decisionCalls = 0 - const result = await improve(promptProfile(), [{ finding: 'prompt needs improvement' }], { - surface: 'prompt', - scenarios, - judge: improvementJudge, - agent: stubAgent, - generator: { - kind: 'stub-controls', - propose: async () => [ - { surface: 'improved prompt', label: 'candidate', rationale: 'test candidate' }, - ], + it('optimizes caller-defined profile components without dropping component state', async () => { + let observedBaseline: MutableSurface | undefined + const profile: AgentProfile = { + name: 'fixture-agent', + prompt: { systemPrompt: 'baseline' }, + tools: { search: false }, + } + const winner: MutableSurface = { + kind: 'components', + components: { + prompt: 'improved prompt', + tools: '{"search":true}', }, - promotionGate: { - name: 'test-hold', - decide: async () => { - decisionCalls += 1 - return { decision: 'hold', reasons: ['test hold'], contributingGates: [] } - }, + } + const result = await improve(profile, { + ...methodOptions( + fixedMethod(winner, (input) => { + observedBaseline = input.baselineSurface + }), + ), + surface: 'agent-profile', + profileComponents: { + read: (current) => ({ + prompt: current.prompt?.systemPrompt ?? '', + tools: JSON.stringify(current.tools ?? {}), + }), + apply: (current, components) => ({ + ...current, + prompt: { ...current.prompt, systemPrompt: components.prompt }, + tools: JSON.parse(components.tools ?? '{}') as Record, + }), }, - onProgress: (event) => progressKinds.push(event.kind), - onProvenance: () => { - provenanceCalls += 1 + agent: async (surface, scenario, ctx) => { + const text = + typeof surface === 'object' && surface.kind === 'components' + ? (surface.components.prompt ?? '') + : String(surface) + return paidText(text, scenario, ctx) }, - expectUsage: 'assert', - captureSource: 'eval-run', - autoOnPromote: 'none', - budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, }) - expect(result.decision).toBe('hold') - expect(decisionCalls).toBe(1) - expect(provenanceCalls).toBe(1) - expect(progressKinds).toContain('baseline.completed') - expect(progressKinds).toContain('gate.decided') + expect(observedBaseline).toEqual({ + kind: 'components', + components: { + prompt: 'baseline', + tools: '{"search":false}', + }, + }) + expect(result.candidate.profile).toMatchObject({ + prompt: { systemPrompt: 'improved prompt' }, + tools: { search: true }, + }) + expect(profile).toMatchObject({ + prompt: { systemPrompt: 'baseline' }, + tools: { search: false }, + }) }) - it('a surface with no zero-config default still fails loud with ConfigError', async () => { - // Prompt, skills, and memory have defaults; config surfaces require a - // caller-supplied generator. This is the - // designed boundary the proposer migration must NOT erase. - const configSurfaces: ImproveSurface[] = ['tools', 'mcp', 'hooks', 'subagents', 'agent-profile'] - for (const surface of configSurfaces) { - await expect( - improve(promptProfile(), [], { surface, gate: 'none', scenarios, judge, agent: stubAgent }), - ).rejects.toBeInstanceOf(ConfigError) - } + it('rejects component candidates that add or remove profile component names', async () => { + await expect( + improve(promptProfile(), { + ...methodOptions( + fixedMethod({ + kind: 'components', + components: { prompt: 'improved prompt' }, + }), + ), + surface: 'agent-profile', + profileComponents: { + read: () => ({ prompt: 'baseline', policy: '{}' }), + apply: (current, components) => ({ + ...current, + prompt: { ...current.prompt, systemPrompt: components.prompt }, + }), + }, + }), + ).rejects.toThrow(/preserve the exact component names/) }) - it.each([ - 'text', - 'external-code', - ] as const)("surface 'code' rejects a top-level proposer before it can return $surfaceKind", async (surfaceKind) => { - let proposerCalls = 0 - const externalCode: CodeSurface = { - kind: 'code', - worktreeRef: '/tmp/externally-owned-code-surface', - baseRef: 'main', - baseCommit: 'a'.repeat(40), - baseTree: 'b'.repeat(40), - candidateCommit: 'c'.repeat(40), - candidateTree: 'd'.repeat(40), - patch: { - format: 'git-diff-binary', - sha256: `sha256:${'e'.repeat(64)}`, - byteLength: 1, - }, - } - const externalProposer: SurfaceProposer = { - kind: 'externally-owned-code-test', - async propose() { - proposerCalls += 1 - return [ - { - surface: surfaceKind === 'text' ? 'not-a-code-surface' : externalCode, - label: 'unsafe external result', - rationale: 'exercise code ownership validation', - }, - ] - }, - } - + it('rejects a profile component adapter that does not apply the measured winner', async () => { await expect( - improve(promptProfile(), [], { - surface: 'code', - scenarios, - judge, - agent: stubAgent, - generator: externalProposer, - code: { repoRoot: '/tmp/must-not-be-opened' }, - budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, + improve(promptProfile(), { + ...methodOptions( + fixedMethod({ + kind: 'components', + components: { prompt: 'improved prompt' }, + }), + ), + surface: 'agent-profile', + profileComponents: { + read: (current) => ({ prompt: current.prompt?.systemPrompt ?? '' }), + apply: (current) => ({ ...current }), + }, + agent: async (surface, scenario, ctx) => + paidText( + typeof surface === 'object' && surface.kind === 'components' + ? (surface.components.prompt ?? '') + : String(surface), + scenario, + ctx, + ), }), - ).rejects.toThrow(/forbids opts\.generator/) - expect(proposerCalls).toBe(0) + ).rejects.toThrow(/round-trip every winning component/) }) - it('the default generation distiller feeds real failures to the next proposal round', async () => { - // Judge fails scenario 'b' with a distinctive reason; everything else is perfect. - const failingJudge: JudgeConfig<{ text: string }, Scenario> = { - name: 'distiller-judge', - dimensions: [{ key: 'q', description: 'fixture quality' }], - score: ({ scenario }) => - scenario.id === 'b' - ? { dimensions: { q: 0 }, composite: 0, notes: 'tour is not a permutation of 0..8' } - : { dimensions: { q: 1 }, composite: 1, notes: 'ok' }, - } - // Proposer stub records the findings it is handed each generation. - const findingsSeen: unknown[][] = [] - const stubProposer = { - kind: 'stub-recorder', - async propose(ctx: { findings: unknown[]; populationSize: number }) { - findingsSeen.push(ctx.findings) - return [{ surface: `candidate-${findingsSeen.length}`, label: 'stub', rationale: 'stub' }] - }, - } - const result = await improve(promptProfile(), [{ seed: 'static-seed-finding' }], { - surface: 'prompt', - scenarios, - judge: failingJudge, - agent: stubAgent, - generator: stubProposer as never, - budget: { generations: 2, populationSize: 1, holdoutFraction: 0.25 }, - }) - expect(typeof result.decision).toBe('string') - expect(findingsSeen.length).toBeGreaterThanOrEqual(2) - // Current selfImprove analyzes the baseline before generation 1, so every - // proposal round starts from measured failures instead of wasting a round - // on the static seed. - for (const seen of findingsSeen) { - const round = JSON.stringify(seen) - expect(round).toContain('"scenario":"b"') - expect(round).toContain('not a permutation') - } - // The digest is TYPED on the wire: real AnalystFinding envelopes, not - // ad-hoc {scenario, composite} objects a consumer must down-cast. - const { isAnalystFinding } = await import('./findings') - expect(findingsSeen[1]!.length).toBeGreaterThanOrEqual(1) - expect(findingsSeen[1]!.every(isAnalystFinding)).toBe(true) + it('rejects an invalid method and malformed profile output', async () => { + await expect( + improve(promptProfile(), { + ...methodOptions(null as never), + method: null as never, + }), + ).rejects.toBeInstanceOf(ConfigError) + + await expect( + improve(promptProfile(), { + ...methodOptions(fixedMethod('not json')), + surface: 'agent-profile', + }), + ).rejects.toThrow(/not valid JSON/) }) - it('a real runDir defaults analyzeGeneration to the RAW-TRACE distiller', async () => { - const { mkdtempSync, rmSync } = await import('node:fs') - const { tmpdir } = await import('node:os') - const { join } = await import('node:path') - const runDir = mkdtempSync(join(tmpdir(), 'improve-rawtrace-')) - const failingJudge: JudgeConfig<{ text: string }, Scenario> = { - name: 'failing-judge', - dimensions: [{ key: 'q', description: 'fixture quality' }], - score: () => ({ dimensions: { q: 0 }, composite: 0, notes: 'always failing' }), - } - const findingsSeen: unknown[][] = [] - const stubProposer = { - kind: 'stub-recorder', - async propose(ctx: { findings: unknown[]; populationSize: number }) { - findingsSeen.push(ctx.findings) - return [{ surface: `candidate-${findingsSeen.length}`, label: 'stub', rationale: 'stub' }] - }, - } - try { - await improve(promptProfile(), [], { - surface: 'prompt', - scenarios, - judge: failingJudge, - agent: stubAgent, - generator: stubProposer as never, - runDir, - budget: { generations: 2, populationSize: 1, holdoutFraction: 0.25 }, - }) - // The durable-run default is rawTraceDistiller: a later round's findings - // are its raw-trace-context envelopes (paths into the recorded traces), - // NOT the distilled failure digest. - const later = findingsSeen.at(-1)! - expect(JSON.stringify(later)).toContain('raw-trace-context') - const { isAnalystFinding } = await import('./findings') - expect(later.every(isAnalystFinding)).toBe(true) - } finally { - rmSync(runDir, { recursive: true, force: true }) - } + it('requires exact resource identity for skills and curated memory', async () => { + await expect( + improve( + { + name: 'fixture-agent', + resources: { + failOnError: true, + skills: [{ kind: 'inline', name: 'review', content: 'baseline skill' }], + }, + }, + { + ...methodOptions(fixedMethod('improved skill')), + surface: 'skills', + }, + ), + ).rejects.toThrow(/requires opts\.skills\.resourceName/) + + await expect( + improve(promptProfile(), { + ...methodOptions(fixedMethod('improved memory')), + surface: 'memory', + }), + ).rejects.toThrow(/requires profile\.resources\.failOnError/) }) +}) - it('the distiller keeps traceback-sized notes intact and clips at the 1500/500 caps', async () => { - // A realistic executable-judge note (~1000 chars) must survive whole; a - // runaway note is clipped to exactly 1500; a cell error is clipped to 500. - const intactNote = `Traceback (most recent call last):\n${' assert tour == expected\n'.repeat(38)}` - expect(intactNote.length).toBeGreaterThan(900) - expect(intactNote.length).toBeLessThan(1500) - const runawayNote = 'n'.repeat(1600) - const longError = 'e'.repeat(800) - const cappingJudge: JudgeConfig<{ text: string }, Scenario> = { - name: 'capping-judge', - dimensions: [{ key: 'q', description: 'fixture quality' }], - score: ({ scenario }) => { - if (scenario.id === 'a') return { dimensions: { q: 0 }, composite: 0, notes: intactNote } - if (scenario.id === 'b') return { dimensions: { q: 0 }, composite: 0, notes: runawayNote } - return { dimensions: { q: 1 }, composite: 1, notes: 'ok' } - }, - } - const findingsSeen: unknown[][] = [] - const stubProposer = { - kind: 'stub-recorder', - async propose(ctx: { findings: unknown[]; populationSize: number }) { - findingsSeen.push(ctx.findings) - return [{ surface: `candidate-${findingsSeen.length}`, label: 'stub', rationale: 'stub' }] - }, - } - const failingAgent = async (surface: unknown, scenario: Scenario, ctx: DispatchContext) => { - // The baseline must stay complete (an incomplete incumbent is refused), - // so the error lands on a generation-1 candidate cell instead. - if (scenario.id === 'c' && typeof surface === 'string' && surface.startsWith('candidate-')) { - throw new Error(longError) - } - return stubAgent(surface, scenario, ctx) - } - await improve(promptProfile(), [], { - surface: 'prompt', - scenarios, - judge: cappingJudge, - agent: failingAgent, - generator: stubProposer as never, - budget: { generations: 2, populationSize: 1, holdoutFraction: 0.25 }, +function createRepo(prefix: string): { + repoRoot: string + git(args: string[]): string + cleanup(): void +} { + const repoRoot = mkdtempSync(join(tmpdir(), prefix)) + const git = (args: string[]) => + execFileSync('git', args, { + cwd: repoRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], }) - expect(findingsSeen.length).toBeGreaterThanOrEqual(1) - // Rows are typed AnalystFinding envelopes; the distilled cell fields ride - // `metadata` so consumers keep one wire shape (see generationFailureDistiller). - const rows = findingsSeen.flat() as Array<{ - metadata?: { scenario?: string; notes?: string; error?: string } - }> - const intact = rows.find((row) => row.metadata?.scenario === 'a') - expect(intact?.metadata?.notes).toBe(intactNote) // below the cap ⇒ untouched - const clipped = rows.find((row) => row.metadata?.scenario === 'b') - expect(clipped?.metadata?.notes).toBe('n'.repeat(1500)) // at the cap ⇒ exactly 1500 - const errored = rows.find((row) => row.metadata?.scenario === 'c') - expect(errored?.metadata?.error).toBeDefined() - expect(errored?.metadata?.error).toContain('e'.repeat(400)) - expect(errored?.metadata?.error?.length).toBeLessThanOrEqual(500) - }) + git(['init', '-q', '-b', 'main']) + git(['config', 'user.email', 'improve@test.local']) + git(['config', 'user.name', 'improve-test']) + writeFileSync(join(repoRoot, 'module.txt'), 'baseline contents\n') + git(['add', 'module.txt']) + git(['commit', '-qm', 'baseline']) + return { + repoRoot, + git, + cleanup: () => rmSync(repoRoot, { recursive: true, force: true }), + } +} - it("surface 'code' + opts.code assembles the worktree pipeline and measures a candidate", async () => { - const { execSync } = await import('node:child_process') - const { mkdtempSync, readFileSync, rmSync, writeFileSync } = await import('node:fs') - const { tmpdir } = await import('node:os') - const { join } = await import('node:path') - const repoRoot = mkdtempSync(join(tmpdir(), 'improve-code-')) - const git = (cmd: string) => execSync(`git ${cmd}`, { cwd: repoRoot, stdio: 'pipe' }) - try { - git('init -q -b main') - git('config user.email improve@test.local') - git('config user.name improve-test') - writeFileSync(join(repoRoot, 'module.txt'), 'baseline contents\n') - git('add module.txt') - git('commit -qm baseline') +function worktreeCount(git: (args: string[]) => string): number { + return git(['worktree', 'list', '--porcelain']).match(/^worktree /gm)?.length ?? 0 +} - // Byte-producer stub via the designed test seam: writes a change into the - // candidate worktree; the driver finalizes it into a CodeSurface. +async function paidCodeText( + surface: MutableSurface, + _scenario: TestScenario, + ctx: DispatchContext, +): Promise { + if (typeof surface !== 'object' || surface === null || !('worktreeRef' in surface)) { + throw new Error('expected a code surface') + } + return paidText( + readFileSync(join(String(surface.worktreeRef), 'module.txt'), 'utf8'), + _scenario, + ctx, + ) +} + +describe('improve code execution', () => { + it('runs candidate generation in isolated worktrees and retains only the winner', async () => { + const repo = createRepo('improve-code-') + try { let generatorCalls = 0 - const startingContents: string[] = [] - const measured: unknown[] = [] - const result = await improve(promptProfile(), [{ finding: 'module.txt is stale' }], { + const result = await improve(promptProfile(), { surface: 'code', - scenarios, + findings: [{ claim: 'module.txt is stale' }], + scenarios: allScenarios, judge: improvementJudge, - agent: async (surface, _scenario, ctx) => { - return paidStubCall(ctx, 'stub-agent', () => { - measured.push(surface) - if (typeof surface !== 'object' || surface === null || !('worktreeRef' in surface)) { - throw new Error('expected code surface') - } - return { - text: readFileSync(join(String(surface.worktreeRef), 'module.txt'), 'utf8'), - } - }) - }, + agent: paidCodeText, code: { - repoRoot, + repoRoot: repo.repoRoot, generator: { - kind: 'stub', - async generate({ worktreePath }) { + kind: 'test-generator', + async generate({ worktreePath }: { worktreePath: string }) { generatorCalls += 1 - const current = readFileSync(join(worktreePath, 'module.txt'), 'utf8') - startingContents.push(current) - writeFileSync( - join(worktreePath, 'module.txt'), - `${current}improved ${generatorCalls}\n`, - ) - return { applied: true, summary: 'stub improvement' } + writeFileSync(join(worktreePath, 'module.txt'), 'improved contents\n') + return { applied: true, summary: 'updated module' } }, }, }, promotionGate: { - name: 'test-hold', + name: 'retain-test-winner', decide: async () => ({ - decision: 'hold', - reasons: ['exercise non-promoted candidate retention'], + decision: 'hold' as const, + reasons: ['exercise detached candidate retention'], contributingGates: [], }), }, - budget: { generations: 2, populationSize: 1, holdoutFraction: 0.25 }, + budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, }) - // The facade assembled a real proposer: the stub produced candidates, the - // loop measured them (code surfaces reached the agent), and the gate decided. - expect(generatorCalls).toBe(2) - expect(startingContents).toEqual(['baseline contents\n', 'baseline contents\nimproved 1\n']) - expect(result.decision).toBe('hold') - const codeSurfaces = measured.filter( - (m) => - typeof m === 'object' && m !== null && 'worktreeRef' in (m as Record), - ) - expect(codeSurfaces.length).toBeGreaterThanOrEqual(2) - expect( - codeSurfaces.some((surface) => - String((surface as { worktreeRef: string }).worktreeRef).includes('incumbent-baseline'), - ), - ).toBe(true) + expect(generatorCalls).toBe(1) + expect(result.mode).toBe('code') expect(result.candidate.profile).toBeUndefined() - if (typeof result.candidate.value === 'string') { - throw new Error('expected code winner') + expect(typeof result.candidate.value).toBe('object') + if (typeof result.candidate.value === 'string' || result.candidate.value.kind !== 'code') { + throw new Error('expected code surface') } expect(readFileSync(join(result.candidate.value.worktreeRef, 'module.txt'), 'utf8')).toBe( - 'baseline contents\nimproved 1\n', + 'improved contents\n', ) - expect(String(git('worktree list --porcelain')).match(/^worktree /gm)).toHaveLength(2) + expect(worktreeCount(repo.git)).toBe(2) await result.dispose() await result.dispose() - expect(String(git('worktree list --porcelain')).match(/^worktree /gm)).toHaveLength(1) + expect(worktreeCount(repo.git)).toBe(1) } finally { - rmSync(repoRoot, { recursive: true, force: true }) + repo.cleanup() } }) - it("surface 'code' keeps the baseline worktree when a baseline-only run returns it", async () => { - const { execSync } = await import('node:child_process') - const { mkdtempSync, readFileSync, rmSync, writeFileSync } = await import('node:fs') - const { tmpdir } = await import('node:os') - const { join } = await import('node:path') - const repoRoot = mkdtempSync(join(tmpdir(), 'improve-code-baseline-')) - const git = (cmd: string) => execSync(`git ${cmd}`, { cwd: repoRoot, stdio: 'pipe' }) + it('retains and disposes the incumbent for a baseline-only code run', async () => { + const repo = createRepo('improve-code-baseline-') try { - git('init -q -b main') - git('config user.email improve@test.local') - git('config user.name improve-test') - writeFileSync(join(repoRoot, 'module.txt'), 'baseline contents\n') - git('add module.txt') - git('commit -qm baseline') - - const result = await improve(promptProfile(), [], { + const result = await improve(promptProfile(), { surface: 'code', gate: 'none', - scenarios, - judge, - agent: async (surface, _scenario, ctx) => { - return paidStubCall(ctx, 'stub-agent', () => { - if (typeof surface !== 'object' || surface === null || !('worktreeRef' in surface)) { - throw new Error('expected code surface') - } - return { - text: readFileSync(join(String(surface.worktreeRef), 'module.txt'), 'utf8'), - } - }) - }, + scenarios: allScenarios, + judge: improvementJudge, + agent: paidCodeText, code: { - repoRoot, + repoRoot: repo.repoRoot, generator: { kind: 'must-not-run', async generate() { - throw new Error('baseline-only run must not call the generator') + throw new Error('baseline-only run must not generate') }, }, }, }) - if (typeof result.candidate.value === 'string') { - throw new Error('expected code winner') - } expect(result.decision).toBe('hold') + if (typeof result.candidate.value === 'string' || result.candidate.value.kind !== 'code') { + throw new Error('expected code surface') + } expect(readFileSync(join(result.candidate.value.worktreeRef, 'module.txt'), 'utf8')).toBe( 'baseline contents\n', ) - expect(String(git('worktree list --porcelain')).match(/^worktree /gm)).toHaveLength(2) + expect(worktreeCount(repo.git)).toBe(2) await result.dispose() - await result.dispose() - expect(String(git('worktree list --porcelain')).match(/^worktree /gm)).toHaveLength(1) + expect(worktreeCount(repo.git)).toBe(1) } finally { - rmSync(repoRoot, { recursive: true, force: true }) + repo.cleanup() } }) - it.each([ - { mode: 'transient' as const, expectedWorktrees: 1, expectedErrors: 1 }, - { mode: 'persistent' as const, expectedWorktrees: 3, expectedErrors: 2 }, - ])('surface code retries $mode cleanup before rejecting without a result', async (fixture) => { - const { execSync } = await import('node:child_process') - const { mkdtempSync, readFileSync, rmSync, writeFileSync } = await import('node:fs') - const { tmpdir } = await import('node:os') - const { join } = await import('node:path') - const repoRoot = mkdtempSync(join(tmpdir(), `improve-code-${fixture.mode}-cleanup-`)) - const git = (cmd: string) => execSync(`git ${cmd}`, { cwd: repoRoot, stdio: 'pipe' }) + it('cleans every worktree when candidate generation fails', async () => { + const repo = createRepo('improve-code-reject-') try { - git('init -q -b main') - git('config user.email improve@test.local') - git('config user.name improve-test') - writeFileSync(join(repoRoot, 'module.txt'), 'baseline contents\n') - git('add module.txt') - git('commit -qm baseline') - - const realWorktree = gitWorktreeAdapter({ repoRoot }) - let discardAttempts = 0 - const flakyWorktree: WorktreeAdapter = { - ...realWorktree, - async discard(worktree: Worktree) { - discardAttempts += 1 - if (fixture.mode === 'persistent' || discardAttempts === 1) { - throw new Error(`${fixture.mode} discard failure ${discardAttempts}`) - } - await realWorktree.discard(worktree) - }, - } - - let caught: unknown - try { - await improve(promptProfile(), [], { + await expect( + improve(promptProfile(), { surface: 'code', - scenarios, + scenarios: allScenarios, judge: improvementJudge, - agent: async (surface, _scenario, ctx) => { - return paidStubCall(ctx, 'stub-agent', () => { - if (typeof surface !== 'object' || surface === null || !('worktreeRef' in surface)) { - throw new Error('expected code surface') - } - return { - text: readFileSync(join(String(surface.worktreeRef), 'module.txt'), 'utf8'), - } - }) - }, + agent: paidCodeText, code: { - repoRoot, - worktree: flakyWorktree, + repoRoot: repo.repoRoot, generator: { - kind: 'cleanup-test', - async generate({ worktreePath }) { - writeFileSync(join(worktreePath, 'module.txt'), 'improved contents\n') - return { applied: true, summary: 'improve cleanup fixture' } - }, - }, - }, - promotionGate: { - name: 'test-hold', - decide: async () => ({ - decision: 'hold', - reasons: ['exercise cleanup recovery'], - contributingGates: [], - }), - }, - budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, - }) - } catch (cause) { - caught = cause - } - - const aggregate = - caught instanceof AggregateError - ? caught - : (caught as { cause?: unknown } | undefined)?.cause - expect(aggregate).toBeInstanceOf(AggregateError) - expect((aggregate as AggregateError).errors).toHaveLength(fixture.expectedErrors) - expect(discardAttempts).toBe(3) - expect(String(git('worktree list --porcelain')).match(/^worktree /gm)).toHaveLength( - fixture.expectedWorktrees, - ) - } finally { - rmSync(repoRoot, { recursive: true, force: true }) - } - }) - - it.each([ - { - mode: 'transient' as const, - expectedWorktrees: 1, - expectedErrors: 2, - expectedDiscardAttempts: 3, - }, - { - mode: 'persistent' as const, - expectedWorktrees: 3, - expectedErrors: 3, - expectedDiscardAttempts: 6, - }, - ])('surface code retries $mode cleanup when selfImprove rejects', async (fixture) => { - const { execSync } = await import('node:child_process') - const { mkdtempSync, readFileSync, rmSync, writeFileSync } = await import('node:fs') - const { tmpdir } = await import('node:os') - const { join } = await import('node:path') - const repoRoot = mkdtempSync(join(tmpdir(), `improve-code-${fixture.mode}-rejection-`)) - const git = (cmd: string) => execSync(`git ${cmd}`, { cwd: repoRoot, stdio: 'pipe' }) - try { - git('init -q -b main') - git('config user.email improve@test.local') - git('config user.name improve-test') - writeFileSync(join(repoRoot, 'module.txt'), 'baseline contents\n') - git('add module.txt') - git('commit -qm baseline') - - const realWorktree = gitWorktreeAdapter({ repoRoot }) - let discardAttempts = 0 - const flakyWorktree: WorktreeAdapter = { - ...realWorktree, - async discard(worktree: Worktree) { - discardAttempts += 1 - if (fixture.mode === 'persistent' || discardAttempts === 1) { - throw new Error(`${fixture.mode} discard failure ${discardAttempts}`) - } - await realWorktree.discard(worktree) - }, - } - let caught: unknown - try { - await improve(promptProfile(), [], { - surface: 'code', - scenarios, - judge, - agent: async (surface, _scenario, ctx) => { - return paidStubCall(ctx, 'stub-agent', () => { - if (typeof surface !== 'object' || surface === null || !('worktreeRef' in surface)) { - throw new Error('expected code surface') - } - return { - text: readFileSync(join(String(surface.worktreeRef), 'module.txt'), 'utf8'), - } - }) - }, - code: { - repoRoot, - worktree: flakyWorktree, - generator: { - kind: 'rejecting-test', + kind: 'rejecting-generator', async generate() { throw new Error('candidate generation failed') }, }, }, budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, - }) - } catch (cause) { - caught = cause - } - - const aggregate = - caught instanceof AggregateError - ? caught - : (caught as { cause?: unknown } | undefined)?.cause - expect(aggregate).toBeInstanceOf(AggregateError) - expect((aggregate as AggregateError).errors).toHaveLength(fixture.expectedErrors) - expect(discardAttempts).toBe(fixture.expectedDiscardAttempts) - expect(String(git('worktree list --porcelain')).match(/^worktree /gm)).toHaveLength( - fixture.expectedWorktrees, - ) + }), + ).rejects.toThrow(/candidate generation failed/) + expect(worktreeCount(repo.git)).toBe(1) } finally { - rmSync(repoRoot, { recursive: true, force: true }) + repo.cleanup() } }) - it.each([ - { mode: 'transient' as const, expectedWorktrees: 1, expectedErrors: 2 }, - { mode: 'persistent' as const, expectedWorktrees: 2, expectedErrors: 3 }, - ])('surface code retries $mode cleanup when baseline finalization rejects', async (fixture) => { - const { execSync } = await import('node:child_process') - const { mkdtempSync, rmSync, writeFileSync } = await import('node:fs') - const { tmpdir } = await import('node:os') - const { join } = await import('node:path') - const repoRoot = mkdtempSync(join(tmpdir(), `improve-code-${fixture.mode}-finalize-`)) - const git = (cmd: string) => execSync(`git ${cmd}`, { cwd: repoRoot, stdio: 'pipe' }) + it('cleans the incumbent when baseline finalization fails', async () => { + const repo = createRepo('improve-code-finalize-') try { - git('init -q -b main') - git('config user.email improve@test.local') - git('config user.name improve-test') - writeFileSync(join(repoRoot, 'module.txt'), 'baseline contents\n') - git('add module.txt') - git('commit -qm baseline') - - const realWorktree = gitWorktreeAdapter({ repoRoot }) - let discardAttempts = 0 + const realWorktree = gitWorktreeAdapter({ repoRoot: repo.repoRoot }) + let discarded = 0 const rejectingWorktree: WorktreeAdapter = { ...realWorktree, async finalize() { throw new Error('baseline finalization failed') }, async discard(worktree: Worktree) { - discardAttempts += 1 - if (fixture.mode === 'persistent' || discardAttempts === 1) { - throw new Error(`${fixture.mode} discard failure ${discardAttempts}`) - } + discarded += 1 await realWorktree.discard(worktree) }, } - let caught: unknown - try { - await improve(promptProfile(), [], { + await expect( + improve(promptProfile(), { surface: 'code', - scenarios, - judge, - agent: stubAgent, + scenarios: allScenarios, + judge: improvementJudge, + agent: paidCodeText, code: { - repoRoot, + repoRoot: repo.repoRoot, worktree: rejectingWorktree, generator: { kind: 'unused', async generate() { - throw new Error('must not generate before baseline finalization') + throw new Error('must not generate') }, }, }, - budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, - }) - } catch (cause) { - caught = cause - } - - expect(caught).toBeInstanceOf(AggregateError) - expect((caught as AggregateError).errors).toHaveLength(fixture.expectedErrors) - expect(discardAttempts).toBe(2) - expect(String(git('worktree list --porcelain')).match(/^worktree /gm)).toHaveLength( - fixture.expectedWorktrees, - ) + }), + ).rejects.toThrow(/baseline finalization failed/) + expect(discarded).toBe(1) + expect(worktreeCount(repo.git)).toBe(1) } finally { - rmSync(repoRoot, { recursive: true, force: true }) + repo.cleanup() } }) }) diff --git a/src/improvement/improve.ts b/src/improvement/improve.ts index 02230299..6cf9c3fe 100644 --- a/src/improvement/improve.ts +++ b/src/improvement/improve.ts @@ -1,47 +1,23 @@ /** + * `improve` runs one complete optimization method against an exact profile + * surface. Runtime extracts and materializes the profile value; agent-eval owns + * optimization, disjoint data partitions, final-test scoring, and uncertainty. * - * `improve` — the ONE public, surface-pluggable RSI verb. - * - * A thin facade over agent-eval's `selfImprove` (the held-out-gated closed - * loop). It removes the two things a caller otherwise has to know to drive the - * loop by hand: WHICH `MutableSurface` of the profile is being optimized, and - * WHICH `SurfaceProposer` mutates that surface. You name a `surface`; the - * facade picks the matching default proposer, extracts the baseline surface from - * the profile, and runs `selfImprove`. It returns a frozen candidate and never - * changes the input profile or caller-owned state. - * - * - `surface: 'prompt'` → `gepaProposer` mutates `profile.prompt.systemPrompt`. - * - `surface: 'skills'` → `skillOptProposer` mutates one named inline skill. - * - `surface: 'memory'` → `memoryCurationProposer` curates the profile's - * additional instructions as bounded durable lessons. - * - `surface: 'rollout-policy'` → `rolloutPolicyProposer` mutates the - * inference-time `StructuralRolloutPolicy` dials ({ k, repairRounds, testgen }) - * persisted in `profile.extensions['structural-rollout']` — deterministic - * bounded neighbor enumeration; the held-out gate does the deciding. No-op - * (nothing proposed, nothing shipped) when the profile has no such extension. - * - `surface: 'agent-profile'` → caller-supplied proposer mutates the complete - * canonical AgentProfile JSON in one candidate. - * - `surface` ∈ {`tools`, `mcp`, `hooks`, `subagents`, `agent-profile`} → no zero-config default - * proposer exists (a code/config proposer needs caller-supplied wiring — a - * worktree repo root, a candidate generator, a serializer). The facade - * requires an explicit `opts.generator` for these and throws a `ConfigError` - * otherwise. This is a designed boundary, not a missing default: there is - * no safe value the facade could invent for those surfaces. Code instead - * requires `opts.code.repoRoot` and accepts only the runtime-owned - * `opts.code.generator` path so every isolated checkout can be released. - * - * Everything else (`scenarios`, `judge`, `agent`, `budget`, `llm`) passes - * straight through to `selfImprove`. + * Code is the sole exception. It uses Runtime's isolated git worktrees because + * checkout ownership and cleanup cannot cross a generic optimizer boundary. * * @experimental */ +import { randomUUID } from 'node:crypto' import { canonicalJson, makeFinding } from '@tangle-network/agent-eval' import { - gepaProposer, + type CompareOptimizationMethodsOptions, + campaignSplitDigest, + compareOptimizationMethods, gitWorktreeAdapter, - memoryCurationProposer, - skillOptProposer, + type OptimizationMethod, + type OptimizationMethodComparison, type Worktree, type WorktreeAdapter, } from '@tangle-network/agent-eval/campaign' @@ -50,7 +26,6 @@ import { type MutableSurface, type Scenario, type SelfImproveBudget, - type SelfImproveLlm, type SelfImproveOptions, type SelfImproveResult, type SurfaceProposer, @@ -60,11 +35,11 @@ import { type AgentProfile, type AgentProfileResourceRef, agentProfileSchema, + type Sha256Digest, } from '@tangle-network/agent-interface' -import { immutableCandidateValue } from '../candidate-execution/digest' +import { canonicalCandidateDigest, immutableCandidateValue } from '../candidate-execution/digest' import { ConfigError } from '../errors' import type { LocalHarness } from '../mcp/local-harness' -import { assertModelAllowed } from '../runtime/supervise/model-policy' import { agenticGenerator, type Verifier } from './agentic-generator' import { rethrowAfterCleanup } from './cleanup' import { @@ -76,7 +51,6 @@ import { rawTraceDistiller } from './raw-trace-distiller' import { applyRolloutPolicyToProfile, normalizeRolloutPolicy, - rolloutPolicyProposer, serializeRolloutPolicy, structuralRolloutPolicyFromProfile, } from './rollout-policy' @@ -98,67 +72,113 @@ export type ImproveSurface = | 'code' | 'rollout-policy' -export type ImproveOptions = Omit< +export type ImproveProfileSurface = Exclude + +export interface ImproveMethodContext { + /** Validated baseline profile. */ + readonly profile: Readonly + /** Exact profile coordinate being optimized. */ + readonly surface: ImproveProfileSurface + /** Exact bytes supplied to the optimization method. */ + readonly baselineSurface: MutableSurface + /** Findings produced before this search, if any. */ + readonly findings: readonly unknown[] +} + +/** Build a complete method after trace findings are available. */ +export type ImproveMethodFactory = ( + context: ImproveMethodContext, +) => OptimizationMethod + +export type ImproveMethodSource = + | OptimizationMethod + | ImproveMethodFactory + +/** Complete-method configuration for every non-code profile surface. */ +export type ImproveMethodOptions = Omit< + CompareOptimizationMethodsOptions, + 'baselineSurface' | 'dispatchWithSurface' | 'methods' | 'optimizationConcurrency' +> & { + /** Exact profile coordinate optimized by `method`. Default `'prompt'`. */ + surface?: ImproveProfileSurface + /** A complete optimizer or a factory that can incorporate current findings. */ + method: ImproveMethodSource + /** Runs one candidate surface on one scenario. */ + agent: ( + surface: MutableSurface, + scenario: TScenario, + ctx: Parameters< + CompareOptimizationMethodsOptions['dispatchWithSurface'] + >[2], + ) => Promise + /** Trace or analyst findings available to a method factory. */ + findings?: readonly unknown[] + /** Select the exact inline skill document for `surface: 'skills'`. */ + skills?: ImproveSkillsOptions + /** + * Map a profile to named text components and apply the winning components. + * Valid only with `surface: 'agent-profile'`. + */ + profileComponents?: ImproveProfileComponents + /** Ship only when the paired final-test interval is entirely above this lift. Default `0`. */ + minimumLift?: number +} + +/** Runtime-owned code search in isolated git worktrees. */ +export type ImproveCodeRunOptions = Omit< SelfImproveOptions, - 'analyzeGeneration' | 'baselineSurface' | 'findings' | 'gate' | 'proposer' + | 'analyzeGeneration' + | 'baselineSurface' + | 'budget' + | 'findings' + | 'gate' + | 'llm' + | 'method' + | 'mutationPrimitives' + | 'proposer' + | 'proposerTarget' + | 'selectionScenarios' > & { - /** Which profile lever to optimize. Default `'prompt'`. Selects the default - * generator + the baseline-surface extraction shape. */ - surface?: ImproveSurface - /** The `SurfaceProposer` that mutates a profile surface. When unset, the facade - * picks the default for prompt, skills, and memory; surfaces - * with no default REQUIRE this (fail-loud otherwise). Forbidden for code; - * use `code.generator` so the runtime owns candidate cleanup. */ - generator?: SurfaceProposer + surface: 'code' + /** Local code-search budget. Method-only selection controls do not apply. */ + budget?: Omit + /** Findings supplied to Runtime's code candidate driver. */ + findings?: readonly unknown[] /** Gate mode. `'holdout'` (default) runs the held-out promotion gate; * `'none'` is a baseline-only run (`budget.generations = 0`). */ gate?: 'holdout' | 'none' - /** Restrict the run to this subset of models. When set, the reflection model - * (`llm.model`, or the default when unset) must be a member, or `improve()` throws - * a `ConfigError` before the generator is built. Unset = unrestricted. */ - allowedModels?: readonly string[] - /** Per-generation findings producer passthrough (see selfImprove.analyzeGeneration). - * DEFAULT: with a real (non-`mem://`) `runDir`, the raw-trace distiller - * (`rawTraceDistiller`) — typed `AnalystFinding`s pointing the proposer at the - * prior generation's actual on-disk traces; for in-memory runs (no traces on - * disk to point at), the built-in failure distiller — the worst-scoring/errored - * cells distilled into typed `AnalystFinding`s for the NEXT proposal round. - * Pass your own producer to replace either; pass `null` to disable and keep the - * static `findings` all the way through. */ + /** Per-generation findings producer for Runtime's code search. + * Pass your own producer to replace the code-trace distiller; pass `null` + * to keep the static findings for every generation. */ analyzeGeneration?: SelfImproveOptions['analyzeGeneration'] | null - /** META-HARNESS mode: instead of the distilled findings, feed the proposer - * RAW-TRACE FILESYSTEM CONTEXT — the PATHS into the prior generation's - * real run traces under `runDir` (per-cell `spans.jsonl` event logs + - * `cached-result.json` scores + artifacts) plus a `grep`/`cat`-to-diagnose - * instruction — so the coding agent reads the actual failures itself rather than - * a pre-summary. Unset (default): raw-trace findings whenever the run is durable - * (a real `runDir` — that is where the traces live), the distilled failure digest - * otherwise; the `memory` surface always defaults to its curation distiller. - * `true` forces `rawTraceDistiller()` even for an in-memory run (it emits a loud - * warning finding instead of paths); `false` forces the digest distiller even - * with a real `runDir`. Ignored when `analyzeGeneration` is set explicitly - * (that wins) or is `null` (disabled). */ + /** Feed code candidates paths to prior raw traces instead of a failure digest. + * Defaults to true for durable runs and false for in-memory runs. */ rawTraceContext?: boolean - /** CODE-surface wiring: name `surface: 'code'`, point at a repo, and the - * facade assembles the whole candidate pipeline — an isolated incumbent plus git worktrees - * (`gitWorktreeAdapter`) driven by `improvementDriver` with the full agentic - * generator (a real coding harness edits each candidate worktree; a `verify` - * hook gates candidates before they are ever measured). Ignored when - * `opts.generator` is supplied. Required for every code run because a real - * repository and base ref are necessary to measure the incumbent. */ - code?: ImproveCodeOptions - /** Select the exact inline skill document to optimize. */ - skills?: ImproveSkillsOptions + /** Isolated repository and candidate generator settings. */ + code: ImproveCodeOptions /** Custom held-back-exam decision. The string `gate` above controls whether * the exam runs; this callback controls how its evidence decides promotion. */ promotionGate?: SelfImproveOptions['gate'] } +/** The canonical improvement API: complete methods for profiles, worktrees for code. */ +export type ImproveOptions = + | ImproveMethodOptions + | ImproveCodeRunOptions + export interface ImproveSkillsOptions { /** `name` of one inline entry in `profile.resources.skills`. */ resourceName: string } +/** Caller-owned mapping for optimizing several profile fields as one candidate. */ +export interface ImproveProfileComponents { + /** Extract the exact named text components optimized together. */ + read(profile: Readonly): Readonly> + /** Apply a complete winning component map to a detached profile. */ + apply(profile: Readonly, components: Readonly>): AgentProfile +} + export interface ImproveCodeOptions { /** Repo root candidate worktrees fork from. */ repoRoot: string @@ -190,63 +210,72 @@ export interface ImprovementCandidate { profile?: AgentProfile } -export interface ImproveResult { +/** Normalized spend reported for one Runtime improvement run. */ +export interface ImproveCost { + totalCostUsd: number + accountingComplete: boolean + incompleteReasons: string[] +} + +/** Optimizer ancestry sealed into downstream candidate experiments. */ +export interface ImproveLineage { + /** Upstream optimizer run when reported, otherwise this Runtime optimization invocation. */ + runId: string + /** Exact train-plus-selection scenario payloads exposed to candidate selection. */ + developmentSplitDigest: Sha256Digest +} + +interface ImproveResultBase { /** Frozen candidate only. Live state is changed through an approved activation. */ candidate: ImprovementCandidate - /** Held-out decision for this search result. */ - decision: SelfImproveResult['gateDecision'] - /** Held-out lift (`winner − baseline` composite). Absent iff - * `budget.holdout === 'deferred'` — no held-out measurement ran, so there - * is no lift to report (never a fabricated 0). */ + /** Final-test decision for this search result. */ + decision: SelfImproveResult['gateDecision'] + /** Final-test lift when one was measured. */ lift?: number - /** Full `selfImprove` result for advanced inspection. For code runs, - * `raw.winner.surface.worktreeRef` remains live after return whether the - * candidate passed or held; call `dispose()` after consuming it. */ - raw: SelfImproveResult + /** Paired final-test confidence interval for method-based profile runs. */ + liftInterval?: { low: number; high: number } + /** Full search and final-test spend. */ + cost: ImproveCost + /** Full wall-clock duration. */ + durationMs: number + /** Optimizer ancestry used when sealing a candidate experiment. */ + lineage: ImproveLineage + /** Number of generations explored by Runtime's code path. */ + generationsExplored?: number /** Release resources owned by this result. Idempotent; currently disposes * the returned code worktree and is a no-op for profile-only surfaces. */ dispose(): Promise } -/** Default model id for the reflective drivers when `llm.model` is unset — a model the Tangle - * router actually serves (callers should pass their own `llm.model`). */ -const defaultReflectionModel = 'deepseek-v4-flash' - -/** The reflective proposers (`gepaProposer`/`skillOptProposer`) take a full - * `LlmClientOptions`; `SelfImproveLlm` is the thin user-facing subset. */ -function llmClientOptions(llm: SelfImproveLlm | undefined): { baseUrl?: string; apiKey?: string } { - return { baseUrl: llm?.baseUrl, apiKey: llm?.apiKey } +export interface ImproveMethodResult extends ImproveResultBase { + mode: 'method' + method: string + /** External optimizer package and resumable run identity, when reported. */ + provenance?: OptimizationMethodComparison['best']['provenance'] + decision: 'ship' | 'hold' + lift: number + liftInterval: { low: number; high: number } + raw: OptimizationMethodComparison } -/** The default proposer for a surface, or `undefined` when the surface has no - * zero-config default (the caller must supply `opts.generator`). */ -function defaultGeneratorFor( - surface: ImproveSurface, - llm: SelfImproveLlm | undefined, -): SurfaceProposer | undefined { - const model = llm?.model ?? defaultReflectionModel - switch (surface) { - case 'prompt': - return gepaProposer({ llm: llmClientOptions(llm), model, target: 'agent system prompt' }) - case 'skills': - return skillOptProposer({ llm: llmClientOptions(llm), model, target: 'agent skill document' }) - case 'memory': - return memoryCurationProposer() - case 'rollout-policy': - // Deterministic bounded enumeration — no LLM, so `llm` is unused here. - return rolloutPolicyProposer() - default: - return undefined - } +export interface ImproveCodeResult + extends ImproveResultBase { + mode: 'code' + raw: SelfImproveResult } -/** Extract the baseline surface a driver mutates from the profile field that - * backs `surface`. Prompt, skills, and memory are text surfaces; config - * surfaces serialize the matching profile record. */ +export type ImproveResult = + | ImproveMethodResult + | ImproveCodeResult + +/** Extract the baseline optimized by a method. Prompt, skills, and memory are + * text; config fields are JSON; a caller may map the full profile to named + * components explicitly. */ function baselineSurfaceFor( profile: AgentProfile, surface: ImproveSurface, skills?: ImproveSkillsOptions, + profileComponents?: ImproveProfileComponents, ): MutableSurface { switch (surface) { case 'prompt': @@ -254,23 +283,27 @@ function baselineSurfaceFor( case 'skills': return inlineSkill(profile, skills).content case 'tools': - return JSON.stringify(profile.tools ?? {}) + return canonicalJson(profile.tools ?? {}) case 'mcp': - return JSON.stringify(profile.mcp ?? {}) + return canonicalJson(profile.mcp ?? {}) case 'hooks': - return JSON.stringify(profile.hooks ?? {}) + return canonicalJson(profile.hooks ?? {}) case 'subagents': - return JSON.stringify(profile.subagents ?? {}) + return canonicalJson(profile.subagents ?? {}) case 'agent-profile': - return canonicalJson(profile) + return profileComponents + ? componentSurface(profileComponents.read(profile), 'profileComponents.read') + : canonicalJson(profile) case 'memory': return profileInstructions(profile) case 'rollout-policy': { - // Empty surface when the profile never opted into structural rollout: the - // proposer reads it as "propose nothing", so the loop runs baseline-only and - // holds — tuning dials nothing consumes would ship dead config. const policy = structuralRolloutPolicyFromProfile(profile) - return policy ? serializeRolloutPolicy(policy) : '' + if (!policy) { + throw new ConfigError( + "improve(): surface 'rollout-policy' requires an existing structural rollout policy", + ) + } + return serializeRolloutPolicy(policy) } case 'code': throw new ConfigError( @@ -366,34 +399,15 @@ function generationFailureDistiller( } } -/** Memory accumulates durable lessons, so keep the caller's seed findings while - * adding fresh judge failures. Curator proposers consume `claim`; the generic - * distiller retains the richer diagnostic fields for reflective proposers. */ -function memoryGenerationDistiller( - staticFindings: unknown[], -): NonNullable['analyzeGeneration']> { - const distillFailures = generationFailureDistiller(staticFindings) - return async (input) => { - const fresh = await distillFailures(input) - return fresh === staticFindings ? staticFindings : [...staticFindings, ...fresh] - } -} - -/** The default `analyzeGeneration` when the caller passed none: raw-trace context - * whenever the run is durable (a real `runDir` is where the traces live — see - * `rawTraceDistiller`), the failure digest for in-memory runs, with - * `rawTraceContext` as the explicit override in either direction. The `memory` - * surface keeps its curation distiller (its proposer consumes `claim` findings, - * not trace paths) unless `rawTraceContext: true` is explicit. */ +/** Default code-run analysis: raw trace paths for durable runs, otherwise a + * bounded digest of failed cells. */ function defaultDistillerFor( - opts: ImproveOptions, + opts: ImproveCodeRunOptions, findings: unknown[], ): NonNullable['analyzeGeneration']> { - const surface = opts.surface ?? 'prompt' const durableRun = opts.runDir !== undefined && !opts.runDir.startsWith('mem://') - const useRawTraces = opts.rawTraceContext ?? (surface === 'memory' ? false : durableRun) + const useRawTraces = opts.rawTraceContext ?? durableRun if (useRawTraces) return rawTraceDistiller({ fallbackFindings: findings }) - if (surface === 'memory') return memoryGenerationDistiller(findings) return generationFailureDistiller(findings) } @@ -419,6 +433,44 @@ function isCodeSurface(surface: MutableSurface | undefined): surface is CodeSurf return typeof surface === 'object' && surface !== null && surface.kind === 'code' } +type ComponentSurface = Extract + +function isComponentSurface(surface: MutableSurface | undefined): surface is ComponentSurface { + return typeof surface === 'object' && surface !== null && surface.kind === 'components' +} + +function componentSurface( + components: Readonly>, + source: string, +): ComponentSurface { + const entries = validateComponents(components, source) + return immutableCandidateValue({ + kind: 'components', + components: Object.fromEntries(entries), + } as ComponentSurface) +} + +function validateComponents( + components: Readonly>, + source: string, +): Array<[string, string]> { + if (typeof components !== 'object' || components === null || Array.isArray(components)) { + throw new ConfigError(`improve(): ${source} must return a component record`) + } + const entries = Object.entries(components) + if (entries.length === 0) { + throw new ConfigError(`improve(): ${source} must return at least one component`) + } + for (const [name, value] of entries) { + if (!name || name.trim() !== name || typeof value !== 'string') { + throw new ConfigError( + `improve(): ${source} must return trimmed component names with string values`, + ) + } + } + return entries +} + /** Create a clean incumbent checkout and the candidate producer for a code run. */ async function prepareCodeRun(code: ImproveCodeOptions): Promise { const baseRef = code.baseRef ?? 'main' @@ -505,13 +557,37 @@ function parseWinnerJson(winner: string, surface: ImproveSurface): T { function assertCandidateSurfaceKind( surface: ImproveSurface, + baseline: MutableSurface, winner: MutableSurface, ): asserts winner is MutableSurface { - if (surface === 'code' ? typeof winner === 'string' : typeof winner !== 'string') { + if (surface === 'code') { + if (isCodeSurface(winner)) return throw new ConfigError( `improve(): the '${surface}' candidate returned an incompatible surface value`, ) } + if (typeof baseline === 'string') { + if (typeof winner === 'string') return + throw new ConfigError( + `improve(): the '${surface}' candidate changed from a text surface to an incompatible surface value`, + ) + } + if (!isComponentSurface(baseline) || !isComponentSurface(winner)) { + throw new ConfigError( + `improve(): the '${surface}' candidate returned an incompatible surface value`, + ) + } + validateComponents(winner.components, `the '${surface}' candidate`) + const baselineNames = Object.keys(baseline.components).sort() + const winnerNames = Object.keys(winner.components).sort() + if ( + baselineNames.length !== winnerNames.length || + baselineNames.some((name, index) => name !== winnerNames[index]) + ) { + throw new ConfigError( + `improve(): the '${surface}' candidate must preserve the exact component names`, + ) + } } /** Materialize a detached profile candidate without changing the baseline. */ @@ -520,9 +596,33 @@ function materializeImprovementProfileCandidate( surface: ImproveSurface, winner: MutableSurface, skills?: ImproveSkillsOptions, + profileComponents?: ImproveProfileComponents, ): AgentProfile | undefined { - if (typeof winner !== 'string') return undefined let candidate: AgentProfile + if (isComponentSurface(winner)) { + if (surface !== 'agent-profile' || !profileComponents) { + throw new ConfigError( + `improve(): the '${surface}' candidate has no profile component mapping`, + ) + } + const winnerComponents = immutableCandidateValue({ ...winner.components }) + candidate = profileComponents.apply(profile, winnerComponents) + const validated = validateProfileCandidate(candidate, surface) + const materializedComponents = Object.fromEntries( + validateComponents(profileComponents.read(validated), 'profileComponents.read after apply'), + ) + const names = Object.keys(winnerComponents) + if ( + names.length !== Object.keys(materializedComponents).length || + names.some((name) => materializedComponents[name] !== winnerComponents[name]) + ) { + throw new ConfigError( + 'improve(): profileComponents.apply must round-trip every winning component exactly', + ) + } + return validated + } + if (typeof winner !== 'string') return undefined switch (surface) { case 'prompt': candidate = { ...profile, prompt: { ...profile.prompt, systemPrompt: winner } } @@ -565,11 +665,6 @@ function materializeImprovementProfileCandidate( } break case 'rollout-policy': { - // An empty winner is the never-opted-in baseline (see baselineSurfaceFor): - // nothing to apply, so no profile candidate — never a parse error. - if (winner === '') return undefined - // Parse + re-validate the winner against the policy's own invariants — a - // custom generator's malformed dial must fail loud, not persist silently. const policy = normalizeRolloutPolicy(parseWinnerJson(winner, surface)) if (!policy) { throw new ConfigError( @@ -583,6 +678,10 @@ function materializeImprovementProfileCandidate( case 'code': return undefined } + return validateProfileCandidate(candidate, surface) +} + +function validateProfileCandidate(candidate: AgentProfile, surface: ImproveSurface): AgentProfile { const parsed = agentProfileSchema.safeParse(candidate) if (!parsed.success) { throw new ConfigError( @@ -647,69 +746,155 @@ function assertFailClosedResources(profile: AgentProfile, surface: 'skills' | 'm } } -/** - * Run the held-out-gated self-improvement loop on ONE profile surface. - * - * @example Optimize the system prompt, default holdout gate: - * - * const out = await improve(profile, findings, { - * surface: 'prompt', - * scenarios, - * judge, - * agent: (surface, scenario, ctx) => runAgent(surface, scenario, ctx.signal), - * }) - * if (out.decision === 'ship') console.log(out.candidate) - */ -export async function improve( +function resolveOptimizationMethod( + source: ImproveMethodSource, + context: ImproveMethodContext, +): OptimizationMethod { + const method = typeof source === 'function' ? source(context) : source + if ( + !method || + typeof method !== 'object' || + typeof method.name !== 'string' || + method.name.trim() !== method.name || + method.name.length === 0 || + typeof method.optimize !== 'function' + ) { + throw new ConfigError( + 'improve(): method must be a complete OptimizationMethod with a trimmed name and optimize(input)', + ) + } + return method +} + +function copyCost(cost: { + totalCostUsd: number + accountingComplete: boolean + incompleteReasons: readonly string[] +}): ImproveCost { + return { + totalCostUsd: cost.totalCostUsd, + accountingComplete: cost.accountingComplete, + incompleteReasons: [...cost.incompleteReasons], + } +} + +function copyProvenance( + provenance: NonNullable, +): NonNullable { + return { + ...provenance, + source: { ...provenance.source }, + ...(provenance.tokenUsage ? { tokenUsage: { ...provenance.tokenUsage } } : {}), + } +} + +function developmentSplitDigest( + trainScenarios: readonly TScenario[], + selectionScenarios: readonly TScenario[], + reps: number, +): Sha256Digest { + return canonicalCandidateDigest({ + train: campaignSplitDigest(trainScenarios, reps), + selection: campaignSplitDigest(selectionScenarios, reps), + }) +} + +async function runMethodImprovement( profile: AgentProfile, - findings: unknown[], - opts: ImproveOptions, -): Promise> { + opts: ImproveMethodOptions, +): Promise { const { surface = 'prompt', - gate = 'holdout', - generator, - allowedModels, - rawTraceContext, - code, + method: methodSource, + agent, + findings: inputFindings = [], skills, - promotionGate, - analyzeGeneration, - ...sharedOptions + profileComponents, + minimumLift = 0, + ...comparisonOptions } = opts - - const parsedProfile = agentProfileSchema.safeParse(profile) - if (!parsedProfile.success) { + if (!Number.isFinite(minimumLift) || minimumLift < 0) { throw new ConfigError( - `improve(): input is not a valid AgentProfile: ${parsedProfile.error.message}`, + 'improve(): minimumLift must be a finite number greater than or equal to 0', ) } - if (surface === 'code' && generator) { - throw new ConfigError( - "improve(): surface 'code' forbids opts.generator because an external SurfaceProposer cannot transfer checkout ownership; pass opts.code.generator instead", - ) + if (profileComponents && surface !== 'agent-profile') { + throw new ConfigError("improve(): profileComponents is valid only with surface 'agent-profile'") } - const usesReflectionModel = !generator && (surface === 'prompt' || surface === 'skills') - if (usesReflectionModel) { - assertModelAllowed(sharedOptions.llm?.model ?? defaultReflectionModel, allowedModels) + const findings = [...inputFindings] + const baselineSurface = baselineSurfaceFor(profile, surface, skills, profileComponents) + const runtimeRunId = `runtime-optimization:${randomUUID()}` + const splitDigest = developmentSplitDigest( + comparisonOptions.trainScenarios, + comparisonOptions.selectionScenarios, + comparisonOptions.optimizationRunOptions?.reps ?? 1, + ) + const method = resolveOptimizationMethod(methodSource, { + profile, + surface, + baselineSurface, + findings, + }) + const startedAt = Date.now() + const raw = await compareOptimizationMethods({ + ...comparisonOptions, + methods: [method], + baselineSurface, + dispatchWithSurface: agent, + }) + const score = raw.best + const winnerSurface = score.winnerSurface + assertCandidateSurfaceKind(surface, baselineSurface, winnerSurface) + const candidateProfile = materializeImprovementProfileCandidate( + profile, + surface, + winnerSurface, + skills, + profileComponents, + ) + if (!candidateProfile) { + throw new ConfigError(`improve(): the '${surface}' method produced no profile candidate`) } + const candidate = immutableCandidateValue({ + surface, + value: winnerSurface, + profile: candidateProfile, + }) - let preparedCode: PreparedCodeRun | undefined - if (surface === 'code') { - if (!code) { - throw new ConfigError( - "improve(): surface 'code' requires opts.code.repoRoot so the incumbent can run from an isolated checkout", - ) - } - preparedCode = await prepareCodeRun(code) - } - const proposer = - preparedCode?.proposer ?? generator ?? defaultGeneratorFor(surface, sharedOptions.llm) - if (!proposer) { - throw new ConfigError( - `improve(): surface '${surface}' has no default generator — pass opts.generator (a SurfaceProposer) explicitly`, - ) + return { + mode: 'method', + method: method.name, + ...(score.provenance ? { provenance: copyProvenance(score.provenance) } : {}), + candidate, + decision: score.liftCi.low > minimumLift ? 'ship' : 'hold', + lift: score.lift, + liftInterval: { ...score.liftCi }, + cost: copyCost(raw.totalCost), + durationMs: Date.now() - startedAt, + lineage: Object.freeze({ + runId: score.provenance?.runId ?? runtimeRunId, + developmentSplitDigest: splitDigest, + }), + raw, + async dispose() {}, } +} + +async function runCodeImprovement( + opts: ImproveCodeRunOptions, +): Promise> { + const { + gate = 'holdout', + findings: inputFindings = [], + rawTraceContext: _rawTraceContext, + code, + promotionGate, + analyzeGeneration, + surface: _surface, + ...sharedOptions + } = opts + const findings = [...inputFindings] + const preparedCode = await prepareCodeRun(code) const budget: SelfImproveBudget = gate === 'none' ? { ...sharedOptions.budget, generations: 0 } : { ...sharedOptions.budget } @@ -718,9 +903,8 @@ export async function improve( try { raw = await selfImprove({ ...sharedOptions, - baselineSurface: - preparedCode?.baseline ?? baselineSurfaceFor(parsedProfile.data, surface, skills), - proposer, + baselineSurface: preparedCode.baseline, + proposer: preparedCode.proposer, budget, findings, ...(promotionGate !== undefined ? { gate: promotionGate } : {}), @@ -732,7 +916,6 @@ export async function improve( }), }) } catch (cause) { - if (!preparedCode) throw cause return rethrowAfterCleanup( cause, () => preparedCode.cleanup(), @@ -741,43 +924,74 @@ export async function improve( } const winnerSurface = raw.winner.surface - assertCandidateSurfaceKind(surface, winnerSurface) - if (preparedCode) { + assertCandidateSurfaceKind('code', preparedCode.baseline, winnerSurface) + try { + await preparedCode.cleanup(winnerSurface) + } catch (cleanupCause) { try { - await preparedCode.cleanup(winnerSurface) - } catch (cleanupCause) { - try { - await preparedCode.cleanup() - } catch (finalCleanupCause) { - throw new AggregateError( - [cleanupCause, finalCleanupCause], - 'improve(): code result cleanup failed, including the final all-worktree retry', - ) - } + await preparedCode.cleanup() + } catch (finalCleanupCause) { throw new AggregateError( - [cleanupCause], - 'improve(): code result cleanup failed; the final all-worktree retry succeeded', + [cleanupCause, finalCleanupCause], + 'improve(): code result cleanup failed, including the final all-worktree retry', ) } + throw new AggregateError( + [cleanupCause], + 'improve(): code result cleanup failed; the final all-worktree retry succeeded', + ) } - const dispose = idempotentDispose(async () => preparedCode?.cleanup()) - const candidateProfile = materializeImprovementProfileCandidate( - parsedProfile.data, - surface, - winnerSurface, - skills, - ) + const dispose = idempotentDispose(async () => preparedCode.cleanup()) const candidate = immutableCandidateValue({ - surface, + surface: 'code', value: winnerSurface, - ...(candidateProfile ? { profile: candidateProfile } : {}), }) return { + mode: 'code', candidate, decision: raw.gateDecision, ...(raw.lift !== undefined ? { lift: raw.lift } : {}), + cost: copyCost(raw.cost), + durationMs: raw.durationMs, + lineage: Object.freeze({ + runId: raw.provenance.runId, + developmentSplitDigest: raw.provenance.evidence.search.splitDigest, + }), + generationsExplored: raw.generationsExplored, raw, dispose, } } + +/** + * Optimize one exact profile surface with a complete method, or optimize code + * through Runtime's isolated worktree path. The input profile is never changed. + */ +export function improve( + profile: AgentProfile, + opts: ImproveMethodOptions, +): Promise +export function improve( + profile: AgentProfile, + opts: ImproveCodeRunOptions, +): Promise> +export function improve( + profile: AgentProfile, + opts: ImproveOptions, +): Promise> +export async function improve( + profile: AgentProfile, + opts: ImproveOptions, +): Promise> { + const parsedProfile = agentProfileSchema.safeParse(profile) + if (!parsedProfile.success) { + throw new ConfigError( + `improve(): input is not a valid AgentProfile: ${parsedProfile.error.message}`, + ) + } + if (opts.surface === 'code') { + return runCodeImprovement(opts) + } + return runMethodImprovement(immutableCandidateValue(parsedProfile.data), opts) +} diff --git a/src/improvement/improvement-driver.ts b/src/improvement/improvement-driver.ts index baf02d15..d41c5860 100644 --- a/src/improvement/improvement-driver.ts +++ b/src/improvement/improvement-driver.ts @@ -1,21 +1,8 @@ /** + * Code-only candidate driver for Runtime-owned git worktrees. * - * `improvementDriver` — the ONE reflective/agentic improvement proposer for - * agent-eval's improvement loop. It implements `SurfaceProposer` and owns - * the candidate lifecycle (worktree create → generate → finalize/discard, - * × populationSize); it delegates the only thing that genuinely varies — HOW - * a candidate change is produced — to a pluggable `CandidateGenerator`. - * - * There is no separate "analyst driver" vs "autoresearch driver": those are - * the SAME driver at two settings of a dial. - * - cheap reflective path → `reflectiveGenerator` (shots=1, no sandbox; - * applies pre-drafted patches) - * - full agentic path → `agenticGenerator` (shots=N, multi-shot - * verify-in-session loop; an agent reads code + - * report, edits, and re-tries on verifier failure) - * Both emit changes into a worktree the driver finalizes into a - * `CodeSurface{ worktreeRef }` the loop measures on the holdout. See - * agent-eval's `docs/design/self-improvement-engine.md`. + * A `CandidateGenerator` edits an isolated checkout. This driver finalizes each + * accepted edit as a `CodeSurface` and disposes rejected worktrees. * * @experimental */ @@ -65,7 +52,7 @@ export interface CandidateGenerator { * reflective generator ignores it). */ maxShots: number signal: AbortSignal - /** Improvement-loop coordinates. Present when called through improvementDriver. */ + /** Generation coordinates supplied by Runtime's internal code candidate driver. */ generation?: number candidateIndex?: number /** Shared run-wide paid-call account supplied by agent-eval 0.117+. */ @@ -98,7 +85,7 @@ export interface ManagedImprovementDriver extends SurfaceProposer } -/** The one reflective/agentic improvement proposer (`SurfaceProposer`): owns the candidate worktree lifecycle and delegates HOW a change is produced to a pluggable `CandidateGenerator`. */ +/** Build the code-only proposer used internally by `improve({ surface: 'code' })`. */ export function improvementDriver(opts: ImprovementDriverOptions): ManagedImprovementDriver { const baseRef = opts.baseRef ?? 'main' const owned = new Map() diff --git a/src/improvement/index.ts b/src/improvement/index.ts index 4c436f07..d7d329a1 100644 --- a/src/improvement/index.ts +++ b/src/improvement/index.ts @@ -1,15 +1,9 @@ /** - * `@tangle-network/agent-runtime` improvement — the CODE-surface proposer for - * agent-eval's improvement loop. + * `@tangle-network/agent-runtime` improvement. * - * The public entry point is `improve()`, a profile-aware facade over agent-eval's - * `selfImprove`. This module also supplies the runtime-specific code candidate - * producer, which mutates an isolated git worktree via a pluggable - * `CandidateGenerator`: - * - `reflectiveGenerator` — cheap, no sandbox, applies pre-drafted patches - * - `agenticGenerator` — full coding harness in the worktree, multi-shot - * - `driverLoopGenerator` — the driver→worker atom: an LLM driver authors, - * observes, rates, and steers the harness sessions (default for tool/mcp) + * The public entry point is `improve()`. Complete agent-eval methods optimize + * profile surfaces. Runtime owns only code candidates that mutate an isolated + * git worktree through a pluggable `CandidateGenerator`. */ export { @@ -25,13 +19,6 @@ export { type VerifyResult, } from './agentic-generator' export { findingLines, mcpBuildPrompt, toolBuildPrompt } from './build-prompts' -export { - type CampaignOtlpOptions, - type CampaignTraceResolverOptions, - campaignCellSpansToOtlp, - campaignTraceResolver, - convertCampaignDirToOtlp, -} from './campaign-otlp' export { type DriverLoopGeneratorOptions, driverLoopGenerator, @@ -44,31 +31,40 @@ export { } from './findings' export { type ImproveCodeOptions, + type ImproveCodeResult, + type ImproveCodeRunOptions, + type ImproveCost, + type ImproveLineage, + type ImproveMethodContext, + type ImproveMethodFactory, + type ImproveMethodOptions, + type ImproveMethodResult, + type ImproveMethodSource, type ImprovementCandidate, type ImproveOptions, + type ImproveProfileComponents, + type ImproveProfileSurface, type ImproveResult, type ImproveSkillsOptions, type ImproveSurface, improve, } from './improve' -export { - type CandidateGenerator, - type ImprovementDriverOptions, - improvementDriver, - type ManagedImprovementDriver, -} from './improvement-driver' +export type { CandidateGenerator } from './improvement-driver' export { type McpServeSpec, mcpServeVerifier } from './mcp-serve-verifier' +export { + type OfficialGepaOptions, + type OfficialOptimizerContextOptions, + OfficialOptimizerUnavailableError, + type OfficialSkillOptOptions, + officialGepa, + officialSkillOpt, +} from './official-optimizers' export { buildDriverSystem, optimizerMethod, researchDriverNote, strategyAuthorMethod, } from './optimizer-prompt' -export { - type ProfileDiffProposerContext, - type ProfileDiffProposerOptions, - profileDiffProposer, -} from './profile-diff-proposer' export { type RawTraceDistillerOptions, rawTraceDistiller, @@ -76,12 +72,9 @@ export { export { type ReflectiveGeneratorOptions, reflectiveGenerator } from './reflective-generator' export { applyRolloutPolicyToProfile, - enumerateNeighborPolicies, normalizeRolloutPolicy, parseRolloutPolicy, - ROLLOUT_POLICY_BOUNDS, ROLLOUT_POLICY_EXTENSION, - rolloutPolicyProposer, serializeRolloutPolicy, structuralRolloutPolicyFromProfile, } from './rollout-policy' diff --git a/src/improvement/official-optimizers.test.ts b/src/improvement/official-optimizers.test.ts new file mode 100644 index 00000000..4fb58c6e --- /dev/null +++ b/src/improvement/official-optimizers.test.ts @@ -0,0 +1,403 @@ +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import type { + DispatchContext, + JudgeConfig, + MutableSurface, + Scenario, +} from '@tangle-network/agent-eval/contract' +import type { AgentProfile } from '@tangle-network/agent-interface' +import { afterEach, describe, expect, it } from 'vitest' +import { improve } from './improve' +import { + OfficialOptimizerUnavailableError, + officialGepa, + officialSkillOpt, +} from './official-optimizers' + +interface OptimizerScenario extends Scenario { + kind: 'fixture' + prompt: string + privateNote: string +} + +interface Artifact { + text: string +} + +const profile: AgentProfile = { + name: 'optimizer-fixture', + prompt: { systemPrompt: 'baseline' }, +} + +const judge: JudgeConfig = { + name: 'improvement', + dimensions: [{ key: 'quality', description: 'candidate is improved' }], + score: ({ artifact }) => { + const quality = artifact.text === 'improved prompt' ? 1 : 0 + return { dimensions: { quality }, composite: quality, notes: '' } + }, +} + +async function agent( + surface: MutableSurface, + _scenario: OptimizerScenario, + ctx: DispatchContext, +): Promise { + const paid = await ctx.cost.runPaidCall({ + channel: 'agent', + actor: 'official-optimizer-test', + model: 'deterministic-test', + maximumCharge: { externallyEnforcedMaximumUsd: 0.0001 }, + execute: async () => ({ text: String(surface) }), + receipt: () => ({ + model: 'deterministic-test', + inputTokens: 1, + outputTokens: 1, + actualCostUsd: 0.0001, + }), + }) + if (!paid.succeeded) throw paid.error + return paid.value +} + +const train: OptimizerScenario[] = [ + { id: 'train', kind: 'fixture', prompt: 'visible train', privateNote: 'TRAIN_SECRET' }, +] +const selection: OptimizerScenario[] = [ + { + id: 'selection', + kind: 'fixture', + prompt: 'visible selection', + privateNote: 'SELECTION_SECRET', + }, +] +const testCases: OptimizerScenario[] = [ + { id: 'test-a', kind: 'fixture', prompt: 'private test a', privateNote: 'TEST_SECRET_A' }, + { id: 'test-b', kind: 'fixture', prompt: 'private test b', privateNote: 'TEST_SECRET_B' }, +] + +const runDirs: string[] = [] + +afterEach(() => { + for (const runDir of runDirs.splice(0)) { + rmSync(runDir, { recursive: true, force: true }) + } +}) + +function runDir(): string { + const value = mkdtempSync(join(tmpdir(), 'runtime-official-optimizer-')) + runDirs.push(value) + return value +} + +function fakeRunner(optimizer: 'gepa' | 'skillopt', observedInputPath: string) { + const optimizerSource = { + package: optimizer, + version: optimizer === 'gepa' ? 'test' : '0.2.0', + ...(optimizer === 'gepa' ? { revision: 'test-revision' } : {}), + sourceSha256: optimizer === 'gepa' ? 'b'.repeat(64) : 'c'.repeat(64), + } + const output = + optimizer === 'gepa' + ? [ + ' bestCandidate: "improved prompt",', + ' bestScore: 1,', + ' totalEvaluations: 0,', + ' recipeKind: input.recipe.kind,', + ' proposerCostAccounting: "unavailable",', + ' upstream: optimizerSource,', + ] + : [ + ' bestCandidate: "improved prompt",', + ' bestScore: 1,', + ' totalEvaluations: 0,', + ' totalSteps: 0,', + ' tokenUsage: { inputTokens: 0, outputTokens: 0, totalTokens: 0, calls: 0 },', + ' upstream: optimizerSource,', + ] + const source = [ + "const fs = require('node:fs')", + "const inputPath = process.argv[process.argv.indexOf('--input') + 1]", + "const outputPath = process.argv[process.argv.indexOf('--output') + 1]", + 'const input = JSON.parse(fs.readFileSync(inputPath, "utf8"))', + `const optimizerSource = ${JSON.stringify(optimizerSource)}`, + 'if (input.operation === "inspect") {', + ' fs.writeFileSync(outputPath, JSON.stringify({ runtime: {', + ' python: { implementation: "CPython", version: "3.12.0" },', + ' bridge: { package: "agent-eval-rpc", version: "0.126.0", sourceSha256: "a".repeat(64) },', + ' optimizer: optimizerSource,', + ' engineModules: [],', + ' } }))', + ' process.exit(0)', + '}', + `fs.writeFileSync(${JSON.stringify(observedInputPath)}, JSON.stringify(input))`, + 'fs.writeFileSync(outputPath, JSON.stringify({', + ...output, + ' runId: input.runId,', + ' resumed: false,', + '}))', + ].join('\n') + return { + command: process.execPath, + args: ['-e', source, '--'], + } +} + +function failingRunner(message: string) { + return { + command: process.execPath, + args: ['-e', `process.stderr.write(${JSON.stringify(message)}); process.exit(1)`, '--'], + } +} + +const testOptimizer = { + model: 'optimizer-model', + baseUrl: 'http://127.0.0.1:1/v1', + apiKey: 'test-api-key', + budget: { + maxCostUsd: 1, + maxRequests: 10, + maxRequestBytes: 100_000, + maxResponseBytes: 100_000, + maxOutputTokensPerRequest: 1_000, + pricing: { + inputUsdPerMillion: 1, + outputUsdPerMillion: 1, + }, + }, +} + +function commonOptions(method: ReturnType>) { + return { + surface: 'prompt' as const, + method, + findings: [{ claim: 'answers omit citations' }], + trainScenarios: train, + selectionScenarios: selection, + testScenarios: testCases, + judges: [judge], + agent, + runDir: runDir(), + resamples: 40, + confidence: 0.95, + } +} + +describe('official optimizer methods', () => { + it('runs the explicit GEPA engine recipe without exposing final-test data', async () => { + const root = runDir() + const observedInputPath = join(root, 'observed-input.json') + const result = await improve(profile, { + ...commonOptions( + officialGepa({ + objective: 'Improve the agent prompt.', + evaluationId: 'prompt-eval', + recipe: { + kind: 'engine', + run: { + engine: 'gepa', + maxEvaluations: 3, + maxProposerCostUsd: 1, + }, + }, + resume: 'if-compatible', + trustResumeState: true, + describeScenario: (scenario: OptimizerScenario) => ({ prompt: scenario.prompt }), + runner: fakeRunner('gepa', observedInputPath), + }), + ), + runDir: join(root, 'run'), + }) + + const observed = JSON.parse(readFileSync(observedInputPath, 'utf8')) as Record + expect(observed).toMatchObject({ + evaluationId: 'prompt-eval', + resume: 'if-compatible', + trustedResumeState: true, + recipe: { + kind: 'engine', + run: { + engine: 'gepa', + maxEvaluations: 3, + maxProposerCostUsd: 1, + }, + }, + trainSet: [{ id: 'train', data: { prompt: 'visible train' } }], + selectionSet: [{ id: 'selection', data: { prompt: 'visible selection' } }], + }) + expect(observed).not.toHaveProperty('version') + expect(String(observed.background)).toContain('answers omit citations') + expect(JSON.stringify(observed)).not.toContain('SECRET') + expect(observed).not.toHaveProperty('testSet') + expect(result.method).toBe('gepa:gepa') + expect(result.provenance).toMatchObject({ + source: { package: 'gepa', version: 'test' }, + runId: observed.runId, + resumed: false, + evaluationCount: 0, + }) + expect(result.decision).toBe('ship') + expect(result.candidate.profile?.prompt?.systemPrompt).toBe('improved prompt') + }) + + it('passes the official Omni recipe through unchanged', async () => { + const root = runDir() + const observedInputPath = join(root, 'observed-omni.json') + const recipe = { + kind: 'omni' as const, + explore: [ + { engine: 'gepa', maxEvaluations: 2, maxProposerCostUsd: 1 }, + { engine: 'autoresearch', maxEvaluations: 3, maxProposerCostUsd: 2 }, + { engine: 'meta_harness', maxEvaluations: 4, maxProposerCostUsd: 3 }, + ], + continueWith: { + engine: 'gepa', + maxEvaluations: 5, + maxProposerCostUsd: 4, + }, + maxWorkers: 3, + } + const result = await improve(profile, { + ...commonOptions( + officialGepa({ + objective: 'Improve the agent prompt.', + evaluationId: 'prompt-eval', + recipe, + describeScenario: (scenario: OptimizerScenario) => ({ prompt: scenario.prompt }), + runner: fakeRunner('gepa', observedInputPath), + }), + ), + runDir: join(root, 'run'), + }) + + const observed = JSON.parse(readFileSync(observedInputPath, 'utf8')) as { + recipe: unknown + } + expect(observed.recipe).toEqual(recipe) + expect(result.method).toBe('gepa:omni:gepa') + }) + + it('runs official Microsoft SkillOpt with the same findings and data boundary', async () => { + const root = runDir() + const observedInputPath = join(root, 'observed-skillopt.json') + const result = await improve(profile, { + ...commonOptions( + officialSkillOpt({ + objective: 'Improve the agent prompt.', + evaluationId: 'prompt-eval', + trainer: { + epochs: 1, + batchSize: 1, + }, + optimizer: testOptimizer, + maxEvaluations: 3, + resume: 'if-compatible', + describeScenario: (scenario: OptimizerScenario) => ({ prompt: scenario.prompt }), + runner: fakeRunner('skillopt', observedInputPath), + }), + ), + runDir: join(root, 'run'), + }) + + const observed = JSON.parse(readFileSync(observedInputPath, 'utf8')) as Record + expect(observed).toMatchObject({ + evaluationId: 'prompt-eval', + resume: 'if-compatible', + trainer: { + epochs: 1, + batchSize: 1, + }, + trainSet: [{ id: 'train', data: { prompt: 'visible train' } }], + selectionSet: [{ id: 'selection', data: { prompt: 'visible selection' } }], + }) + expect(observed).not.toHaveProperty('version') + expect(observed.trainer).toEqual({ epochs: 1, batchSize: 1 }) + expect(String(observed.background)).toContain('answers omit citations') + expect(JSON.stringify(observed)).not.toContain('SECRET') + expect(JSON.stringify(observed)).not.toContain('test-api-key') + expect(observed).not.toHaveProperty('testSet') + expect(result.method).toBe('skillopt') + expect(result.provenance).toMatchObject({ + source: { package: 'skillopt', version: '0.2.0' }, + runId: observed.runId, + resumed: false, + evaluationCount: 0, + }) + expect(result.decision).toBe('ship') + }) + + it.each([ + [ + 'gepa', + 'gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f', + () => + officialGepa({ + objective: 'Improve the agent prompt.', + evaluationId: 'prompt-eval', + recipe: { + kind: 'engine', + run: { engine: 'gepa', maxEvaluations: 1, maxProposerCostUsd: 1 }, + }, + runner: { command: '/definitely/missing/python' }, + }), + ], + [ + 'skillopt', + 'microsoft/SkillOpt.git@61735e3922efc2b90c6d6cab561e62e98452ca90', + () => + officialSkillOpt({ + objective: 'Improve the agent prompt.', + evaluationId: 'prompt-eval', + trainer: { + epochs: 1, + batchSize: 1, + }, + optimizer: testOptimizer, + maxEvaluations: 3, + runner: { + command: '/definitely/missing/python', + }, + }), + ], + ] as const)('fails clearly instead of falling back when %s is unavailable', async (optimizer, installFragment, method) => { + await expect( + improve(profile, { + ...commonOptions(method()), + }), + ).rejects.toMatchObject({ + name: 'OfficialOptimizerUnavailableError', + optimizer, + message: expect.stringMatching( + new RegExp( + `Runtime did not use a local fallback[\\s\\S]*${installFragment.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}`, + ), + ), + } satisfies Partial) + }) + + it('does not relabel an unrelated optimizer process failure as a missing dependency', async () => { + const method = officialGepa({ + objective: 'Improve the agent prompt.', + evaluationId: 'prompt-eval', + recipe: { + kind: 'engine', + run: { engine: 'gepa', maxEvaluations: 1, maxProposerCostUsd: 1 }, + }, + runner: failingRunner( + 'RuntimeError in agent_eval_rpc.gepa_bridge: application-owned callback failed', + ), + }) + + let failure: unknown + try { + await improve(profile, { ...commonOptions(method) }) + } catch (cause) { + failure = cause + } + expect(failure).toBeInstanceOf(Error) + expect(failure).not.toBeInstanceOf(OfficialOptimizerUnavailableError) + expect((failure as Error).message).toContain('application-owned callback failed') + }) +}) diff --git a/src/improvement/official-optimizers.ts b/src/improvement/official-optimizers.ts new file mode 100644 index 00000000..0794e5f8 --- /dev/null +++ b/src/improvement/official-optimizers.ts @@ -0,0 +1,204 @@ +import { canonicalJson } from '@tangle-network/agent-eval' +import { + type GepaOptimizationMethodConfig, + gepaOptimizationMethod, + type OptimizationMethod, + type SkillOptOptimizationMethodConfig, + skillOptOptimizationMethod, +} from '@tangle-network/agent-eval/campaign' +import { ConfigError } from '../errors' +import type { ImproveMethodContext, ImproveMethodFactory } from './improve' + +const DEFAULT_MAX_FINDINGS_CHARS = 50_000 +const PYTHON_CLIENT_DOCS = 'https://github.com/tangle-network/agent-eval/tree/main/clients/python' +const GEPA_INSTALL = + '`python -m pip install agent-eval-rpc`, then ' + + '`python -m pip install "gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f"`' +const SKILLOPT_INSTALL = + '`python -m pip install agent-eval-rpc`, then ' + + '`python -m pip install "skillopt @ git+https://github.com/microsoft/SkillOpt.git@61735e3922efc2b90c6d6cab561e62e98452ca90"`' + +/** Runtime context appended to an official optimizer's own configuration. */ +export interface OfficialOptimizerContextOptions { + /** Context supplied to the optimizer before Runtime appends the profile surface and findings. */ + background?: string + /** Include current trace or analyst findings in the optimizer background. Default true. */ + includeFindings?: boolean + /** Reject oversized serialized findings before starting Python. Default 50,000 characters. */ + maxFindingsChars?: number +} + +/** Official GEPA configuration plus bounded Runtime findings context. */ +export type OfficialGepaOptions< + TScenario extends { id: string; kind: string }, + TArtifact = unknown, +> = Omit, 'background'> & + OfficialOptimizerContextOptions + +/** Official SkillOpt configuration plus bounded Runtime findings context. */ +export type OfficialSkillOptOptions< + TScenario extends { id: string; kind: string }, + TArtifact = unknown, +> = Omit, 'background'> & + OfficialOptimizerContextOptions + +/** Missing optional Python dependencies for an official optimizer. */ +export class OfficialOptimizerUnavailableError extends ConfigError { + readonly optimizer: 'gepa' | 'skillopt' + + constructor(optimizer: 'gepa' | 'skillopt', cause: unknown) { + const detail = cause instanceof Error ? cause.message : String(cause) + const install = + optimizer === 'gepa' + ? `Install the Python bridge and pinned GEPA source: ${GEPA_INSTALL}.` + : `Install Microsoft SkillOpt: ${SKILLOPT_INSTALL}.` + super( + [ + `Official ${optimizer === 'gepa' ? 'GEPA' : 'SkillOpt'} could not start.`, + 'Runtime did not use a local fallback.', + install, + `Setup: ${PYTHON_CLIENT_DOCS}.`, + `Cause: ${detail}`, + ].join(' '), + { cause }, + ) + this.optimizer = optimizer + } +} + +/** + * Build a complete method backed by GEPA's official Optimize Anything API. + * + * The recipe is passed through unchanged. Use `engine`, `sequential`, + * `adaptive-sequential`, `best-of`, `vote`, or `omni` explicitly. + */ +export function officialGepa( + options: OfficialGepaOptions, +): ImproveMethodFactory { + const { background, includeFindings = true, maxFindingsChars, ...config } = options + assertMaxFindingsChars('officialGepa', maxFindingsChars) + return (context) => + withDependencyHelp( + 'gepa', + gepaOptimizationMethod({ + ...config, + background: methodBackground({ + context, + background, + includeFindings, + maxFindingsChars, + label: 'officialGepa', + }), + }), + ) +} + +/** Build a complete method backed by Microsoft's official SkillOpt trainer. */ +export function officialSkillOpt< + TScenario extends { id: string; kind: string }, + TArtifact = unknown, +>( + options: OfficialSkillOptOptions, +): ImproveMethodFactory { + const { background, includeFindings = true, maxFindingsChars, ...config } = options + assertMaxFindingsChars('officialSkillOpt', maxFindingsChars) + return (context) => + withDependencyHelp( + 'skillopt', + skillOptOptimizationMethod({ + ...config, + background: methodBackground({ + context, + background, + includeFindings, + maxFindingsChars, + label: 'officialSkillOpt', + }), + }), + ) +} + +function assertMaxFindingsChars(label: string, value: number | undefined): void { + if (value !== undefined && (!Number.isSafeInteger(value) || value <= 0)) { + throw new ConfigError(`${label}: maxFindingsChars must be a positive safe integer`) + } +} + +function methodBackground(options: { + context: ImproveMethodContext + background: string | undefined + includeFindings: boolean + maxFindingsChars: number | undefined + label: string +}): string { + const { + context, + background, + includeFindings, + maxFindingsChars = DEFAULT_MAX_FINDINGS_CHARS, + label, + } = options + const sections = [ + background?.trim(), + `Agent profile: ${context.profile.name}. Surface: ${context.surface}.`, + ].filter((value): value is string => Boolean(value)) + if (includeFindings && context.findings.length > 0) { + let serialized: string + try { + serialized = canonicalJson(context.findings) + } catch (cause) { + throw new ConfigError(`${label}: findings must be JSON-serializable`, { cause }) + } + if (serialized.length > maxFindingsChars) { + throw new ConfigError( + `${label}: serialized findings exceed maxFindingsChars (${serialized.length} > ${maxFindingsChars})`, + ) + } + sections.push(`Observed failures:\n${serialized}`) + } + return sections.join('\n\n') +} + +function withDependencyHelp( + optimizer: 'gepa' | 'skillopt', + method: OptimizationMethod, +): OptimizationMethod { + return { + ...method, + async optimize(input) { + try { + return await method.optimize(input) + } catch (cause) { + if (isMissingDependency(optimizer, cause)) { + throw new OfficialOptimizerUnavailableError(optimizer, cause) + } + throw cause + } + }, + } +} + +function isMissingDependency(optimizer: 'gepa' | 'skillopt', cause: unknown): boolean { + const message = cause instanceof Error ? cause.message : String(cause) + const common = [ + `${optimizer === 'gepa' ? 'GEPA' : 'SkillOpt'} bridge could not start`, + 'source inspection could not start', + "No module named 'agent_eval_rpc'", + `No module named 'agent_eval_rpc.${optimizer === 'gepa' ? 'gepa_bridge' : 'skillopt_bridge'}'`, + ] + const specific = + optimizer === 'gepa' + ? [ + 'requires GEPA', + "requires GEPA's Optimize Anything", + "No module named 'gepa'", + 'gepa is importable but its package metadata is unavailable', + ] + : [ + 'requires skillopt', + 'requires SkillOpt', + "No module named 'skillopt'", + 'skillopt is importable but its package metadata is unavailable', + ] + return [...common, ...specific].some((fragment) => message.includes(fragment)) +} diff --git a/src/improvement/profile-diff-proposer.test.ts b/src/improvement/profile-diff-proposer.test.ts deleted file mode 100644 index 2bb7bde6..00000000 --- a/src/improvement/profile-diff-proposer.test.ts +++ /dev/null @@ -1,132 +0,0 @@ -import type { - ProposeContext, - ProposedCandidate, - SurfaceProposer, -} from '@tangle-network/agent-eval/campaign' -import { compositeProposer } from '@tangle-network/agent-eval/campaign' -import { defineAgentProfileDiff, defineInlineResource } from '@tangle-network/agent-interface' -import { describe, expect, it } from 'vitest' -import { profileDiffProposer } from './profile-diff-proposer' - -const context = (populationSize = 2): ProposeContext => ({ - currentSurface: JSON.stringify({ - name: 'fixture', - prompt: { systemPrompt: 'baseline' }, - resources: { failOnError: true }, - }), - history: [], - findings: [], - populationSize, - generation: 0, - signal: new AbortController().signal, -}) - -describe('profileDiffProposer', () => { - it('puts source-grounded profile diffs and generated mutations in one candidate pool', async () => { - const discovered = profileDiffProposer({ - proposeDiffs: () => [ - defineAgentProfileDiff({ - kind: 'agent-profile-diff', - id: 'github-review-skill', - title: 'Pinned review skill', - rationale: 'Existing skill covers the observed review failure.', - source: { - kind: 'frontier-author', - artifacts: ['https://github.com/example/skills/tree/0123456789abcdef/review'], - notes: ['MIT license verified by the research worker.'], - }, - set: { - resources: { - skills: [ - defineInlineResource( - 'review.SKILL.md', - '# Review\n\nRead the failing test before editing.', - ), - ], - failOnError: true, - }, - }, - }), - ], - }) - const generated: SurfaceProposer = { - kind: 'generated', - propose: async () => [ - { - surface: JSON.stringify({ - name: 'fixture', - prompt: { systemPrompt: 'generated mutation' }, - resources: { failOnError: true }, - }), - label: 'generated prompt', - rationale: 'Mutation proposer output.', - }, - ], - } - - const candidates = await compositeProposer({ - proposers: [discovered, generated], - }).propose(context(2)) - - expect(candidates).toHaveLength(2) - const researched = candidates[0] as ProposedCandidate - expect(researched.label).toBe('agent-profile-diff:Pinned review skill') - expect(researched.rationale).toContain('Existing skill') - expect(researched.rationale).toContain( - 'https://github.com/example/skills/tree/0123456789abcdef/review', - ) - expect(researched.rationale).toMatch(/sha256:[a-f0-9]{64}/) - expect(JSON.parse(researched.surface as string)).toMatchObject({ - name: 'fixture', - prompt: { systemPrompt: 'baseline' }, - resources: { - skills: [ - { - kind: 'inline', - name: 'review.SKILL.md', - }, - ], - failOnError: true, - }, - }) - expect((candidates[1] as ProposedCandidate).label).toBe('generated:generated prompt') - }) - - it('rejects source output with fields the shared diff contract would discard', async () => { - const proposer = profileDiffProposer({ - proposeDiffs: () => [ - { - kind: 'agent-profile-diff', - set: { prompt: { systemPrompt: 'candidate' } }, - unknownField: true, - } as never, - ], - }) - await expect(proposer.propose(context(1))).rejects.toThrow(/unsupported or non-canonical/) - }) - - it('drops no-op diffs and respects the shared population budget', async () => { - const proposer = profileDiffProposer({ - proposeDiffs: () => [ - defineAgentProfileDiff({ kind: 'agent-profile-diff' }), - defineAgentProfileDiff({ - kind: 'agent-profile-diff', - set: { prompt: { systemPrompt: 'baseline' } }, - }), - defineAgentProfileDiff({ - kind: 'agent-profile-diff', - set: { prompt: { systemPrompt: 'first' } }, - }), - defineAgentProfileDiff({ - kind: 'agent-profile-diff', - set: { prompt: { systemPrompt: 'first' } }, - }), - ], - }) - const candidates = await proposer.propose(context(2)) - expect(candidates).toHaveLength(1) - expect( - JSON.parse((candidates[0] as ProposedCandidate).surface as string).prompt.systemPrompt, - ).toBe('first') - }) -}) diff --git a/src/improvement/profile-diff-proposer.ts b/src/improvement/profile-diff-proposer.ts deleted file mode 100644 index 90e2ddbc..00000000 --- a/src/improvement/profile-diff-proposer.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { canonicalJson } from '@tangle-network/agent-eval' -import type { - MutableSurface, - ProposeContext, - ProposedCandidate, - SurfaceProposer, -} from '@tangle-network/agent-eval/campaign' -import { - type AgentProfile, - type AgentProfileDiff, - changedAgentProfileAxes, -} from '@tangle-network/agent-interface' -import { canonicalCandidateDigest } from '../candidate-execution/digest' -import { - applyExactAgentProfileDiff, - parseExactAgentProfile, - parseExactAgentProfileDiff, -} from '../candidate-execution/profile' - -export type ProfileDiffProposerContext = ProposeContext & { - profile: AgentProfile -} - -export interface ProfileDiffProposerOptions { - proposeDiffs( - context: ProfileDiffProposerContext, - ): Promise | readonly AgentProfileDiff[] -} - -/** - * Turn exact AgentProfileDiffs from any source into full profile candidates for - * the shared optimization loop. Research, catalogs, humans, and trace miners - * differ only in `proposeDiffs`; measurement and promotion stay identical. - */ -export function profileDiffProposer( - options: ProfileDiffProposerOptions, -): SurfaceProposer { - return { - kind: 'agent-profile-diff', - async propose(ctx): Promise> { - if (typeof ctx.currentSurface !== 'string') { - throw new Error('profileDiffProposer requires a serialized AgentProfile surface') - } - const profile = parseSerializedProfile(ctx.currentSurface) - const proposals = await options.proposeDiffs({ - ...ctx, - profile, - }) - const candidates: ProposedCandidate[] = [] - const normalizedBaseline = applyExactAgentProfileDiff( - profile, - { kind: 'agent-profile-diff' }, - 'profile diff baseline', - ) - const seen = new Set([canonicalJson(normalizedBaseline)]) - for (const [index, proposal] of proposals.entries()) { - if (candidates.length >= ctx.populationSize || ctx.signal.aborted) break - const diff = parseExactAgentProfileDiff(proposal, `profile diff proposal ${index}`) - const axes = changedAgentProfileAxes(diff) - if (axes.length === 0) continue - const candidate = applyExactAgentProfileDiff( - profile, - diff, - `profile diff candidate ${index}`, - ) - const surface = canonicalJson(candidate) - if (seen.has(surface)) continue - seen.add(surface) - candidates.push({ - surface, - label: diff.title ?? diff.id ?? `Change ${axes.join(', ')}`, - rationale: candidateRationale(diff, axes), - }) - } - return candidates - }, - } -} - -function candidateRationale(diff: AgentProfileDiff, axes: readonly string[]): string { - const reason = diff.rationale ?? `Changes profile axes: ${axes.join(', ')}.` - const source = diff.source - ? [diff.source.kind, ...(diff.source.artifacts ?? [])].join(' · ') - : 'source unspecified' - return `${reason} Source: ${source}. Diff: ${canonicalCandidateDigest(diff)}.` -} - -function parseSerializedProfile(surface: string): AgentProfile { - let raw: unknown - try { - raw = JSON.parse(surface) - } catch (cause) { - throw new Error('profileDiffProposer current surface is not valid AgentProfile JSON', { - cause, - }) - } - return parseExactAgentProfile(raw, 'profileDiffProposer current profile') -} diff --git a/src/improvement/raw-trace-distiller.ts b/src/improvement/raw-trace-distiller.ts index 5d7db35b..493359ee 100644 --- a/src/improvement/raw-trace-distiller.ts +++ b/src/improvement/raw-trace-distiller.ts @@ -75,10 +75,11 @@ interface CellTrace { * FILESYSTEM CONTEXT — paths into the prior generation's real run traces plus a * grep/cat-to-diagnose instruction — instead of a pre-summarized digest. * - * Drop-in for `opts.analyzeGeneration` on `improve()` / `selfImprove()`: + * Drop-in for `analyzeGeneration` on `improve({ surface: 'code' })`: * - * await improve(profile, seedFindings, { + * await improve(profile, { * surface: 'code', + * findings: seedFindings, * code: { repoRoot }, * runDir: '/abs/run', // MUST be a real path — the traces live here * analyzeGeneration: rawTraceDistiller(), diff --git a/src/improvement/reflective-generator.ts b/src/improvement/reflective-generator.ts index f6c7d5ee..7e98bc57 100644 --- a/src/improvement/reflective-generator.ts +++ b/src/improvement/reflective-generator.ts @@ -6,9 +6,8 @@ * the candidate worktree. `maxShots` is ignored — reflection is single-shot by * construction (the patches are already drafted). * - * This is the `shots=1, sandbox=off` setting of the one improvement driver. - * The `agenticGenerator` (a multi-shot verify-in-session loop) is the - * `shots=N` setting — both plug into the same `improvementDriver`. + * This is the `shots=1, sandbox=off` code-candidate setting. + * `agenticGenerator` supplies the multi-shot verify-in-session setting. * * @experimental */ diff --git a/src/improvement/rollout-policy.test.ts b/src/improvement/rollout-policy.test.ts index 9d6a186b..e1d8ff8c 100644 --- a/src/improvement/rollout-policy.test.ts +++ b/src/improvement/rollout-policy.test.ts @@ -1,149 +1,23 @@ -/** - * `'rollout-policy'` surface proof. - * - * What this guards: `improve()` can tune the inference-time structuralRollout - * dials { k, repairRounds, testgen } through agent-eval's generic string-surface - * contract — deterministic bounded candidate enumeration, held-out gate deciding, - * winner persisted into `profile.extensions['structural-rollout']`, and a strict - * no-op when the profile never opted into structural rollout. - * - * Deterministic and offline throughout: the proposer is enumeration (no LLM), the - * judge is a pure function of the candidate policy's `k` dial, and the stub agent - * reports token-bearing cost so the backend-integrity guard sees a real backend. - */ - -import { isProposedCandidate, type ProposedCandidate } from '@tangle-network/agent-eval/campaign' -import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract' +import { + inMemoryCampaignStorage, + type OptimizationMethod, +} from '@tangle-network/agent-eval/campaign' +import type { JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract' import type { AgentProfile } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import type { StructuralRolloutPolicy } from '../runtime/structural-rollout' import { improve } from './improve' import { applyRolloutPolicyToProfile, - enumerateNeighborPolicies, normalizeRolloutPolicy, parseRolloutPolicy, - ROLLOUT_POLICY_BOUNDS, ROLLOUT_POLICY_EXTENSION, - rolloutPolicyProposer, serializeRolloutPolicy, structuralRolloutPolicyFromProfile, } from './rollout-policy' -const inBounds = (p: StructuralRolloutPolicy) => - p.k >= ROLLOUT_POLICY_BOUNDS.k.min && - p.k <= ROLLOUT_POLICY_BOUNDS.k.max && - p.repairRounds >= ROLLOUT_POLICY_BOUNDS.repairRounds.min && - p.repairRounds <= ROLLOUT_POLICY_BOUNDS.repairRounds.max && - p.testgen >= ROLLOUT_POLICY_BOUNDS.testgen.min && - p.testgen <= ROLLOUT_POLICY_BOUNDS.testgen.max - -describe('enumerateNeighborPolicies — bounded single-dial moves', () => { - it('emits all six in-bounds neighbors of an interior policy, none equal to it', () => { - const base: StructuralRolloutPolicy = { k: 5, repairRounds: 2, testgen: 6 } - const neighbors = enumerateNeighborPolicies(base) - expect(neighbors.map((n) => [n.k, n.repairRounds, n.testgen])).toEqual([ - [7, 2, 6], - [3, 2, 6], - [5, 3, 6], - [5, 1, 6], - [5, 2, 9], - [5, 2, 3], - ]) - for (const n of neighbors) expect(inBounds(n)).toBe(true) - expect(neighbors.map(serializeRolloutPolicy)).not.toContain(serializeRolloutPolicy(base)) - }) - - it('clamps at the floor: downward moves that clamp to a no-op are dropped', () => { - const neighbors = enumerateNeighborPolicies({ k: 1, repairRounds: 0, testgen: 0 }) - expect(neighbors.map((n) => [n.k, n.repairRounds, n.testgen])).toEqual([ - [3, 0, 0], - [1, 1, 0], - [1, 0, 3], - ]) - for (const n of neighbors) expect(inBounds(n)).toBe(true) - }) - - it('clamps at the ceiling: upward moves that clamp to a no-op are dropped', () => { - const neighbors = enumerateNeighborPolicies({ k: 10, repairRounds: 3, testgen: 10 }) - expect(neighbors.map((n) => [n.k, n.repairRounds, n.testgen])).toEqual([ - [8, 3, 10], - [10, 2, 10], - [10, 3, 7], - ]) - for (const n of neighbors) expect(inBounds(n)).toBe(true) - }) - - it('a clamped move that lands in-bounds survives (k=2 reaches the k=1 preset)', () => { - const neighbors = enumerateNeighborPolicies({ k: 2, repairRounds: 0, testgen: 0 }) - expect(neighbors.map((n) => n.k)).toContain(1) - for (const n of neighbors) expect(inBounds(n)).toBe(true) - }) - - it('never mutates diverse/temperature — they ride through every neighbor', () => { - const neighbors = enumerateNeighborPolicies({ - k: 5, - repairRounds: 2, - testgen: 6, - diverse: true, - temperature: 0.7, - }) - for (const n of neighbors) { - expect(n.diverse).toBe(true) - expect(n.temperature).toBe(0.7) - } - }) -}) - -describe('rolloutPolicyProposer — deterministic bounded proposal', () => { - const ctx = (overrides: Record = {}) => - ({ - currentSurface: serializeRolloutPolicy({ k: 5, repairRounds: 2, testgen: 6 }), - history: [], - findings: [], - populationSize: 4, - generation: 0, - signal: new AbortController().signal, - ...overrides, - }) as never - - const asProposed = (proposals: Array): ProposedCandidate[] => - proposals.filter((p): p is ProposedCandidate => isProposedCandidate(p as ProposedCandidate)) - - it('caps at min(populationSize, 4) candidates, every surface a valid in-bounds policy', async () => { - const raw = await rolloutPolicyProposer().propose(ctx()) - const proposals = asProposed(raw) - // Every proposal carries its {label, rationale} — never a bare surface. - expect(proposals.length).toBe(raw.length) - expect(proposals.length).toBe(4) - for (const p of proposals) { - const parsed = parseRolloutPolicy(p.surface) - expect(parsed).toBeDefined() - expect(inBounds(parsed as StructuralRolloutPolicy)).toBe(true) - expect(p.label).toMatch(/^(k|repairRounds|testgen) \d+→\d+$/) - expect(p.rationale.length).toBeGreaterThan(0) - } - const two = await rolloutPolicyProposer().propose(ctx({ populationSize: 2 })) - expect(two.length).toBe(2) - }) - - it('rotates the neighbor window by generation, so a held generation explores differently', async () => { - const gen0 = asProposed(await rolloutPolicyProposer().propose(ctx({ generation: 0 }))) - const gen1 = asProposed(await rolloutPolicyProposer().propose(ctx({ generation: 1 }))) - expect(new Set(gen0.map((p) => p.surface))).not.toEqual(new Set(gen1.map((p) => p.surface))) - }) - - it('proposes nothing when the surface carries no policy (empty, malformed, or code-tier)', async () => { - const proposer = rolloutPolicyProposer() - expect(await proposer.propose(ctx({ currentSurface: '' }))).toEqual([]) - expect(await proposer.propose(ctx({ currentSurface: 'not json' }))).toEqual([]) - expect(await proposer.propose(ctx({ currentSurface: '{"k": 0}' }))).toEqual([]) - expect(await proposer.propose(ctx({ currentSurface: { worktreeRef: 'refs/x' } }))).toEqual([]) - }) -}) - -describe('policy persistence — profile extensions round-trip', () => { - it('applyRolloutPolicyToProfile → structuralRolloutPolicyFromProfile is identity', () => { +describe('rollout policy profile coordinate', () => { + it('serializes, parses, and applies an exact policy without mutating the profile', () => { const policy: StructuralRolloutPolicy = { k: 3, repairRounds: 1, @@ -152,13 +26,14 @@ describe('policy persistence — profile extensions round-trip', () => { temperature: 0.2, } const profile: AgentProfile = { name: 'fixture' } - const next = applyRolloutPolicyToProfile(profile, policy) + const serialized = serializeRolloutPolicy(policy) + const next = applyRolloutPolicyToProfile(profile, parseRolloutPolicy(serialized)!) + expect(structuralRolloutPolicyFromProfile(next)).toEqual(policy) - // Never mutates the input profile. expect(profile.extensions).toBeUndefined() }) - it('a partial extension resolves the missing dials to the measured defaults', () => { + it('fills omitted dials and rejects corrupt policy values', () => { const profile: AgentProfile = { extensions: { [ROLLOUT_POLICY_EXTENSION]: { k: 1 } }, } @@ -167,90 +42,87 @@ describe('policy persistence — profile extensions round-trip', () => { repairRounds: 2, testgen: 6, }) - }) - it('a corrupt extension reads as not-configured, never a fabricated recipe', () => { for (const bad of [{ k: 0 }, { k: 1.5 }, { repairRounds: -1 }, { testgen: 'six' }]) { - const profile: AgentProfile = { - extensions: { [ROLLOUT_POLICY_EXTENSION]: bad as never }, - } - expect(structuralRolloutPolicyFromProfile(profile)).toBeUndefined() + expect( + structuralRolloutPolicyFromProfile({ + extensions: { [ROLLOUT_POLICY_EXTENSION]: bad as never }, + }), + ).toBeUndefined() } expect(normalizeRolloutPolicy(null)).toBeUndefined() expect(normalizeRolloutPolicy([1, 2])).toBeUndefined() }) }) -// ── improve() end-to-end: the gate decides, the winner persists ───────────────────── - -// Eight scenarios at holdoutFraction 0.5 → 4 train + 4 holdout: enough held-out -// cells for the gate statistic on a deterministic score gradient. -const scenarios: Scenario[] = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'].map((id) => ({ - id, - kind: 'fixture', -})) +interface PolicyScenario extends Scenario { + kind: 'fixture' +} -// The agent parses the policy surface into the artifact; the judge rewards the k -// dial (composite = k/10). Baseline k=5 → 0.5; the k=7 neighbor → 0.7 on every -// scenario: a +0.2 held-out lift the default gate (deltaThreshold 0.05) ships. -async function policyAgent( - surface: unknown, - _scenario: Scenario, - ctx: DispatchContext, -): Promise<{ policy: StructuralRolloutPolicy | undefined }> { - const paid = await ctx.cost.runPaidCall({ - channel: 'agent', - actor: 'stub-agent', - model: 'deterministic-test', - maximumCharge: { externallyEnforcedMaximumUsd: 0.0001 }, - execute: async () => ({ policy: parseRolloutPolicy(String(surface)) }), - receipt: () => ({ - model: 'deterministic-test', - inputTokens: 1, - outputTokens: 1, - actualCostUsd: 0.0001, - }), - }) - if (!paid.succeeded) throw paid.error - return paid.value +interface PolicyArtifact { + policy?: StructuralRolloutPolicy } -const kJudge: JudgeConfig<{ policy: StructuralRolloutPolicy | undefined }, Scenario> = { - name: 'k-dial-judge', - dimensions: [{ key: 'q', description: 'rewards selection breadth' }], +const train: PolicyScenario[] = [{ id: 'train', kind: 'fixture' }] +const selection: PolicyScenario[] = [{ id: 'selection', kind: 'fixture' }] +const testCases: PolicyScenario[] = [ + { id: 'test-a', kind: 'fixture' }, + { id: 'test-b', kind: 'fixture' }, +] + +const judge: JudgeConfig = { + name: 'k', + dimensions: [{ key: 'quality', description: 'selection breadth' }], score: ({ artifact }) => { - const composite = (artifact.policy?.k ?? 0) / 10 - return { dimensions: { q: composite }, composite, notes: '' } + const quality = (artifact.policy?.k ?? 0) / 10 + return { dimensions: { quality }, composite: quality, notes: '' } }, } -describe("improve() surface 'rollout-policy'", () => { - it('a gated win persists the winning policy into profile.extensions', async () => { +function policyMethod( + policy: StructuralRolloutPolicy, +): OptimizationMethod { + return { + name: 'external-policy-search', + async optimize() { + return { + winnerSurface: serializeRolloutPolicy(policy), + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + } + }, + } +} + +describe("improve surface 'rollout-policy'", () => { + it('applies a complete method result to profile.extensions', async () => { const profile: AgentProfile = { name: 'fixture-agent', extensions: { [ROLLOUT_POLICY_EXTENSION]: { k: 5, repairRounds: 2, testgen: 6 }, }, } - const result = await improve(profile, [], { + const result = await improve(profile, { surface: 'rollout-policy', - scenarios, - judge: kJudge, - agent: policyAgent, - budget: { generations: 1, populationSize: 4, holdoutFraction: 0.5 }, + method: policyMethod({ k: 7, repairRounds: 2, testgen: 6 }), + trainScenarios: train, + selectionScenarios: selection, + testScenarios: testCases, + judges: [judge], + agent: async (surface) => ({ policy: parseRolloutPolicy(surface) }), + runDir: 'mem://rollout-policy-method', + storage: inMemoryCampaignStorage(), + resamples: 40, + confidence: 0.95, + expectUsage: 'off', }) expect(result.decision).toBe('ship') expect(result.lift).toBeCloseTo(0.2, 5) - // The k=7 neighbor won and was written back where the runtime reads it. - const candidateProfile = result.candidate.profile - if (!candidateProfile) throw new Error('expected a rollout-policy profile candidate') - expect(structuralRolloutPolicyFromProfile(candidateProfile)).toEqual({ + expect(structuralRolloutPolicyFromProfile(result.candidate.profile!)).toEqual({ k: 7, repairRounds: 2, testgen: 6, }) - // Input profile untouched (applyWinnerToProfile is copy-on-write). expect(structuralRolloutPolicyFromProfile(profile)).toEqual({ k: 5, repairRounds: 2, @@ -258,20 +130,25 @@ describe("improve() surface 'rollout-policy'", () => { }) }) - it('no-ops when the profile has no structural rollout config', async () => { - const profile: AgentProfile = { name: 'fixture-agent', prompt: { systemPrompt: 'base' } } - const result = await improve(profile, [], { - surface: 'rollout-policy', - scenarios, - judge: kJudge, - agent: policyAgent, - budget: { generations: 1, populationSize: 4, holdoutFraction: 0.5 }, - }) - - // The proposer proposed nothing, so nothing could ship; the winning surface - // is the (empty) baseline, so no fabricated extension and no dial changes. - expect(result.decision).toBe('hold') - const heldProfile = result.candidate.profile ?? profile - expect(structuralRolloutPolicyFromProfile(heldProfile)).toBeUndefined() + it('rejects optimization when the profile has no active rollout policy', async () => { + await expect( + improve( + { name: 'fixture-agent' }, + { + surface: 'rollout-policy', + method: policyMethod({ k: 7, repairRounds: 2, testgen: 6 }), + trainScenarios: train, + selectionScenarios: selection, + testScenarios: testCases, + judges: [judge], + agent: async (surface) => ({ policy: parseRolloutPolicy(surface) }), + runDir: 'mem://rollout-policy-missing', + storage: inMemoryCampaignStorage(), + resamples: 40, + confidence: 0.95, + expectUsage: 'off', + }, + ), + ).rejects.toThrow(/requires an existing structural rollout policy/) }) }) diff --git a/src/improvement/rollout-policy.ts b/src/improvement/rollout-policy.ts index 05a6edc5..9a11cac9 100644 --- a/src/improvement/rollout-policy.ts +++ b/src/improvement/rollout-policy.ts @@ -1,34 +1,6 @@ -/** - * `rolloutPolicyProposer` — the `'rollout-policy'` surface for `improve()`: the - * inference-time `StructuralRolloutPolicy` dials { k, repairRounds, testgen } as a - * held-out-gated optimizable surface. - * - * Why this seam: agent-eval's loop contract is already generic — `MutableSurface` - * admits any string, documented as "serialized tool config" — so the policy rides - * the SAME serialize→propose→gate→parse-back cycle the tools/mcp/hooks surfaces - * use. No agent-eval changes; the only net-new piece is this proposer. - * - * Why deterministic: prompt-wording proposals are a measured zero on this stack, - * and the policy space is tiny and fully enumerable. The proposer emits bounded - * single-dial neighbors (k±2 in [1,10], repairRounds±1 in [0,3], testgen±3 in - * [0,10], ≤4 per generation) and lets the held-out gate do ALL the deciding — an - * LLM proposer would add cost and nondeterminism with nothing to reason about. - * - * Persistence: the policy lives in `profile.extensions['structural-rollout']` - * (AgentProfile's designed slot for runtime-specific config). A gated winner is - * written back there by `improve()`, the same profile-field write-back every other - * config surface gets; `structuralRolloutPolicyFromProfile` is the read side a - * runtime caller feeds to `structuralRollout({ policy })`. - * - * @experimental - */ +/** Serialize and apply Runtime's structural rollout profile coordinate. */ -import type { - MutableSurface, - ProposeContext, - ProposedCandidate, - SurfaceProposer, -} from '@tangle-network/agent-eval/campaign' +import type { MutableSurface } from '@tangle-network/agent-eval/contract' import type { AgentProfile } from '@tangle-network/agent-interface' import { defaultStructuralRolloutPolicy, @@ -38,31 +10,12 @@ import { /** The profile extensions namespace the policy persists under. */ export const ROLLOUT_POLICY_EXTENSION = 'structural-rollout' -/** Proposal bounds per dial. These are the SEARCH bounds (what the proposer may - * explore), chosen so every reachable value is a measured-sane recipe: k=1 is the - * low-compute preset, testgen=0 disables check authoring, repairRounds caps where - * the measured increment flattens (+1–3pp beyond round 2). */ -export const ROLLOUT_POLICY_BOUNDS = { - k: { min: 1, max: 10, step: 2 }, - repairRounds: { min: 0, max: 3, step: 1 }, - testgen: { min: 0, max: 10, step: 3 }, -} as const - -/** Max candidates per generation — the search space is 3 dials, so a small - * neighborhood per generation converges without burning gate budget. */ -const MAX_CANDIDATES_PER_GENERATION = 4 - -const clamp = (v: number, min: number, max: number) => Math.min(max, Math.max(min, v)) - const isBoundedInt = (v: unknown, min: number): v is number => typeof v === 'number' && Number.isInteger(v) && v >= min -/** Parse a serialized policy surface. Defensive by design — the proposer reads - * `ctx.currentSurface`, which the loop types as `string | CodeSurface`. Returns - * `undefined` (never throws) for non-strings, malformed JSON, or a shape that - * violates the policy's own invariants: the no-op signal. Unknown dials are - * dropped; `diverse`/`temperature` ride through untouched (the proposer never - * mutates them — `diverse` is a measured paired null). */ +/** Parse a serialized policy surface. Returns `undefined` for non-strings, + * malformed JSON, or values outside the policy invariants. Unknown fields are + * dropped; supported optional fields are preserved. */ export function parseRolloutPolicy(surface: MutableSurface): StructuralRolloutPolicy | undefined { if (typeof surface !== 'string' || surface.trim().length === 0) return undefined let raw: unknown @@ -97,8 +50,7 @@ export function normalizeRolloutPolicy(raw: unknown): StructuralRolloutPolicy | } } -/** Stable serialization — dial order is fixed so identical policies produce - * identical surfaces (the loop dedupes/hashes candidates by surface content). */ +/** Stable serialization with fixed field order. */ export function serializeRolloutPolicy(policy: StructuralRolloutPolicy): string { return JSON.stringify({ k: policy.k, @@ -110,8 +62,7 @@ export function serializeRolloutPolicy(policy: StructuralRolloutPolicy): string } /** Read the persisted policy off the profile. `undefined` when the profile does - * not opt into structural rollout — the improve() surface no-ops then, because - * tuning dials nothing consumes would ship dead config. */ + * not opt into structural rollout. */ export function structuralRolloutPolicyFromProfile( profile: AgentProfile, ): StructuralRolloutPolicy | undefined { @@ -120,8 +71,7 @@ export function structuralRolloutPolicyFromProfile( return normalizeRolloutPolicy(bag) } -/** Persist a policy into the profile's extensions namespace. Shallow copy; never - * mutates the input profile (the applyWinnerToProfile contract). */ +/** Persist a detached policy under the profile extension without mutating the input. */ export function applyRolloutPolicyToProfile( profile: AgentProfile, policy: StructuralRolloutPolicy, @@ -138,75 +88,3 @@ export function applyRolloutPolicyToProfile( extensions: { ...profile.extensions, [ROLLOUT_POLICY_EXTENSION]: bag }, } } - -/** All bounded single-dial neighbors of `policy`, in a fixed priority order: k - * first (selection breadth carries 85–92% of the measured effect), then - * repairRounds, then testgen. Steps clamp to the dial's bounds; clamped-to-no-op - * and duplicate policies are dropped. */ -export function enumerateNeighborPolicies( - policy: StructuralRolloutPolicy, -): StructuralRolloutPolicy[] { - const moves: Array<{ dial: 'k' | 'repairRounds' | 'testgen'; delta: 1 | -1 }> = [ - { dial: 'k', delta: 1 }, - { dial: 'k', delta: -1 }, - { dial: 'repairRounds', delta: 1 }, - { dial: 'repairRounds', delta: -1 }, - { dial: 'testgen', delta: 1 }, - { dial: 'testgen', delta: -1 }, - ] - const seen = new Set([serializeRolloutPolicy(policy)]) - const neighbors: StructuralRolloutPolicy[] = [] - for (const move of moves) { - const bounds = ROLLOUT_POLICY_BOUNDS[move.dial] - const next = clamp(policy[move.dial] + move.delta * bounds.step, bounds.min, bounds.max) - const candidate: StructuralRolloutPolicy = { ...policy, [move.dial]: next } - const key = serializeRolloutPolicy(candidate) - if (seen.has(key)) continue - seen.add(key) - neighbors.push(candidate) - } - return neighbors -} - -function candidateLabel(base: StructuralRolloutPolicy, next: StructuralRolloutPolicy): string { - for (const dial of ['k', 'repairRounds', 'testgen'] as const) { - if (next[dial] !== base[dial]) return `${dial} ${base[dial]}→${next[dial]}` - } - return 'unchanged' -} - -/** - * The deterministic `SurfaceProposer` for the `'rollout-policy'` surface. - * - * Each generation: parse the current policy surface, enumerate its bounded - * single-dial neighbors, and return at most `min(populationSize, 4)` of them, - * rotating the enumeration window by generation so successive generations explore - * different neighbors when nothing promoted. Proposes NOTHING when the surface - * carries no policy (the profile never opted in) — an empty proposal is the - * loop-native no-op, mirroring `improvementDriver`'s no-findings behavior. - */ -export function rolloutPolicyProposer(): SurfaceProposer { - return { - kind: 'rollout-policy', - async propose(ctx: ProposeContext): Promise { - const policy = parseRolloutPolicy(ctx.currentSurface) - if (!policy) return [] - const neighbors = enumerateNeighborPolicies(policy) - if (neighbors.length === 0) return [] - const cap = Math.max(1, Math.min(ctx.populationSize, MAX_CANDIDATES_PER_GENERATION)) - const start = (ctx.generation * cap) % neighbors.length - const window: StructuralRolloutPolicy[] = [] - for (let i = 0; i < Math.min(cap, neighbors.length); i += 1) { - window.push(neighbors[(start + i) % neighbors.length] as StructuralRolloutPolicy) - } - return window.map((candidate) => ({ - surface: serializeRolloutPolicy(candidate), - label: candidateLabel(policy, candidate), - rationale: - 'bounded single-dial neighbor of the current structuralRollout policy; ' + - 'the held-out gate decides (deterministic enumeration — the dial space is tiny ' + - 'and prompt-style reflective proposals are a measured zero here)', - })) - }, - } -} diff --git a/src/index.ts b/src/index.ts index 9482c9b1..9011baa0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -117,14 +117,14 @@ export { ValidationError, } from './errors' // ── Improvement (self-improvement surfaces) ────────────────────────── -// `improve` is the one pluggable RSI verb (facade over agent-eval's -// `selfImprove`); the rest are the code-surface driver + its generators. +// Complete agent-eval methods optimize profile fields. Runtime owns only +// isolated code/worktree candidate execution. export * from './improvement' // ── Knowledge orchestration ────────────────────────────────────────── // Runtime owns live agent orchestration; agent-knowledge owns the KB/RAG/memory state. // These wrappers bridge the two without making agent-knowledge import runtime. export * from './knowledge' -// ── Delegated loop-runner (configured code/research/review/audit/self-improve) ── +// ── Delegated loop-runner (configured code/research/review/audit/improvement) ── export { auditLoopRunner, DELEGATED_LOOP_MODES, @@ -138,7 +138,6 @@ export { type RunDelegatedLoopOptions, researchLoopRunner, runDelegatedLoop, - selfImproveLoopRunner, type VetoedFact, type WorktreeLoopRunnerOptions, worktreeLoopRunner, diff --git a/src/intelligence/improvement-cycle.ts b/src/intelligence/improvement-cycle.ts index 66cac56b..93bada42 100644 --- a/src/intelligence/improvement-cycle.ts +++ b/src/intelligence/improvement-cycle.ts @@ -205,8 +205,8 @@ function sealAgentImprovementExperiment( const candidateLineage: AgentCandidateLineage = { source: 'optimizer', parentDigests: [material.baseline.digest], - runIds: [improvement.raw.provenance.runId], - developmentSplitDigest: improvement.raw.provenance.evidence.search.splitDigest, + runIds: [improvement.lineage.runId], + developmentSplitDigest: improvement.lineage.developmentSplitDigest, } return sealCandidateExperiment({ ...material, candidateLineage }) } @@ -332,7 +332,10 @@ export async function proposeAgentImprovement( - options: SelfImproveOptions, -): DelegatedLoopRunner> { - return async () => selfImprove(options) -} - /** * `audit` mode — analyst loop over captured trace/run data. * diff --git a/src/mcp/memory-server.ts b/src/mcp/memory-server.ts index 60b3f2e5..db097420 100644 --- a/src/mcp/memory-server.ts +++ b/src/mcp/memory-server.ts @@ -10,8 +10,7 @@ * * Retrieval is DETERMINISTIC lexical overlap (no vectors, no LLM): a lift * measured with this memory mounted is attributable to the lessons - * themselves, never to retrieval-model noise — the same discipline as - * agent-eval's `memoryCurationProposer` (deterministic curation). Every + * themselves, never to retrieval-model noise. Every * `memory_search` can append one JSONL row to a retrieval log (`logPath`) — * the per-query record an off-policy retrieval estimator (agent-knowledge's * `RetrievalHoldout`) consumes. agent-knowledge is NOT a dependency of this diff --git a/src/runtime/structural-rollout.ts b/src/runtime/structural-rollout.ts index acd14685..cc9d4ac9 100644 --- a/src/runtime/structural-rollout.ts +++ b/src/runtime/structural-rollout.ts @@ -20,7 +20,7 @@ * poison repair at saturation — the glm /47,/116 regressions). * * Placement rule: this is an INFERENCE-TIME capability (it wraps the model call via the - * strategy seam). It does not belong in improve()/selfImprove (training-time); improve() + * strategy seam). It does not belong in `improve()` (training-time); `improve()` * may later tune `StructuralRolloutPolicy` as an optimizable surface. */ diff --git a/src/runtime/supervise-surface.ts b/src/runtime/supervise-surface.ts index 9ef67f52..f5a7d0a0 100644 --- a/src/runtime/supervise-surface.ts +++ b/src/runtime/supervise-surface.ts @@ -11,7 +11,7 @@ * (`strategy.ts` → `supervise/`), so a surface-solving worker cannot be a supervise built-in without an * import cycle. It is therefore a COMPOSITION of `supervise()` + `runAgentic` at the layer above both — * the right home for "supervise over a graded surface". The within-run self-improvement is the analyst - * (authored content, swap `analysts`); the across-run kind wraps this call in `improve()`/`selfImprove`. + * (authored content, swap `analysts`); the across-run kind wraps this call in `improve()`. */ import type { AgentProfile } from '@tangle-network/sandbox' import type { AnalystRegistry, MakeWorkerAgent } from '../mcp/tools/coordination' diff --git a/tests/improve.test.ts b/tests/improve.test.ts deleted file mode 100644 index 1f30d3b2..00000000 --- a/tests/improve.test.ts +++ /dev/null @@ -1,205 +0,0 @@ -import type { - DispatchContext, - JudgeConfig, - MutableSurface, - Scenario, - SurfaceProposer, -} from '@tangle-network/agent-eval/contract' -import type { AgentProfile } from '@tangle-network/agent-interface' -import { describe, expect, it } from 'vitest' -import { ConfigError } from '../src/errors' -import { improve } from '../src/improvement' - -interface DemoScenario extends Scenario { - kind: 'demo' -} - -const scenarios: DemoScenario[] = Array.from({ length: 12 }, (_, i) => ({ - id: `s${i}`, - kind: 'demo' as const, -})) - -/** The agent returns the surface verbatim as the artifact AND reports token - * usage so agent-eval's backend-integrity guard (`expectUsage: 'assert'`, the - * default) sees a real backend rather than a stub-zero cell. No LLM. */ -const agent = async ( - surface: MutableSurface, - _scenario: DemoScenario, - ctx: DispatchContext, -): Promise => { - const paid = await ctx.cost.runPaidCall({ - channel: 'agent', - actor: 'test', - model: 'deterministic-test', - maximumCharge: { externallyEnforcedMaximumUsd: 0.0001 }, - execute: async () => String(surface), - receipt: () => ({ - model: 'deterministic-test', - inputTokens: 1, - outputTokens: 1, - actualCostUsd: 0.0001, - }), - }) - if (!paid.succeeded) throw paid.error - return paid.value -} - -/** Deterministic judge: the literal string `PROMOTED` scores 1.0, anything else - * 0.0 — no LLM, no opinion. */ -const judge: JudgeConfig = { - name: 'literal', - dimensions: [{ key: 'q', description: 'q' }], - score: ({ artifact }) => { - const composite = artifact.includes('PROMOTED') ? 1 : 0 - return { dimensions: { q: composite }, composite, notes: '' } - }, -} - -/** A scripted `SurfaceProposer` that always proposes the winning surface — - * the hand-written stand-in for `gepaProposer`, no router call. */ -const scriptedWinner: SurfaceProposer = { - kind: 'scripted-winner', - async propose() { - return [{ surface: 'PROMOTED', label: 'win', rationale: 'scripted' }] - }, -} - -function profileWith(systemPrompt: string): AgentProfile { - return { name: 'demo', prompt: { systemPrompt } } -} - -describe('improve() — facade over selfImprove', () => { - it('gate:none returns a detached baseline candidate', async () => { - const profile = profileWith('BASELINE') - const out = await improve(profile, [], { - surface: 'prompt', - gate: 'none', - generator: scriptedWinner, - scenarios, - judge, - agent, - }) - - expect(out.decision).not.toBe('ship') - expect(out.candidate).toMatchObject({ - surface: 'prompt', - value: 'BASELINE', - profile: { prompt: { systemPrompt: 'BASELINE' } }, - }) - expect(profile.prompt?.systemPrompt).toBe('BASELINE') - }) - - it('forwards dispatchTimeoutMs through improve() to abort a stalled cell', async () => { - let observedAbort = false - const stalledAgent = ( - _surface: MutableSurface, - _scenario: DemoScenario, - ctx: DispatchContext, - ): Promise => - new Promise(() => { - ctx.signal.addEventListener( - 'abort', - () => { - observedAbort = true - }, - { once: true }, - ) - }) - - await expect( - improve(profileWith('BASELINE'), [], { - surface: 'prompt', - gate: 'none', - generator: scriptedWinner, - scenarios: scenarios.slice(0, 2), - judge, - agent: stalledAgent, - budget: { holdoutFraction: 0.5 }, - dispatchTimeoutMs: 25, - expectUsage: 'off', - maxConcurrency: 1, - }), - ).rejects.toThrow(/dispatch exceeded 25ms/) - - expect(observedAbort).toBe(true) - }) - - it('returns a frozen prompt candidate on a ship decision without changing the profile', async () => { - const profile = profileWith('BASELINE') - const out = await improve(profile, [], { - surface: 'prompt', - generator: scriptedWinner, - scenarios, - judge, - agent, - // A perfect +1.0 lift at this n/reps clears the default held-out gate. - budget: { generations: 1, populationSize: 2, reps: 3, holdoutFraction: 0.5 }, - }) - - expect(out.decision).toBe('ship') - expect(out.lift).toBeGreaterThan(0) - expect(out.candidate.value).toBe('PROMOTED') - expect(out.candidate.profile?.prompt?.systemPrompt).toBe('PROMOTED') - expect(Object.isFrozen(out.candidate)).toBe(true) - expect(Object.isFrozen(out.candidate.profile?.prompt)).toBe(true) - expect(profile.prompt?.systemPrompt).toBe('BASELINE') - }) - - it('throws ConfigError for a surface with no default generator and no generator passed', async () => { - const profile = profileWith('BASELINE') - await expect( - improve(profile, [], { surface: 'tools', scenarios, judge, agent }), - ).rejects.toBeInstanceOf(ConfigError) - }) - - it('allowedModels throws ConfigError when the reflection model is outside the set', async () => { - const profile = profileWith('BASELINE') - await expect( - improve(profile, [], { - surface: 'prompt', - gate: 'none', - scenarios, - judge, - agent, - llm: { model: 'gpt-4.1' }, - allowedModels: ['deepseek-v4-flash'], - }), - ).rejects.toBeInstanceOf(ConfigError) - }) - - it('allowedModels passes when the reflection model is in the set', async () => { - const profile = profileWith('BASELINE') - const out = await improve(profile, [], { - surface: 'prompt', - gate: 'none', - scenarios, - judge, - agent, - llm: { model: 'deepseek-v4-flash' }, - allowedModels: ['deepseek-v4-flash'], - }) - expect(out.decision).not.toBe('ship') - }) - - it('throws ConfigError when a JSON-surface candidate is not valid JSON', async () => { - const profile: AgentProfile = { name: 'demo', tools: {} } - // Scores 1.0 but is not valid JSON for the `tools` surface. Candidate - // materialization must fail with a typed error, not a raw SyntaxError. - const malformedWinner: SurfaceProposer = { - kind: 'malformed-json', - async propose() { - return [{ surface: 'PROMOTED not json {{{', label: 'win', rationale: 'x' }] - }, - } - await expect( - improve(profile, [], { - surface: 'tools', - generator: malformedWinner, - scenarios, - judge, - agent, - budget: { generations: 1, populationSize: 2, reps: 3, holdoutFraction: 0.5 }, - }), - ).rejects.toBeInstanceOf(ConfigError) - }) -}) diff --git a/tests/improvement-cycle.test.ts b/tests/improvement-cycle.test.ts index 50913c58..bc6496a5 100644 --- a/tests/improvement-cycle.test.ts +++ b/tests/improvement-cycle.test.ts @@ -1,10 +1,10 @@ import type { AnalystFinding } from '@tangle-network/agent-eval' +import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' import type { DispatchContext, JudgeConfig, MutableSurface, Scenario, - SurfaceProposer, } from '@tangle-network/agent-eval/contract' import { sealCandidateBenchmarkSuite, @@ -20,6 +20,7 @@ import { agentCandidateProfileAsAgentProfile, freezeGenericAgentCandidateProfile, } from '../src/candidate-execution/profile' +import type { ImproveMethodFactory } from '../src/improvement' import { createAgentImprovementActivationResult, executeAgentImprovementActivation, @@ -77,6 +78,9 @@ const improvementScenarios: ImprovementScenario[] = Array.from({ length: 12 }, ( id: `improvement-${index}`, kind: 'improvement-test', })) +const improvementTrain = improvementScenarios.slice(0, 4) +const improvementSelection = improvementScenarios.slice(4, 8) +const improvementTest = improvementScenarios.slice(8) const improvementAgent = async ( surface: MutableSurface, @@ -109,11 +113,35 @@ const improvementJudge: JudgeConfig = { }, } -const improvementProposer: SurfaceProposer = { - kind: 'deterministic-candidate', - propose: async () => [ - { surface: 'PROMOTED', label: 'measured candidate', rationale: 'test fixture' }, - ], +const improvementMethod: ImproveMethodFactory = (context) => ({ + name: 'deterministic-complete-method', + async optimize() { + if ( + !context.findings.some((item) => (item as { finding_id?: string }).finding_id === 'finding-1') + ) { + throw new Error('analysis findings did not reach the optimization method') + } + return { + winnerSurface: 'PROMOTED', + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + } + }, +}) + +function improvementOptions() { + return { + surface: 'prompt' as const, + method: improvementMethod, + trainScenarios: improvementTrain, + selectionScenarios: improvementSelection, + testScenarios: improvementTest, + judges: [improvementJudge], + agent: improvementAgent, + runDir: `mem://improvement-cycle-${Math.random()}`, + storage: inMemoryCampaignStorage(), + resamples: 40, + confidence: 0.95, + } } afterEach(() => { @@ -153,14 +181,7 @@ describe('agent improvement lifecycle', () => { findingsStore: null, log: () => {}, }, - improvement: { - surface: 'prompt', - generator: improvementProposer, - scenarios: improvementScenarios, - judge: improvementJudge, - agent: improvementAgent, - budget: { generations: 1, populationSize: 2, reps: 3, holdoutFraction: 0.5 }, - }, + improvement: improvementOptions(), buildExperiment: ({ improvement }) => { if (!improvement.candidate.profile) throw new Error('expected a profile candidate') const candidate = redigestCandidateBundle(seed.experiment.baseline, { @@ -223,8 +244,8 @@ describe('agent improvement lifecycle', () => { expect(result.experiment.candidateLineage).toEqual({ source: 'optimizer', parentDigests: [measured.experiment.baseline.digest], - runIds: [result.improvement.raw.provenance.runId], - developmentSplitDigest: result.improvement.raw.provenance.evidence.search.splitDigest, + runIds: [result.improvement.lineage.runId], + developmentSplitDigest: result.improvement.lineage.developmentSplitDigest, }) expect(activation.targets[0].expectedBaseDigest).toBe(promptSurfaceDigest(measured)) expect(activationResult.outcome.status).toBe('applied') @@ -257,14 +278,7 @@ describe('agent improvement lifecycle', () => { findingsStore: null, log: () => {}, }, - improvement: { - surface: 'prompt', - generator: improvementProposer, - scenarios: improvementScenarios, - judge: improvementJudge, - agent: improvementAgent, - budget: { generations: 1, populationSize: 2, reps: 3, holdoutFraction: 0.5 }, - }, + improvement: improvementOptions(), buildExperiment: ({ improvement }) => { if (!improvement.candidate.profile) throw new Error('expected a profile candidate') const substituted = { @@ -334,14 +348,7 @@ describe('agent improvement lifecycle', () => { findingsStore: null, log: () => {}, }, - improvement: { - surface: 'prompt', - generator: improvementProposer, - scenarios: improvementScenarios, - judge: improvementJudge, - agent: improvementAgent, - budget: { generations: 1, populationSize: 2, reps: 3, holdoutFraction: 0.5 }, - }, + improvement: improvementOptions(), buildExperiment: ({ improvement }) => { if (!improvement.candidate.profile) throw new Error('expected a profile candidate') const candidate = redigestCandidateBundle(seed.experiment.baseline, { @@ -387,14 +394,7 @@ describe('agent improvement lifecycle', () => { findingsStore: null, log: () => {}, }, - improvement: { - surface: 'prompt', - generator: improvementProposer, - scenarios: improvementScenarios, - judge: improvementJudge, - agent: improvementAgent, - budget: { generations: 1, populationSize: 2, reps: 3, holdoutFraction: 0.5 }, - }, + improvement: improvementOptions(), buildExperiment: ({ improvement }) => { if (!improvement.candidate.profile) throw new Error('expected a profile candidate') const candidate = redigestCandidateBundle(seed.experiment.baseline, { @@ -411,7 +411,7 @@ describe('agent improvement lifecycle', () => { ...taskMaterial, benchmark: { ...task.benchmark, - splitDigest: improvement.raw.provenance.evidence.search.splitDigest, + splitDigest: improvement.lineage.developmentSplitDigest, }, }) return { diff --git a/tests/improvement-driver.test.ts b/tests/improvement-driver.test.ts index 792e504b..a5a9f4ab 100644 --- a/tests/improvement-driver.test.ts +++ b/tests/improvement-driver.test.ts @@ -13,8 +13,9 @@ import { import { afterEach, beforeEach, describe, expect, it } from 'vitest' import type { SurfaceImprovementEdit } from '../src/agent/improvement-adapter' import type { ImprovementEditBatch, ImprovementProposalSource } from '../src/analyst-loop/types' -import { improvementDriver, reflectiveGenerator } from '../src/improvement' import type { CandidateGenerator } from '../src/improvement/improvement-driver' +import { improvementDriver } from '../src/improvement/improvement-driver' +import { reflectiveGenerator } from '../src/improvement/reflective-generator' function git(args: string[], cwd: string): string { return execFileSync('git', args, { cwd, encoding: 'utf8' }).trim() diff --git a/tests/loop-runner.test.ts b/tests/loop-runner.test.ts index 58b02e07..f8500664 100644 --- a/tests/loop-runner.test.ts +++ b/tests/loop-runner.test.ts @@ -8,6 +8,11 @@ const clock = () => { } describe('runDelegatedLoop — mode dispatch', () => { + it('does not expose the legacy proposer-driven self-improvement factory', async () => { + const runtime = await import('../src/index') + expect(runtime).not.toHaveProperty('selfImproveLoopRunner') + }) + it('routes to the registered runner and returns a uniform ok result', async () => { const registry: DelegatedLoopRegistry = { research: async () => ({ grounded: 3 }), diff --git a/tests/profile-improvement-stack.test.ts b/tests/profile-improvement-stack.test.ts index 8564163d..c2f0e7e8 100644 --- a/tests/profile-improvement-stack.test.ts +++ b/tests/profile-improvement-stack.test.ts @@ -1,20 +1,20 @@ import { - gepaProposer, - runImprovementLoop, - runOptimization, - skillOptProposer, + gepaOptimizationMethod, + inMemoryCampaignStorage, + type OptimizationMethod, + skillOptOptimizationMethod, } from '@tangle-network/agent-eval/campaign' import type { DispatchContext, JudgeConfig, MutableSurface, Scenario, - SurfaceProposer, } from '@tangle-network/agent-eval/contract' import type { AgentProfile } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../src/durable/spawn-journal' -import { improve } from '../src/improvement' +import * as runtimeImprovement from '../src/improvement' +import { improve, officialGepa, officialSkillOpt } from '../src/improvement' import { loopUntil } from '../src/runtime/personify/combinators' import { definePersona, runPersonified } from '../src/runtime/personify/persona' import type { Outcome } from '../src/runtime/personify/wave-types' @@ -36,23 +36,23 @@ const scenarios: ProfileScenario[] = Array.from({ length: 12 }, (_, i) => ({ id: `profile-${i}`, kind: 'profile-stack' as const, })) +const trainScenarios = scenarios.slice(0, 4) +const selectionScenarios = scenarios.slice(4, 8) +const testScenarios = scenarios.slice(8) const baseProfile = (): AgentProfile => ({ name: 'incident-responder', prompt: { systemPrompt: 'Handle the task directly.' }, }) -const improvingProposer: SurfaceProposer = { - kind: 'scripted-gepa-slot', - async propose() { - return [ - { - surface: - 'Handle the task directly.\n\nREPAIR_ON_FAILURE: after a failed draft, revise once using the failure signal.', - label: 'repair-on-failure', - rationale: 'Add the missing retry policy as a profile instruction.', - }, - ] +const improvingMethod: OptimizationMethod = { + name: 'scripted-complete-method', + async optimize() { + return { + winnerSurface: + 'Handle the task directly.\n\nREPAIR_ON_FAILURE: after a failed draft, revise once using the failure signal.', + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + } }, } @@ -168,22 +168,30 @@ function foldedAnswer(settled: Settled>): string | null return String((out as { answer: unknown }).answer) } -describe('profile improvement stack — agent-eval optimizer plus runtime loop', () => { - it('uses current agent-eval optimizer exports instead of local mutators', () => { - expect(typeof gepaProposer).toBe('function') - expect(typeof skillOptProposer).toBe('function') - expect(typeof runOptimization).toBe('function') - expect(typeof runImprovementLoop).toBe('function') +describe('profile improvement stack', () => { + it('exposes complete official methods without public proposer defaults', () => { + expect(typeof gepaOptimizationMethod).toBe('function') + expect(typeof skillOptOptimizationMethod).toBe('function') + expect(typeof officialGepa).toBe('function') + expect(typeof officialSkillOpt).toBe('function') + expect(runtimeImprovement).not.toHaveProperty('improvementDriver') + expect(runtimeImprovement).not.toHaveProperty('profileDiffProposer') }) it('runs a detached profile candidate through loopUntil()', async () => { - const improved = await improve(baseProfile(), [{ failure: 'single draft gets stuck' }], { + const improved = await improve(baseProfile(), { surface: 'prompt', - scenarios, - judge: promptJudge, + method: improvingMethod, + findings: [{ failure: 'single draft gets stuck' }], + trainScenarios, + selectionScenarios, + testScenarios, + judges: [promptJudge], agent: promptAgent, - generator: improvingProposer, - budget: { generations: 1, populationSize: 1, reps: 3, holdoutFraction: 0.5 }, + runDir: 'mem://profile-improvement-stack', + storage: inMemoryCampaignStorage(), + resamples: 40, + confidence: 0.95, }) expect(improved.decision).toBe('ship') From 561b4ebc5bb679021e79f8c2d03d432db04533fc Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 24 Jul 2026 01:47:25 -0600 Subject: [PATCH 04/14] chore(release): refresh proposal fixture --- src/testing/fixtures/agent-improvement-proposal.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index e79d1375..a3f7c8f6 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:e62706513d285835a6a447f33ee158ea2d6c3ce59cccd0ddf7029d3270ce083e", + "digest": "sha256:3bfcc7c157327b447ba74cd2cbbaab6e73c893d606dba8a4f70ba2f4752002de", "evaluation": { "decision": { "contributingChecks": [ @@ -2472,7 +2472,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.103.1" + "runtimeVersion": "0.104.0" }, "objectives": [ { @@ -2583,8 +2583,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:349ce5f4eed607ee58b901b239696c5b3b64d39d76f79b97c0910cc512336377", - "runId": "agent-runtime-0.103.1-proposal-fixture", + "recordDigest": "sha256:6de14fd199e2cd140dab91ef10a0f258732e990525a67783735ff2ec75e8358a", + "runId": "agent-runtime-0.104.0-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -2610,5 +2610,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.103.1-proposal-fixture" + "runId": "agent-runtime-0.104.0-proposal-fixture" } From 538818d61830c2e7a19ef16902d206ddb53be4d3 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 24 Jul 2026 01:49:57 -0600 Subject: [PATCH 05/14] docs(optimization): document official engine setup --- CHANGELOG.md | 8 ++++++++ README.md | 24 ++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4791e368..68e72935 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 0.104.0 + +- Add `officialGepa(...)` and `officialSkillOpt(...)` as Runtime adapters over the upstream GEPA and Microsoft SkillOpt implementations in `@tangle-network/agent-eval`. +- Require one complete `OptimizationMethod` for profile improvement and keep final-test scenarios outside optimizer input. +- Keep code improvement on Runtime-owned isolated Git worktrees. +- Remove the retired local prompt, profile-diff, campaign OTLP, and record-only optimizer paths. +- Require `@tangle-network/agent-eval` 0.126.x. + ## 0.103.1 - Declare and test compatibility with `@tangle-network/agent-eval` 0.125.x; runtime behavior is unchanged. diff --git a/README.md b/README.md index 1571c735..3ec2766e 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,25 @@ The profile is never changed. ```ts import { improve, officialGepa } from '@tangle-network/agent-runtime' +const optimizer = { + model: process.env.OPTIMIZER_MODEL!, + baseUrl: process.env.OPTIMIZER_BASE_URL!, + apiKey: process.env.OPTIMIZER_API_KEY!, + budget: { + maxCostUsd: 10, + maxRequests: 50, + maxRequestBytes: 2_000_000, + maxResponseBytes: 2_000_000, + maxOutputTokensPerRequest: 16_384, + pricing: { + inputUsdPerMillion: Number(process.env.OPTIMIZER_INPUT_USD_PER_MILLION), + cachedInputUsdPerMillion: Number(process.env.OPTIMIZER_CACHED_INPUT_USD_PER_MILLION), + cacheWriteUsdPerMillion: Number(process.env.OPTIMIZER_CACHE_WRITE_USD_PER_MILLION), + outputUsdPerMillion: Number(process.env.OPTIMIZER_OUTPUT_USD_PER_MILLION), + }, + }, +} + const result = await improve(baseProfile, { surface: 'prompt', method: officialGepa({ @@ -128,6 +147,7 @@ const result = await improve(baseProfile, { maxProposerCostUsd: 10, }, }, + optimizer, resume: 'if-compatible', describeScenario: ({ input }) => ({ input }), }), @@ -169,8 +189,8 @@ python -m pip install "skillopt @ git+https://github.com/microsoft/SkillOpt.git@ The source pins are deliberate. The published GEPA wheel lacks Optimize Anything, and the published SkillOpt wheel lacks files required by `ReflACTTrainer`. -SkillOpt requires `optimizer: { model, baseUrl, apiKey, budget }`. -GEPA accepts the same optional `optimizer` block for its standard reflection engine. +SkillOpt and GEPA's standard reflection engine require `optimizer: { model, baseUrl, apiKey, budget }`. +Agent-based GEPA engines may own their model connection instead. Agent Eval proxies those model calls, enforces the nested budget, and records their cost. SkillOpt accepts one text surface. From 3ae359a870743b796463f4de16fbaea34d3d1d2b Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 24 Jul 2026 01:50:11 -0600 Subject: [PATCH 06/14] fix(bench): restore strict type safety --- bench/src/gate.ts | 2 +- bench/src/hev-eval.mts | 7 +++-- bench/src/smoke-structural-rollout.mts | 24 ++++++++++------ bench/src/swe-local-proof.mts | 7 ++++- bench/src/swe-stream.mts | 6 ++-- bench/src/tb-container-executor.test.mts | 36 ++++++++++++++++++++---- bench/src/tb-supervisor-sidecar.mts | 3 +- 7 files changed, 63 insertions(+), 22 deletions(-) diff --git a/bench/src/gate.ts b/bench/src/gate.ts index 4014627d..9adb59f9 100644 --- a/bench/src/gate.ts +++ b/bench/src/gate.ts @@ -427,7 +427,7 @@ export async function runGate(opts: RunGateOptions): Promise { root: a.label, nodes: [], total: acc.get(a.label)!.spend, - statusCounts: { done: 0, failed: 0, cancelled: 0, pending: 0 }, + statusCounts: { done: 0, failed: 0, cancelled: 0, pending: 0, waiting: 0 }, }, })) const equalK = equalKOnCost(equalKArms) diff --git a/bench/src/hev-eval.mts b/bench/src/hev-eval.mts index dfba923a..07d6d50d 100644 --- a/bench/src/hev-eval.mts +++ b/bench/src/hev-eval.mts @@ -27,6 +27,7 @@ async function complete(base: string, key: string, model: string, prompt: string async function main(): Promise { const key = process.env.TANGLE_API_KEY if (!key) throw new Error('TANGLE_API_KEY required') + const apiKey: string = key const base = process.env.ROUTER_BASE ?? 'https://api.together.xyz/v1' const model = process.env.WORKER_MODEL ?? 'meta-llama/Meta-Llama-3-8B-Instruct-Lite' const instruction = process.env.INSTRUCTION_FILE @@ -51,8 +52,10 @@ async function main(): Promise { let i = 0 async function worker(): Promise { while (i < tasks.length) { - const t = tasks[i++] - const reply = await complete(base, key, model, `${instruction}\n\n\`\`\`python\n${t.prompt}\`\`\``, maxTokens) + const t = tasks[i] + i += 1 + if (!t) continue + const reply = await complete(base, apiKey, model, `${instruction}\n\n\`\`\`python\n${t.prompt}\`\`\``, maxTokens) const { pass: p } = await runChecker(t, extractCode(reply)) if (p === 1) pass += 1 else fails.push(t.taskId) diff --git a/bench/src/smoke-structural-rollout.mts b/bench/src/smoke-structural-rollout.mts index c2a42fc9..c8c65525 100644 --- a/bench/src/smoke-structural-rollout.mts +++ b/bench/src/smoke-structural-rollout.mts @@ -199,14 +199,25 @@ async function runTask(t: HumanEvalTask): Promise { `${t.taskId}: ${result.selection.length} receipts vs ${scored.length} scored candidates`, ) } - for (const r of result.selection) { + const receipts = result.selection.map((receipt) => { + if (receipt.score === undefined || receipt.reason === undefined) { + throw new Error(`${t.taskId}: receipt #${receipt.candidateIndex} is missing score or reason`) + } + return { + candidateIndex: receipt.candidateIndex, + selected: receipt.selected, + score: receipt.score, + reason: receipt.reason, + } + }) + for (const r of receipts) { const rec = scored[r.candidateIndex] if (!rec || Math.abs(r.score - visibleCheckScore(rec.outcome)) > 1e-9) { throw new Error(`${t.taskId}: receipt #${r.candidateIndex} score ${r.score} does not match the recorded outcome`) } } - const sampleCount = result.selection.filter((r) => r.reason.startsWith('sample')).length + const sampleCount = receipts.filter((r) => r.reason.startsWith('sample')).length const samples = scored.slice(0, sampleCount) if (samples.length === 0) throw new Error(`${t.taskId}: no sample candidates settled`) @@ -223,7 +234,7 @@ async function runTask(t: HumanEvalTask): Promise { const sampleHidden = await Promise.all(samples.map((s) => grade(s.candidate))) const selectedIdx = selectBestIndex(samples.map((s) => s.outcome)) const selectedHidden = sampleHidden[selectedIdx] as number - const winner = result.selection.find((r) => r.selected) + const winner = receipts.find((r) => r.selected) if (!winner) throw new Error(`${t.taskId}: no receipt marked selected`) const finalIdx = winner.candidateIndex const finalHidden = await grade((scored[finalIdx] as ScoredCandidate).candidate) @@ -241,12 +252,7 @@ async function runTask(t: HumanEvalTask): Promise { finalIdx, finalHidden, selectedVisible: visibleCheckScore((samples[selectedIdx] as ScoredCandidate).outcome), - receipts: result.selection.map((r) => ({ - candidateIndex: r.candidateIndex, - selected: r.selected, - score: r.score, - reason: r.reason, - })), + receipts, tokens: result.tokens, usd: result.usd, ms: result.ms, diff --git a/bench/src/swe-local-proof.mts b/bench/src/swe-local-proof.mts index 7a03f6da..c45a06b6 100644 --- a/bench/src/swe-local-proof.mts +++ b/bench/src/swe-local-proof.mts @@ -46,6 +46,9 @@ async function main(): Promise { const { environment, tasks, adapter } = await createSweBenchEnvironment(ids.length, { ids, enableRun }) const taskList = await tasks(0, ids.length) + const benchTaskById = new Map( + (await adapter.loadTasks({ ids, split: 'test' })).map((task) => [task.id, task]), + ) // One shot per pinned id: proxy score() to capture the exact judged bytes + a NON-DESTRUCTIVE // apply-coherence check, then delegate the verdict to the real Docker judge. @@ -123,7 +126,9 @@ async function main(): Promise { // Cached judge: identical patch ⇒ identical verdict; don't pay for a second Docker run. let s = judged.get(effPatch) if (!s) { - s = await adapter.judge(task, effPatch) + const benchTask = benchTaskById.get(task.id) + if (!benchTask) throw new Error(`swe-local-proof: unknown benchmark task ${task.id}`) + s = await adapter.judge(benchTask, effPatch) judged.set(effPatch, s) } rec.score = s diff --git a/bench/src/swe-stream.mts b/bench/src/swe-stream.mts index 872a7591..df73014a 100644 --- a/bench/src/swe-stream.mts +++ b/bench/src/swe-stream.mts @@ -700,6 +700,8 @@ async function acquireRepro( // hands the worker a no-code plan. Primitives replicated from supervisor-arena.mts (that file runs // main() on import, so it cannot be imported) — evidence from execution-verified inputs only. ---------- +type RepairFailureKind = 'wrong-fix' | 'apply-failed' | 'empty-diff' + /** Evidence for the supervisor: the issue, the worker's own candidate diff, and the tail of the * gold-verified reproduction's output on that diff. Execution-verified / model-visible ONLY — never * FAIL_TO_PASS, never the gold patch, never any worker self-report. Bounded to maxChars. */ @@ -709,7 +711,7 @@ function renderRepairEvidence( reproTail: string, reproExit: number | null, maxChars: number, - failureKind: 'wrong-fix' | 'apply-failed' = 'wrong-fix', + failureKind: RepairFailureKind = 'wrong-fix', ): string { const header = failureKind === 'apply-failed' @@ -789,7 +791,7 @@ async function superviseRepair( reproExit: number | null, marks: readonly string[], deadlineAt: number, - failureKind: 'wrong-fix' | 'apply-failed' = 'wrong-fix', + failureKind: RepairFailureKind = 'wrong-fix', ): Promise { const md = bt.metadata as Record const evidence = renderRepairEvidence(String(md.problem_statement ?? ''), candidateDiff, reproTail, reproExit, 14_000, failureKind) diff --git a/bench/src/tb-container-executor.test.mts b/bench/src/tb-container-executor.test.mts index 8afc3569..d738866d 100644 --- a/bench/src/tb-container-executor.test.mts +++ b/bench/src/tb-container-executor.test.mts @@ -2,8 +2,17 @@ import assert from 'node:assert/strict' import { chmod, mkdtemp, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import path from 'node:path' -import type { AgentSpec, ExecutorContext } from '@tangle-network/agent-runtime/loops' -import { buildTbDockerExecArgs, createTbContainerExecutor } from './tb-container-executor.mts' +import type { + AgentSpec, + Executor, + ExecutorContext, + ExecutorResult, +} from '@tangle-network/agent-runtime/loops' +import { + buildTbDockerExecArgs, + createTbContainerExecutor, + type TbExecOutput, +} from './tb-container-executor.mts' const spec: AgentSpec = { profile: { name: 'tb-test-worker' }, harness: null } @@ -11,6 +20,21 @@ function context(): ExecutorContext { return { signal: new AbortController().signal, seams: {} } } +function isAsyncIterable(value: unknown): value is AsyncIterable { + return typeof value === 'object' && value !== null && Symbol.asyncIterator in value +} + +async function executeOneShot( + executor: Executor, + task: unknown, +): Promise> { + const result = executor.execute(task, new AbortController().signal) + if (isAsyncIterable(result)) { + throw new Error('tb-container-executor returned a stream instead of a one-shot result') + } + return await result +} + async function executable(name: string, body: string): Promise { const dir = await mkdtemp(path.join(tmpdir(), 'tb-container-executor-')) const file = path.join(dir, name) @@ -52,7 +76,7 @@ printf 'argv:%s\\n' "$*" })(spec, context()) assert.equal(metered.budgetExempt, false, 'usage parser makes the executor metered') - const meteredResult = await metered.execute({ command: 'echo hello' }, new AbortController().signal) + const meteredResult = await executeOneShot(metered, { command: 'echo hello' }) assert.equal(meteredResult.out.containerId, 'cid') assert.equal(meteredResult.out.command, 'echo hello') assert.match( @@ -65,7 +89,7 @@ printf 'argv:%s\\n' "$*" const free = createTbContainerExecutor({ containerId: 'cid', dockerBin: fakeDocker })(spec, context()) assert.equal(free.budgetExempt, true, 'unmetered shell commands are explicit budget-exempt work') - const freeResult = await free.execute('printf ok', new AbortController().signal) + const freeResult = await executeOneShot(free, 'printf ok') assert.equal(freeResult.spent.iterations, 0) assert.deepEqual(freeResult.spent.tokens, { input: 0, output: 0 }) assert.equal(freeResult.spent.usd, 0) @@ -83,13 +107,13 @@ exit 7 failOnNonZeroExit: true, })(spec, context()) await assert.rejects( - strict.execute('do work', new AbortController().signal), + executeOneShot(strict, 'do work'), /command exited 7/, 'strict mode treats non-zero command exit as infrastructure failure', ) const lenient = createTbContainerExecutor({ containerId: 'cid', dockerBin: failingDocker })(spec, context()) - const lenientResult = await lenient.execute('do work', new AbortController().signal) + const lenientResult = await executeOneShot(lenient, 'do work') assert.equal(lenientResult.out.exitCode, 7, 'default mode returns non-zero exits as task artifacts') assert.match(lenientResult.out.stderr, /failed/) diff --git a/bench/src/tb-supervisor-sidecar.mts b/bench/src/tb-supervisor-sidecar.mts index be84ce6d..52445328 100644 --- a/bench/src/tb-supervisor-sidecar.mts +++ b/bench/src/tb-supervisor-sidecar.mts @@ -159,12 +159,13 @@ async function main(): Promise { | undefined if (out) { const usage = out.stdout ? parseWorkerUsage({ stdout: out.stdout, stderr: '', exitCode: null }) : undefined + const workerId = 'id' in w && typeof w.id === 'string' ? w.id : undefined if (usage) { workerInput += usage.input workerOutput += usage.output } workerOutputs.push({ - id: w.id, + id: workerId, status: w.status, command: out.command, exitCode: out.exitCode ?? null, From 4cf6f90fc319cba0d632257856ccfff7ba0d6c07 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 24 Jul 2026 11:06:32 -0600 Subject: [PATCH 07/14] feat(optimization): complete official method integration --- README.md | 46 ++- bench/package.json | 2 +- bench/src/hev-improve.mts | 31 +- bench/src/swe-arena/gepa-seat.test.mts | 6 +- bench/src/swe-improve.mts | 35 +- bench/src/trata-gepa.mts | 40 ++- bench/tsconfig.json | 1 + docs/api/primitive-catalog.md | 10 +- docs/canonical-api.md | 170 +++++----- examples/ablation-suite/gepa-driver-prompt.ts | 50 ++- examples/improve/README.md | 9 +- examples/improve/improve.ts | 26 +- .../intelligence-recommend.ts | 4 +- package.json | 4 +- pnpm-lock.yaml | 14 +- scripts/verify-package-exports.mjs | 2 +- src/candidate-execution/profile.ts | 2 +- src/improvement/improve.test.ts | 258 ++++++++++++-- src/improvement/improve.ts | 316 +++++++++++++----- src/improvement/index.ts | 5 + src/improvement/official-optimizers.test.ts | 163 +++++++-- src/improvement/official-optimizers.ts | 131 +++++++- src/improvement/profile-types.ts | 12 + src/improvement/rollout-policy.test.ts | 9 +- src/improvement/rollout-policy.ts | 10 +- src/intelligence/improvement-cycle.ts | 3 - src/intelligence/index.ts | 6 +- src/intelligence/intelligence.test.ts | 5 + src/{intelligence => }/redact.ts | 16 +- tests/profile-improvement-stack.test.ts | 16 +- 30 files changed, 1031 insertions(+), 371 deletions(-) create mode 100644 src/improvement/profile-types.ts rename src/{intelligence => }/redact.ts (81%) diff --git a/README.md b/README.md index 3ec2766e..1a523e99 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # @tangle-network/agent-runtime -A TypeScript runtime for running AI agents — as a **chat turn**, a **one-shot task**, or a **team of agents** working toward a goal — that records every run and uses those records to **measure and improve** agents against real pass/fail checks. It is the engine Tangle's own production agents run on. +A TypeScript runtime for running AI agents: as a **chat turn**, a **one-shot task**, or a **team of agents** working toward a goal: that records every run and uses those records to **measure and improve** agents against real pass/fail checks. It is the engine Tangle's own production agents run on. Domain behavior (models, tools, knowledge) plugs in as adapters; the scoring statistics and the ship decision come from [`@tangle-network/agent-eval`](https://www.npmjs.com/package/@tangle-network/agent-eval); sandboxed execution from [`@tangle-network/sandbox`](https://www.npmjs.com/package/@tangle-network/sandbox). @@ -56,9 +56,9 @@ Run it from a clone of this repo and you get exactly this: ```bash $ pnpm i && pnpm build $ pnpm tsx examples/quickstart/quickstart.ts -shot 0: reject — "Shipped one-click restore." -shot 1: PASS — "Shipped one-click restore with an instant rollback path." -decision: pick-winner — winner: shot 1 +shot 0: reject: "Shipped one-click restore." +shot 1: PASS: "Shipped one-click restore with an instant rollback path." +decision: pick-winner: winner: shot 1 ``` The fully annotated version of this loop, with every seam explained, is [`examples/driver-loop`](./examples/driver-loop). @@ -114,6 +114,13 @@ The profile is never changed. ```ts import { improve, officialGepa } from '@tangle-network/agent-runtime' +import { canonicalCandidateDigest } from '@tangle-network/agent-interface' + +const executionRef = canonicalCandidateDigest({ + deployment: process.env.AGENT_DEPLOYMENT_SHA!, + model: process.env.AGENT_MODEL!, + tools: process.env.AGENT_TOOLSET_SHA!, +}) const optimizer = { model: process.env.OPTIMIZER_MODEL!, @@ -127,8 +134,6 @@ const optimizer = { maxOutputTokensPerRequest: 16_384, pricing: { inputUsdPerMillion: Number(process.env.OPTIMIZER_INPUT_USD_PER_MILLION), - cachedInputUsdPerMillion: Number(process.env.OPTIMIZER_CACHED_INPUT_USD_PER_MILLION), - cacheWriteUsdPerMillion: Number(process.env.OPTIMIZER_CACHE_WRITE_USD_PER_MILLION), outputUsdPerMillion: Number(process.env.OPTIMIZER_OUTPUT_USD_PER_MILLION), }, }, @@ -136,9 +141,9 @@ const optimizer = { const result = await improve(baseProfile, { surface: 'prompt', + executionRef, method: officialGepa({ objective: 'Improve the complete support-agent prompt.', - evaluationId: 'support-prompt', recipe: { kind: 'engine', run: { @@ -149,6 +154,7 @@ const result = await improve(baseProfile, { }, optimizer, resume: 'if-compatible', + trustResumeState: true, describeScenario: ({ input }) => ({ input }), }), findings, @@ -156,8 +162,10 @@ const result = await improve(baseProfile, { selectionScenarios, testScenarios, judges: [judge], - agent, + agent: (candidateProfile, scenario, ctx) => + runProfile(candidateProfile, scenario, ctx), runDir: '.runs/support-prompt', + costCeiling: 25, }) if (result.decision === 'ship') { @@ -167,9 +175,9 @@ if (result.decision === 'ship') { `officialGepa(...)` delegates the complete search to GEPA's upstream Optimize Anything API through agent-eval. Pass one explicit `engine`, `sequential`, `adaptive-sequential`, `best-of`, `vote`, or `omni` recipe. -`evaluationId` identifies the dispatch, judges, models, and scoring logic. -Change it whenever any of those behaviors change. -With `resume: 'if-compatible'`, agent-eval resumes only when the saved run identity matches the candidate, recipe, data, optimizer settings, runner, and evaluation ID. +Runtime derives the upstream resume identity from `executionRef`, the complete baseline profile, and the selected surface. +With `resume: 'if-compatible'`, agent-eval resumes only when the saved run identity matches the candidate, recipe, data, optimizer settings, runner, and derived execution identity. +Set `trustResumeState: true` only when that run directory is private to the current operator. Use `resume: 'required'` to fail when no matching run exists. `result.provenance` reports the upstream package, run ID, resume status, evaluation count, and artifact directory. There is no local fallback. @@ -192,16 +200,28 @@ The published GEPA wheel lacks Optimize Anything, and the published SkillOpt whe SkillOpt and GEPA's standard reflection engine require `optimizer: { model, baseUrl, apiKey, budget }`. Agent-based GEPA engines may own their model connection instead. Agent Eval proxies those model calls, enforces the nested budget, and records their cost. +`costCeiling` is the total limit for optimizer calls, candidate runs, judges, and final scoring. +Runtime returns `hold` when any part of that cost is unknown. +Runtime rejects a reported total above the limit. SkillOpt accepts one text surface. GEPA accepts text or named components. Any complete method from `@tangle-network/agent-eval` uses the same call. +The `agent` callback receives the complete immutable candidate profile, not a raw prompt or component fragment. +Runtime uses that exact profile for every candidate run and returns the same measured profile in `result.candidate.profile`. +`executionRef` is a content digest of the agent callback, profile component mapping, model, tools, and closure settings. +Runtime combines it with the complete baseline profile and selected surface for saved work. +Changing any of them runs the affected work again. For a skill, set `surface: 'skills'` and `skills.resourceName`. For the complete profile, set `surface: 'agent-profile'`. To optimize several named profile fields together, also provide `profileComponents.read` and `profileComponents.apply`. Tools, MCP, hooks, subagents, curated instructions, and rollout policy are also exact profile coordinates. Runtime does not choose an optimizer for them. +Official optimizer factories reject a selected profile surface containing common live credentials or private values. +They redact common private values from findings, described scenarios, described artifacts, and judge notes before sending those values to Python. +`describeScenario` still controls the exact train and selection fields sent to the external optimizer, so return only approved fields. + Code is the exception. It uses Runtime's isolated git worktrees and coding-agent candidate execution: @@ -365,7 +385,7 @@ Use `importPrimeIntellectTraces(...)` to convert them to agent-eval `RunRecord`s ## Primitives -The general-purpose pieces, by import path. Every export with its one-line summary lives in the generated [`docs/api/primitive-catalog.md`](./docs/api/primitive-catalog.md) — check it before building anything new. +The general-purpose pieces, by import path. Every export with its one-line summary lives in the generated [`docs/api/primitive-catalog.md`](./docs/api/primitive-catalog.md): check it before building anything new. | Primitive | What it does | Import | |---|---|---| @@ -406,7 +426,7 @@ All 29 live in [`examples/`](./examples). - New here? [`docs/concepts.md`](./docs/concepts.md), the mental model in plain terms. - [`docs/canonical-api.md`](./docs/canonical-api.md), find the primitive: "I want to ___ → use ___". - [`docs/api/primitive-catalog.md`](./docs/api/primitive-catalog.md), every export in one generated, never-stale list with its import path. Check it before building anything new. -- [`docs/design.md`](./docs/design.md), the design philosophy and the internal research docs behind it — background reading, not required to use the package. +- [`docs/design.md`](./docs/design.md), the design philosophy and the internal research docs behind it: background reading, not required to use the package. - [`bench/HARNESS.md`](./bench/HARNESS.md), the experiment harness and how to run a benchmark. **Contributing:** `pnpm i && pnpm build && pnpm test` gets you running; the full local gate is the [`package.json`](./package.json) scripts (`lint`, `typecheck`, `docs:check`). diff --git a/bench/package.json b/bench/package.json index 22272261..3ac8ea4e 100644 --- a/bench/package.json +++ b/bench/package.json @@ -40,7 +40,7 @@ "verify:pier": "tsx scripts/verify-pier-pair.mts" }, "dependencies": { - "@tangle-network/agent-eval": "0.126.0", + "@tangle-network/agent-eval": "0.126.1", "@tangle-network/agent-interface": "0.32.0", "@tangle-network/agent-knowledge": "^4.1.0", "@tangle-network/agent-runtime": "workspace:*", diff --git a/bench/src/hev-improve.mts b/bench/src/hev-improve.mts index cf88ebfa..081c176b 100644 --- a/bench/src/hev-improve.mts +++ b/bench/src/hev-improve.mts @@ -11,8 +11,15 @@ * Worker + reflect models call the zai coding endpoint directly (no tangle router, * no WAF, no 503): TANGLE_API_KEY=$ZAI_API_KEY ROUTER_BASE=https://api.z.ai/api/coding/paas/v4 */ -import { improve, officialGepa } from '@tangle-network/agent-runtime' -import type { AgentProfile } from '@tangle-network/agent-interface' +import { + improve, + officialGepa, + type ReadonlyAgentProfile, +} from '@tangle-network/agent-runtime' +import { + canonicalCandidateDigest, + type AgentProfile, +} from '@tangle-network/agent-interface' import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract' import { extractCode, loadHumanEval, runChecker, type HumanEvalTask } from './benchmarks/humaneval' import { @@ -70,13 +77,6 @@ async function main(): Promise { const reflectMaxTokens = Number(process.env.REFLECT_MAX_TOKENS ?? 8000) const maxConcurrency = Number(process.env.MAX_CONCURRENCY ?? 4) const runDir = process.env.RUN_DIR ?? '.runs/humaneval-official-gepa' - const evaluationId = [ - 'humaneval-prompt-eval', - `worker=${workerModel}`, - `endpoint=${new URL(base).origin}`, - `tokens=${workerMaxTokens}`, - 'checker=local-python', - ].join('|') if (process.env.DRYRUN) { console.log( `DRYRUN: imports OK (improve=${typeof improve}, officialGepa=${typeof officialGepa})`, @@ -110,8 +110,9 @@ async function main(): Promise { console.log(`runDir=${runDir}\n`) const stats = { n: 0 } - const agent = async (surface: unknown, scenario: Scenario, ctx: DispatchContext): Promise => { - const instr = String(surface) + const agent = async (candidate: ReadonlyAgentProfile, scenario: Scenario, ctx: DispatchContext): Promise => { + const instr = candidate.prompt?.systemPrompt + if (instr === undefined) throw new Error('agent: candidate profile has no system prompt') const t = byId.get(scenario.id) if (!t) throw new Error(`agent: unknown scenario ${scenario.id}`) const prompt = `${instr}\n\n\`\`\`python\n${t.prompt}\`\`\`` @@ -175,10 +176,16 @@ async function main(): Promise { const out = await improve(profile, { surface: 'prompt', + executionRef: canonicalCandidateDigest({ + callback: 'bench/hev-improve', + model: workerModel, + endpoint: new URL(base).origin, + maxTokens: workerMaxTokens, + checker: 'local-python', + }), method: officialGepa({ objective: 'Improve the complete instruction for a small model that writes Python functions which pass hidden unit tests.', - evaluationId, background: 'Prefer behavioral strategies over wording changes. Address algorithm choice, edge cases, boundary values, type behavior, and self-checking. Return only the complete instruction.', recipe: { diff --git a/bench/src/swe-arena/gepa-seat.test.mts b/bench/src/swe-arena/gepa-seat.test.mts index 11a75ba7..e1ffb9f8 100644 --- a/bench/src/swe-arena/gepa-seat.test.mts +++ b/bench/src/swe-arena/gepa-seat.test.mts @@ -308,7 +308,7 @@ describe('recordGepaSeatInnerRun', () => { kind: 'package', evidence: 'observed', package: 'agent-eval-rpc', - version: '0.126.0', + version: '0.126.1', sourceSha256: bridgeHash, }, modules: [{ module: 'example.engine', sourceSha256: moduleHash }], @@ -456,7 +456,7 @@ const fullProvenance = ( kind: 'package', evidence: 'observed', package: 'agent-eval-rpc', - version: '0.126.0', + version: '0.126.1', sourceSha256: '2'.repeat(64), }, modules: [{ module: 'custom_gepa_engines', sourceSha256: '3'.repeat(64) }], @@ -639,7 +639,7 @@ describe('fanOutLoopsGenerator with the gepa seat', () => { }, bridge: { package: 'agent-eval-rpc', - version: '0.126.0', + version: '0.126.1', sourceSha256: '2'.repeat(64), }, modules: [{ module: 'custom_gepa_engines', sourceSha256: '3'.repeat(64) }], diff --git a/bench/src/swe-improve.mts b/bench/src/swe-improve.mts index 0fbca03a..a1b3f7ec 100644 --- a/bench/src/swe-improve.mts +++ b/bench/src/swe-improve.mts @@ -23,8 +23,15 @@ */ import { execFile } from 'node:child_process' import { promisify } from 'node:util' -import { improve, officialGepa } from '@tangle-network/agent-runtime' -import type { AgentProfile } from '@tangle-network/agent-interface' +import { + improve, + officialGepa, + type ReadonlyAgentProfile, +} from '@tangle-network/agent-runtime' +import { + canonicalCandidateDigest, + type AgentProfile, +} from '@tangle-network/agent-interface' import type { AgenticSurface, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/loops' import { refine, runAgentic } from '@tangle-network/agent-runtime/loops' import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract' @@ -62,15 +69,6 @@ async function main(): Promise { // Default OFF ⇒ reproduces the read/edit-only baseline denominator unchanged. const enableRun = ['1', 'true', 'yes'].includes((process.env.RUN_TOOL ?? '').toLowerCase()) const SEED_PROMPT = enableRun ? SWE_SEED_PROMPT_WITH_RUN : SWE_SEED_PROMPT - const evaluationId = [ - 'swe-prompt-eval', - `worker=${workerModel}`, - `router=${new URL(routerBaseUrl).origin}`, - `turns=${innerTurns}`, - `tokens=${workerMaxTokens}`, - `shots=${budgetShots}`, - `runTool=${String(enableRun)}`, - ].join('|') const allIds = [...new Set([...trainIds, ...selectionIds, ...testIds])] console.log('=== SWE-bench prompt optimization with official GEPA ===') @@ -104,8 +102,9 @@ async function main(): Promise { // one instance, return the git-diff patch. A per-call proxy captures the patch in // score() BEFORE runAgentic closes (rm) the workspace; its score is a cheap // patch-exists proxy so the ONLY Docker run per cell is the improve judge. - const agent = async (surface: unknown, scenario: Scenario, ctx: DispatchContext): Promise => { - const promptText = String(surface) + const agent = async (candidate: ReadonlyAgentProfile, scenario: Scenario, ctx: DispatchContext): Promise => { + const promptText = candidate.prompt?.systemPrompt + if (promptText === undefined) throw new Error('agent: candidate profile has no system prompt') const bt = byId.get(scenario.id) if (!bt) throw new Error(`agent: unknown scenario ${scenario.id}`) const task = { id: bt.id, systemPrompt: promptText, userPrompt: bt.prompt, meta: { instanceId: bt.id } } @@ -207,10 +206,18 @@ async function main(): Promise { const out = await improve(profile, { surface: 'prompt', + executionRef: canonicalCandidateDigest({ + callback: 'bench/swe-improve', + model: workerModel, + endpoint: new URL(routerBaseUrl).origin, + innerTurns, + maxTokens: workerMaxTokens, + budgetShots, + enableRun, + }), method: officialGepa({ objective: 'Improve the system prompt of a coding agent that fixes real GitHub bugs with list_files, read_file, edit_file, and optional run tools.', - evaluationId, background: 'Return the complete system prompt. Preserve tool names and require evidence from repository files and tests.', recipe: { diff --git a/bench/src/trata-gepa.mts b/bench/src/trata-gepa.mts index 3946e810..056c8f15 100644 --- a/bench/src/trata-gepa.mts +++ b/bench/src/trata-gepa.mts @@ -37,15 +37,21 @@ * CORPUS path to write JSONL run records (optional) */ -import type { AgentProfile } from '@tangle-network/agent-interface' +import { + canonicalCandidateDigest, + type AgentProfile, +} from '@tangle-network/agent-interface' import type { DispatchContext, JudgeConfig, JudgeScore, - MutableSurface, Scenario, } from '@tangle-network/agent-eval/campaign' -import { improve, officialGepa } from '@tangle-network/agent-runtime' +import { + improve, + officialGepa, + type ReadonlyAgentProfile, +} from '@tangle-network/agent-runtime' import { appendFileSync, writeFileSync } from 'node:fs' import { createTrataHedgeAdapter } from './benchmarks/trata-hedge' import type { BenchTask } from './benchmarks/types' @@ -165,14 +171,6 @@ async function main(): Promise { const corpusPath = process.env.CORPUS const baselineSurface = process.env.BASELINE_DIRECTIVE ?? DEFAULT_TRATA_SYSTEM const runDir = process.env.RUN_DIR ?? '.runs/trata-official-gepa' - const evaluationId = [ - 'trata-hedge-prompt', - `worker=${model}`, - `workerEndpoint=${new URL(routerBaseUrl).origin}`, - `judge=${process.env.JUDGE_MODEL ?? 'adapter-default'}`, - `rounds=${kRounds}`, - `tokens=${workerMaxTokens}`, - ].join('|') const workerPricing = requiredTokenPricing(process.env, 'WORKER') const optimizer = officialOptimizerModel({ env: process.env, @@ -235,16 +233,15 @@ async function main(): Promise { } const runWithSurface = async ( - surface: MutableSurface, + candidate: ReadonlyAgentProfile, scenario: TrataScenario, ctx: DispatchContext, ): Promise => { - if (typeof surface !== 'string') { - throw new Error('Trata prompt optimization requires a text surface') - } + const systemPrompt = candidate.prompt?.systemPrompt + if (systemPrompt === undefined) throw new Error('Trata candidate profile has no system prompt') // Round 1: initial analysis under the candidate system prompt. const r1 = await runPaidCompletion(ctx, 'trata-worker-round-1', [ - { role: 'system', content: surface }, + { role: 'system', content: systemPrompt }, { role: 'user', content: scenario.task.prompt }, ]) let answer = r1.content @@ -252,7 +249,7 @@ async function main(): Promise { // Round 2 (optional): self-critique for rubric coverage. if (kRounds >= 2 && answer.trim()) { const r2 = await runPaidCompletion(ctx, 'trata-worker-round-2', [ - { role: 'system', content: surface }, + { role: 'system', content: systemPrompt }, { role: 'user', content: scenario.task.prompt }, { role: 'assistant', content: answer }, { role: 'user', content: REFINE_INSTRUCTION }, @@ -297,10 +294,17 @@ async function main(): Promise { } const result = await improve(profile, { surface: 'prompt', + executionRef: canonicalCandidateDigest({ + callback: 'bench/trata-gepa', + model, + endpoint: new URL(routerBaseUrl).origin, + maxTokens: workerMaxTokens, + rounds: kRounds, + judgeModel: process.env.JUDGE_MODEL ?? 'adapter-default', + }), method: officialGepa({ objective: 'Improve the complete system instruction for a financial analyst that writes evidence-backed investment memos.', - evaluationId, background: 'The judge awards partial credit for covering every requested analytical theme with specific quantitative claims, named peer comparisons, explicit calculations, source citations, and a decisive synthesis. Preserve the required ANALYSIS: prefix.', recipe: { diff --git a/bench/tsconfig.json b/bench/tsconfig.json index 7501a77c..0662034a 100644 --- a/bench/tsconfig.json +++ b/bench/tsconfig.json @@ -12,6 +12,7 @@ "esModuleInterop": true, "resolveJsonModule": true, "paths": { + "@tangle-network/agent-runtime": ["../src/index.ts"], "@tangle-network/agent-runtime/loops": ["../src/runtime/index.ts"], "@tangle-network/sandbox": ["../node_modules/@tangle-network/sandbox"] } diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index bc5f24a5..48715269 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.104.0` and `@tangle-network/agent-eval@0.126.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.104.0` and `@tangle-network/agent-eval@0.126.1` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -15,7 +15,7 @@ Every subpath this package declares in `package.json` `exports`. Reach for these ### Root — task lifecycle, conversation, RSI verbs, observability -Import from `@tangle-network/agent-runtime` — 380 exports. +Import from `@tangle-network/agent-runtime` — 386 exports. | Symbol | Kind | Summary | |---|---|---| @@ -234,6 +234,7 @@ Import from `@tangle-network/agent-runtime` — 380 exports. | `ImproveMethodFactory` | type | Build a complete method after trace findings are available. | | `ImproveMethodOptions` | type | Complete-method configuration for every non-code profile surface. | | `ImproveOptions` | type | The canonical improvement API: complete methods for profiles, worktrees for code. | +| `ImproveProfileAgent` | type | Runs one exact materialized profile on one scenario. | | `ImproveSurface` | type | The executable agent lever `improve` optimizes. Profile fields remain | | `OfficialGepaOptions` | type | Official GEPA configuration plus bounded Runtime findings context. | | `OfficialSkillOptOptions` | type | Official SkillOpt configuration plus bounded Runtime findings context. | @@ -241,6 +242,7 @@ Import from `@tangle-network/agent-runtime` — 380 exports. | `OpenAIChatToolChoice` | type | `tool_choice` parameter for OpenAI-compat chat. Same shape as the OpenAI | | `PersonaDriver` | type | A persona that drives the conversation: either a full driver `AgentProfile` | | `PropagatedHeaders` | type | Header bag carried through `AgentBackendContext.propagatedHeaders` so | +| `ReadonlyAgentProfile` | type | Complete immutable profile value used during measured execution. | | `RetryableErrorPredicate` | type | Pure judgment of whether an error is worth retrying. Defaults: TimeoutError, AbortError, fetch-level network errors. | | `RetryBackoff` | type | Backoff between attempts. Constant ms, or `(attempt: 1-indexed) => ms`. | | `RuntimeHookPhase` | type | Runtime hook contracts. Hooks are execution-scoped observers, not part of an | @@ -250,7 +252,7 @@ Import from `@tangle-network/agent-runtime` — 380 exports. | `VerifiedAgentCandidateTaskOutcome` | type | Branded task outcome that has survived independent evaluator verification. | | `Verifier` | type | Verifies the edited worktree. Sync or async; throws only on a setup fault | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidatePreparationEvidence`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateVerificationPorts`, `AgentCandidateWorkspaceArchiveLimits`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgenticGeneratorShotReceipt`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `BackendCallPolicy`, `CanonicalCandidateDocument`, `CaptureAgentCandidateWorkspaceOptions`, `CapturedAgentCandidateWorkspace`, `ChatTurnHooks`, `ChatTurnResult`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `CreateAgentCandidateWorkspacePortOptions`, `CreateKnowledgeImprovementActivationExecutorOptions`, `CreateProtectedAgentCandidateModelPortOptions`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `DisposePreparedAgentCandidateOptions`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `ExactProcessCandidateExecutorOptions`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `HaltContext`, `HaltSignal`, `ImproveCodeOptions`, `ImproveCodeResult`, `ImprovementCandidate`, `ImproveMethodContext`, `ImproveMethodResult`, `ImproveSkillsOptions`, `KnowledgeImprovementActivationExecutor`, `KnowledgeImprovementCandidatePair`, `KnowledgeImprovementExperimentBundles`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `OtelAttribute`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `RecoverExpiredAgentCandidateOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolveAgentBackendOptions`, `ResolvedAgentCandidateContainer`, `ResolvedChatModel`, `RunChatTurnInput`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeEventOtelOptions`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunHandle`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeTelemetryOptions`, `RunToolLoopOptions`, `SanitizedKnowledgeReadinessReport`, `StreamToolLoopOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `ToAnalystFindingsOptions`, `ToolLoopResult`, `VerifiedAgentCandidate`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ControlDecision`, `ConversationStreamEvent`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `ImproveMethodSource`, `ImproveProfileSurface`, `ImproveResult`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeStreamEvent`, `StreamToolLoopYield`, `SupervisedKnowledgeUpdater`, `ToolLoopEvent`, `TurnOrder`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidatePreparationEvidence`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateVerificationPorts`, `AgentCandidateWorkspaceArchiveLimits`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgenticGeneratorShotReceipt`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `BackendCallPolicy`, `CanonicalCandidateDocument`, `CaptureAgentCandidateWorkspaceOptions`, `CapturedAgentCandidateWorkspace`, `ChatTurnHooks`, `ChatTurnResult`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `CreateAgentCandidateWorkspacePortOptions`, `CreateKnowledgeImprovementActivationExecutorOptions`, `CreateProtectedAgentCandidateModelPortOptions`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `DisposePreparedAgentCandidateOptions`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `ExactProcessCandidateExecutorOptions`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `HaltContext`, `HaltSignal`, `ImproveCodeOptions`, `ImproveCodeResult`, `ImprovementCodeCandidate`, `ImprovementProfileCandidate`, `ImproveMethodContext`, `ImproveMethodResult`, `ImproveSkillsOptions`, `KnowledgeImprovementActivationExecutor`, `KnowledgeImprovementCandidatePair`, `KnowledgeImprovementExperimentBundles`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `OtelAttribute`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `RecoverExpiredAgentCandidateOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolveAgentBackendOptions`, `ResolvedAgentCandidateContainer`, `ResolvedChatModel`, `RunChatTurnInput`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeEventOtelOptions`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunHandle`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeTelemetryOptions`, `RunToolLoopOptions`, `SanitizedKnowledgeReadinessReport`, `StreamToolLoopOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `ToAnalystFindingsOptions`, `ToolLoopResult`, `VerifiedAgentCandidate`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ControlDecision`, `ConversationStreamEvent`, `DeepReadonly`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `ImprovementCandidate`, `ImproveMethodSource`, `ImproveOptimizationRunOptions`, `ImproveProfileSurface`, `ImproveResult`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeStreamEvent`, `StreamToolLoopYield`, `SupervisedKnowledgeUpdater`, `ToolLoopEvent`, `TurnOrder`. ### Vertical agent — manifest + surface proposal source @@ -1504,7 +1506,7 @@ Import from `@tangle-network/agent-eval/campaign` — 320 exports. | `LlmJudgeDimension` | type | A rubric dimension as a bare key or the full `{ key, description }` shape. A | | `MutableSurface` | type | The mutable surface a proposer changes. Tiers (see | | `ObjectiveSource` | type | Where an objective's per-cell scalar comes from. `composite` reads the | -| `OptimizationMethodRunOptions` | type | Per-method campaign settings. Each method receives its own spend account. | +| `OptimizationMethodRunOptions` | type | Shared campaign settings applied to every optimization method. | | `OptimizationProposer` | type | Optional vocabulary alias. The loop is the optimizer; this object is the | | `ProfileDispatchFn` | type | Dispatch for one cell: render `profile` against `scenario`, returning the | | `PromotionPolicy` | type | A promotion strategy: a pure function from the evidence vector to a verdict. | diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 3e1b834b..882f5115 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -1,54 +1,54 @@ -# `@tangle-network/agent-runtime` — Canonical API Reference +# `@tangle-network/agent-runtime`: Canonical API Reference - + -> **Version 0.104.0.** The export inventory + per-symbol signatures live in the generated `docs/api/` reference: **`docs/api/primitive-catalog.md`** is the never-stale, grouped list of every primitive to reuse (own surface + the agent-eval judge / authenticity / verification / statistics / campaign / token-usage surfaces), with each one's import path and one-line summary read live from source; the per-module pages hold the full signatures. The pinned substrate is agent-eval `>=0.126.0 <0.127.0`; the sandbox substrate that materializes profiles into harness shapes is `@tangle-network/sandbox` (peer `>=0.11.1 <1.0.0`). The neutral contract types (`AgentProfile`, `AgentProfileMcpServer`, `HarnessType`, `ReasoningEffort`, `Part`/`ToolPart`/`ToolState`, plus environment-provider types) are owned by **`@tangle-network/agent-interface`** (peer `>=0.32.0 <0.33.0`) — the single source of truth. Substrate primitives are re-exported through `@tangle-network/agent-eval/contract` (or `/campaign`), not local to this package — the catalog's §2 shows exactly which subpath each lives under. +> **Version 0.104.0.** The export inventory + per-symbol signatures live in the generated `docs/api/` reference: **`docs/api/primitive-catalog.md`** is the never-stale, grouped list of every primitive to reuse (own surface + the agent-eval judge / authenticity / verification / statistics / campaign / token-usage surfaces), with each one's import path and one-line summary read live from source; the per-module pages hold the full signatures. The pinned substrate is agent-eval `>=0.126.1 <0.127.0`; the sandbox substrate that materializes profiles into harness shapes is `@tangle-network/sandbox` (peer `>=0.11.1 <1.0.0`). The neutral contract types (`AgentProfile`, `AgentProfileMcpServer`, `HarnessType`, `ReasoningEffort`, `Part`/`ToolPart`/`ToolState`, plus environment-provider types) are owned by **`@tangle-network/agent-interface`** (peer `>=0.32.0 <0.33.0`): the single source of truth. Substrate primitives are re-exported through `@tangle-network/agent-eval/contract` (or `/campaign`), not local to this package: the catalog's §2 shows exactly which subpath each lives under. > -> **`./loops` is the runtime barrel** — `package.json` maps it to `src/runtime/index.ts`. Everything below labelled `/loops` is the recursive-atom + loop-kernel surface. +> **`./loops` is the runtime barrel**: `package.json` maps it to `src/runtime/index.ts`. Everything below labelled `/loops` is the recursive-atom + loop-kernel surface. > -> **Read this before writing any orchestration, optimization, or measurement code in this repo.** If you are about to write a persona⟷agent conversation runner, a "skill optimizer", a "profile-seam", a depth-vs-breadth A/B harness, a bootstrap loop, or a `new Sandbox(...)` + stream + read dance — **stop**, it already exists, and a parallel copy will silently break one of the guarantees the existing primitives enforce: equal compute per compared arm ("equal-k"), the attempt-picker never being the grader ("selector≠judge"), complete usage capture, or eval running the same code path as production. +> **Read this before writing any orchestration, optimization, or measurement code in this repo.** If you are about to write a persona⟷agent conversation runner, a "skill optimizer", a "profile-seam", a depth-vs-breadth A/B harness, a bootstrap loop, or a `new Sandbox(...)` + stream + read dance: **stop**, it already exists, and a parallel copy will silently break one of the guarantees the existing primitives enforce: equal compute per compared arm ("equal-k"), the attempt-picker never being the grader ("selector≠judge"), complete usage capture, or eval running the same code path as production. -## 1. Mental model — the spine +## 1. Mental model: the spine -> **Legend** — five terms the rest of this doc leans on, in plain terms: -> - **profile** — an `AgentProfile`: the whole agent as data (prompt + skills + tools + mcp + knowledge/memory). Internal design docs also call this the agent's "genome". -> - **driver ⟷ worker** — one agent (the driver) spawns and steers other agents (workers) and reads their output; both are the same building block playing different roles. -> - **conserved budget pool** — one shared compute budget split across workers, so two different topologies cost the same and a comparison is fair. -> - **combinator** — a reusable topology shape (`loopUntil` = depth/refine, `fanout` = breadth/sample) you compose instead of hand-writing a loop. -> - **holdout** — fresh problems held back from tuning, so a measured win can't be memorization. +> **Legend**: five terms the rest of this doc leans on, in plain terms: +> - **profile**: an `AgentProfile`: the whole agent as data (prompt + skills + tools + mcp + knowledge/memory). Internal design docs also call this the agent's "genome". +> - **driver ⟷ worker**: one agent (the driver) spawns and steers other agents (workers) and reads their output; both are the same building block playing different roles. +> - **conserved budget pool**: one shared compute budget split across workers, so two different topologies cost the same and a comparison is fair. +> - **combinator**: a reusable topology shape (`loopUntil` = depth/refine, `fanout` = breadth/sample) you compose instead of hand-writing a loop. +> - **holdout**: fresh problems held back from tuning, so a measured win can't be memorization. The system is four steps, each with a named entry point: 1. **Describe the agent as data.** A **profile** is the whole agent: `systemPrompt + skills + tools + mcp + knowledge + memory + rag`, one combined surface. -2. **Run it.** A driver steers workers over rounds — `runPersonified` composes a combinator (`loopUntil`, `fanout`, …) over the `Supervisor`, spending K rounds against one persistent, journaled artifact from a *conserved budget pool*, so any two topologies you compare cost the same by construction. +2. **Run it.** A driver steers workers over rounds: `runPersonified` composes a combinator (`loopUntil`, `fanout`, …) over the `Supervisor`, spending K rounds against one persistent, journaled artifact from a *conserved budget pool*, so any two topologies you compare cost the same by construction. 3. **Score it on a benchmark.** Either the `ADAPTERS` registry driven by `runGate` over the Supervisor, or an `AgenticSurface` driven by `runBenchmark`/`runAgentic`. -4. **Improve it on three partitions.** `improve(profile, { method, trainScenarios, selectionScenarios, testScenarios })` runs a complete agent-eval `OptimizationMethod`. The method receives train and selection cases only. Runtime scores its selected candidate on the untouched final test and returns `ship` only when the paired interval clears the required lift. +4. **Improve it on three partitions.** `improve(profile, { executionRef, method, trainScenarios, selectionScenarios, testScenarios, agent })` runs a complete agent-eval `OptimizationMethod`. The method receives train and selection cases only. Runtime materializes each surface as a complete detached profile and passes that exact profile to `agent`. `executionRef` identifies the callback, component mapping, model, tools, and closure settings. Agent Eval scores the selected profile on the untouched final test. Runtime returns `ship` only when the paired interval clears the required lift and all spend is accounted for. Two standing rules: the model that picks the best attempt is never the model that grades it, and observation attaches to the *loop* via `RuntimeHooks`, never to the portable profile. One known limit: the current `Supervisor` records completed settlements but does not resume a live tree after coordinator restart. (The original one-sentence compressed form of this spine is preserved in [design.md](./design.md).) -Two substrates implement the same recursive-atom over the one `Executor` port and share `defaultSelectWinner` — a deliberate pair, **do not invent a third**: the **reactive** `Supervisor`/`Scope` + personify combinators (the agent-driver; equal-k by construction via the conserved budget pool — prefer for NEW recursive/keystone work) and the round-synchronous **`runLoop`** kernel (the leaf; what most sandbox benches drive today). `inlineSandboxClient` adapts any non-box `Executor` into a `SandboxClient` for `runLoop`, and `settledToIteration` bridges reactive `Settled`s into the kernel's `Iteration`, so the two interoperate without forking selection or metering. +Two substrates implement the same recursive-atom over the one `Executor` port and share `defaultSelectWinner`: a deliberate pair, **do not invent a third**: the **reactive** `Supervisor`/`Scope` + personify combinators (the agent-driver; equal-k by construction via the conserved budget pool: prefer for NEW recursive/keystone work) and the round-synchronous **`runLoop`** kernel (the leaf; what most sandbox benches drive today). `inlineSandboxClient` adapts any non-box `Executor` into a `SandboxClient` for `runLoop`, and `settledToIteration` bridges reactive `Settled`s into the kernel's `Iteration`, so the two interoperate without forking selection or metering. -## 1.5 The AgentProfile rule — author the profile, the substrate materializes it +## 1.5 The AgentProfile rule: author the profile, the substrate materializes it -**An agent IS its `AgentProfile`, and the profile is the WHOLE agent — not just a prompt.** The surface is `systemPrompt + skills + tools + mcp + subagents + hooks + permissions + memory/rag + model` (the `AgentProfile*` family in `@tangle-network/sandbox`, constructed via `defineAgentProfile`). **System prompt ≠ skills** — skills are separate, invokable how-tos the agent reads *when prompted to invoke them*; never concatenate a skill body into the system prompt. +**An agent IS its `AgentProfile`, and the profile is the WHOLE agent: not just a prompt.** The surface is `systemPrompt + skills + tools + mcp + subagents + hooks + permissions + memory/rag + model` (the `AgentProfile*` family in `@tangle-network/sandbox`, constructed via `defineAgentProfile`). **System prompt ≠ skills**: skills are separate, invokable how-tos the agent reads *when prompted to invoke them*; never concatenate a skill body into the system prompt. -**You change an agent's behavior by changing its PROFILE — never by writing orchestration code around it.** The behaviors we keep hand-rolling are profile properties: -- **Self-verification** is a profile lever, three ways, all configuration and zero glue code: (1) *steered* — the prompt says "run the tests, read failures, fix, repeat"; (2) *process-defined* — its instructions make verify-after-every-change its standing process; or (3) a **post-finish hook** that auto-runs the check and feeds failures back. The harness runs that loop. **You do not write a per-round judge, a `while(!done)`, or a bash hill-climb.** +**You change an agent's behavior by changing its PROFILE: never by writing orchestration code around it.** The behaviors we keep hand-rolling are profile properties: +- **Self-verification** is a profile lever, three ways, all configuration and zero glue code: (1) *steered*: the prompt says "run the tests, read failures, fix, repeat"; (2) *process-defined*: its instructions make verify-after-every-change its standing process; or (3) a **post-finish hook** that auto-runs the check and feeds failures back. The harness runs that loop. **You do not write a per-round judge, a `while(!done)`, or a bash hill-climb.** - **Iteration, delegation, audit-against-spec** are likewise hooks / subagents / skills / process *in the profile*. -**The sandbox substrate materializes a profile into the harness's real shapes — so author the GENERAL profile and NEVER code to a harness.** `@tangle-network/sandbox` renders an `AgentProfile` into whatever the running harness needs (instructions file, tool/MCP config, mounted skills, hooks, subagents). opencode / Claude Code / Codex are interchangeable *targets*; opencode is only the local **test** substrate behind the cli-bridge. **Do NOT write harness-specific config or a `profile → opencode.json` realizer.** A lever that isn't materialized yet is a **substrate gap to fill in `@tangle-network/sandbox`**, not a bespoke realizer here. +**The sandbox substrate materializes a profile into the harness's real shapes: so author the GENERAL profile and NEVER code to a harness.** `@tangle-network/sandbox` renders an `AgentProfile` into whatever the running harness needs (instructions file, tool/MCP config, mounted skills, hooks, subagents). opencode / Claude Code / Codex are interchangeable *targets*; opencode is only the local **test** substrate behind the cli-bridge. **Do NOT write harness-specific config or a `profile → opencode.json` realizer.** A lever that isn't materialized yet is a **substrate gap to fill in `@tangle-network/sandbox`**, not a bespoke realizer here. -**Therefore the supervisor's only intelligence is AUTHORING full profiles** — the optimizable self-improvement surface: read the task, decompose it, and for each sub-task author the *complete* profile (which prompt, skills, tools/MCP, hooks, subagents, model). The quality of a worker IS the quality of the profile authored for it. **The harness executes; you compose.** +**Therefore the supervisor's only intelligence is AUTHORING full profiles**: the optimizable self-improvement surface: read the task, decompose it, and for each sub-task author the *complete* profile (which prompt, skills, tools/MCP, hooks, subagents, model). The quality of a worker IS the quality of the profile authored for it. **The harness executes; you compose.** -## 2. Decision table — "I want to ___ → use ___ → NOT ___" +## 2. Decision table: "I want to ___ → use ___ → NOT ___" -This table is judgment-only: it maps an intent to the ONE primitive to reach for and the thing NOT to build. It is not an inventory — **for the full list of what exists (every export, its import path, its one-line summary) see the generated `docs/api/primitive-catalog.md`; for full signatures, the per-module `docs/api/` pages.** Each row tags its import subpath; a row is a LOCAL export of this package unless tagged with a substrate package (`agent-eval/contract`, `agent-eval/campaign`, `@tangle-network/sandbox`) or `bench`. +This table is judgment-only: it maps an intent to the ONE primitive to reach for and the thing NOT to build. It is not an inventory: **for the full list of what exists (every export, its import path, its one-line summary) see the generated `docs/api/primitive-catalog.md`; for full signatures, the per-module `docs/api/` pages.** Each row tags its import subpath; a row is a LOCAL export of this package unless tagged with a substrate package (`agent-eval/contract`, `agent-eval/campaign`, `@tangle-network/sandbox`) or `bench`. -### "A loop" is not one thing — read this before reaching for one +### "A loop" is not one thing: read this before reaching for one -A general "loop" primitive is the single most common modelling error in this repo; it has produced a `defineLoop` facade **twice** (see [`research/loop-facade-postmortem.md`](./research/loop-facade-postmortem.md)). "Iterate agents toward a goal" splits on **one question: is the structure FIXED (you write the shape, it's code) or DYNAMIC (a model decides the shape at runtime)?** How many agents (1 vs N) is *orthogonal* — every shape below is 1..N agents, so "two agents (proposer→verifier)" is not special, it's a 2-stage chain. +A general "loop" primitive is the single most common modelling error in this repo; it has produced a `defineLoop` facade **twice** (see [`research/loop-facade-postmortem.md`](./research/loop-facade-postmortem.md)). "Iterate agents toward a goal" splits on **one question: is the structure FIXED (you write the shape, it's code) or DYNAMIC (a model decides the shape at runtime)?** How many agents (1 vs N) is *orthogonal*: every shape below is 1..N agents, so "two agents (proposer→verifier)" is not special, it's a 2-stage chain. **FIXED shape → a combinator you compose:** @@ -56,96 +56,96 @@ A general "loop" primitive is the single most common modelling error in this rep |---|---|---| | refine ONE artifact over rounds until a check passes | depth | `loopUntil` | | try N independent attempts, keep the best | breadth | `fanout` | -| ordered stages, each feeds the next — this is what "propose → verify" is | chain | `pipeline` | +| ordered stages, each feeds the next: this is what "propose → verify" is | chain | `pipeline` | | many independent views of one artifact, then aggregate | ensemble | `panel` | | expand only the promising branches | adaptive search | `widen` + `flatWidenGate` | **DYNAMIC shape → this is orchestration, NOT a loop.** When an LLM decides *at runtime* what to spawn next and when to stop (decompose a messy goal, react to each result, no fixed round count), it is a reactive tree, not a loop: `Scope` + Supervisor in-process (`supervise` / `runPersonified`), or `createCoordinationTools` for a sandbox driver. Its topology is *data*, so no fixed-round "loop" grammar can describe it. -**The trap** is a single grammar (`defineLoop`, a `runXxxLoop`) spanning all of the above — there can't be one, because some are code and one is a model deciding. No new loop primitive lands without a tiny executable proof, **over real agents**, of the exact substrate join it claims to simplify. +**The trap** is a single grammar (`defineLoop`, a `runXxxLoop`) spanning all of the above: there can't be one, because some are code and one is a model deciding. No new loop primitive lands without a tiny executable proof, **over real agents**, of the exact substrate join it claims to simplify. | I want to… | Use (import) | Do NOT build | |---|---|---| -| **Just run a supervisor to a goal (one call, scaffolding defaulted)** — START HERE (running agents) | `supervise(profile, task, { budget, backend? })` — `/loops` | hand-wiring `createSupervisor().run` + `blobs`/`perWorker`/`journal`/`executors`; reaching for the lower-level run-verbs below before you need a specific counterparty | -| **Supervise agents to solve a graded `AgenticSurface` task** (workers `runAgentic` the surface, settle on its own check, driver self-improves from the failing tests) | `superviseSurface(profile, task, { surface, worker })` — `/loops` | a worker-seam + a "self-improving supervisor" wrapper around `supervise()`; passing a custom `makeWorkerAgent` that runs `runAgentic` | -| Run a profile through a topology shape over the keystone Supervisor, end-to-end | `runPersonified({ persona, shape, task, budget })` — `/loops` | a hand-rolled `createSupervisor().run` + seam-wiring helper | -| Loop a worker over one evolving artifact, K rounds, stop-when-good | `loopUntil(seed, spec)` as the `shape` — `/loops` | a `while(!done){runWorker();decide()}` hand-loop or "multi-attempt refine driver" | -| Run a worker agent under test conversing with a **simulated-user persona**, K rounds, worker-only metered | `runPersonaConversation({ worker, persona, backendFor, systemPromptOf })` — root `.` (also `/loops`) | a hand-rolled per-agent `dispatchWithSurface` bridge / eval-dispatch loop | +| **Just run a supervisor to a goal (one call, scaffolding defaulted)**: START HERE (running agents) | `supervise(profile, task, { budget, backend? })`: `/loops` | hand-wiring `createSupervisor().run` + `blobs`/`perWorker`/`journal`/`executors`; reaching for the lower-level run-verbs below before you need a specific counterparty | +| **Supervise agents to solve a graded `AgenticSurface` task** (workers `runAgentic` the surface, settle on its own check, driver self-improves from the failing tests) | `superviseSurface(profile, task, { surface, worker })`: `/loops` | a worker-seam + a "self-improving supervisor" wrapper around `supervise()`; passing a custom `makeWorkerAgent` that runs `runAgentic` | +| Run a profile through a topology shape over the keystone Supervisor, end-to-end | `runPersonified({ persona, shape, task, budget })`: `/loops` | a hand-rolled `createSupervisor().run` + seam-wiring helper | +| Loop a worker over one evolving artifact, K rounds, stop-when-good | `loopUntil(seed, spec)` as the `shape`: `/loops` | a `while(!done){runWorker();decide()}` hand-loop or "multi-attempt refine driver" | +| Run a worker agent under test conversing with a **simulated-user persona**, K rounds, worker-only metered | `runPersonaConversation({ worker, persona, backendFor, systemPromptOf })`: root `.` (also `/loops`) | a hand-rolled per-agent `dispatchWithSurface` bridge / eval-dispatch loop | | Run **two `AgentProfile`s head-to-head** with a separate resumable session for each actor | `runConversation(...)` from root `.` | a hand-rolled two-agent turn loop | -| Drop a persona⟷agent conversation into an eval matrix as its dispatch | `runPersonaDispatch` → `runProfileMatrix({ dispatch })` — root `.` / `agent-eval/campaign` | a per-agent custom dispatch bridge | -| Best-of-N / parallel-research / map-reduce at equal compute | `fanout(items, opts)` — `/loops` | `Promise.all` over N calls + manual argmax/merge (bypasses the budget pool → breaks equal-k) | -| Produce-then-gate with a real checker | `verify(spec)` — `/loops` | "generate, then self-check with the same model, ship if ok" (collapses selector+judge) | -| Multi-judge review / rubric quorum over one artifact | `panel(spec)` — `/loops` | a judge ensemble that feeds one judge's score into another | -| Fixed sequential chain (plan→implement→…) | `pipeline(stages)` — `/loops` | hand-chained `await`s passing outputs along | -| Adaptive tree search / progressive widening | `widen(spec)` + `flatWidenGate()` — `/loops` | a best-first/MCTS that reads child *scores* to expand (selector=judge); keep `flatWidenGate()` until your gate is proven | -| Define the profile record for a personified run | `definePersona(input)` — `/loops` | a "profile-seam" / agent-config wrapper carrying model+prompt+tools+role | -| Make a worker self-verify / iterate / audit | a **hook / process / skill on its authored `AgentProfile`** — §1.5 | a per-round judge, a `while(!done)` loop, or a bash hill-climb (it's a profile lever) | -| Run an authored profile on a real harness | author the `AgentProfile`, hand it to the **sandbox substrate** — `@tangle-network/sandbox` (`defineAgentProfile`) | a `profile → opencode.json` realizer or any harness-specific config writer | -| Have the supervisor design its workers | author a **full `AgentProfile`** per sub-task (prompt+skills+tools+mcp+hooks+subagents) — `/loops` | author a bare `systemPrompt` string (a worker can't act on levers it has no levers for) | -| Write a custom driver Agent and run it directly | `createSupervisor().run(root, task, opts)` — `/loops` | a bespoke orchestrator that spawns sub-agents and tallies cost (equal-compute claim breaks there) | -| Run depth-vs-breadth (or a custom strategy) over a stateful tool domain | `runAgentic({ surface, task, mode\|strategy, budget })` — `/loops` | a hand-rolled `Supervisor.run` + journal/registry, or a depth/breadth loop | -| Author a new topology/strategy compactly | `defineStrategy(name, body)` using `ctx.shot()`+`ctx.critique()` — `/loops` | a 70-line driver with `scope.spawn`/`scope.next` ceremony, or trusting a body-returned score | -| Compare strategies + get a significance report on a domain | `runBenchmark({ environment, tasks, worker, strategies })` — `/loops` | your own strategy-comparison loop / paired-bootstrap / Pareto math | -| Add a stateful tool-using domain | implement `AgenticSurface` (5 hooks: open/tools/call/score/close) — `/loops` | a bespoke per-benchmark agent runner / tool-loop harness | -| Run a sandbox coding rollout, round-synchronous (fresh box per round) | `runLoop(options)` — `/loops` | a `new Sandbox()`+acquire+stream+parse+delete loop, or a 2nd winner-selector | -| Run **agent-eval fixture folders** through runtime `runLoop` | agent-eval fixture loading/planning, then `loopCampaignDispatch(...)` — `/loops` | a one-off `runCampaign` dispatch that hand-builds `ExecCtx`, drops loop traces, or forgets token/cost reporting | -| Run + **resume** ONE persistent box across turns | `openSandboxRun(client, opts, deliverable)` — `/loops` | a per-domain `new Sandbox`+`box.fs.read`+delete copy | -| Run **ONE agent turn** on any substrate — box (`streamPrompt`, or the sandbox SDK's autonomous-task `streamTask` via the `box-task` kind with per-task `TaskOptions`), cli-bridge/router `Executor`, or in-process chat backend — as ONE normalized `RuntimeStreamEvent` stream with a guaranteed terminal result+usage event; opt into in-stream `tool_call`/`tool_result` with `preserveToolParts`, or tap the raw sandbox events with `onRawEvent` | `streamAgentTurn(backend, prompt, { signal, timeoutMs, preserveToolParts?, onRawEvent? })` + `collectAgentTurn(stream)` — `/loops` | a per-provider stream→event mapper zoo, a bespoke `streamTask` parser, a hand-faked box around a non-box executor, or raw fetch leaking through the turn abstraction | -| Pick the **execution transport a driven loop runs on** (`sandbox` box / cli-bridge / router) from a product flag | `resolveSandboxClient({ backend })` — `/loops` | a per-product `if (backend === 'router') …` branch re-wiring `createExecutor` + `inlineSandboxClient` | -| Pick the **chat backend an in-process turn runs on** (`router`/`tcloud`/`cli-bridge`/`sandbox`) from a product flag | `resolveAgentBackend({ backend })` — root `.` | the copy-pasted `backend-name → createOpenAICompatibleBackend` branch every eval product hand-rolled (the copies drift) | -| Pick / register a leaf backend, or bring your own agent | `createExecutor({ backend })` / `createExecutorRegistry()` / implement `Executor` — `/loops` | a per-vendor adapter or closed `inline\|sandbox\|cli` switch (won't report through the `UsageEvent` channel) | -| Optimize text or named components with upstream GEPA | `officialGepa({ evaluationId, recipe, ... })`, passed as `improve(...).method` from root `.` | a local GEPA approximation, prompt mutation loop, or silent fallback when Python is unavailable | -| Optimize one text surface with Microsoft SkillOpt | `officialSkillOpt({ evaluationId, trainer, optimizer, ... })`, passed as `improve(...).method` from root `.` | Runtime-owned SkillOpt search or a silent local fallback | -| Improve one profile coordinate | `improve(profile, { surface, method, trainScenarios, selectionScenarios, testScenarios, judges, agent })` from root `.` | an implicit per-surface optimizer, a method that sees final-test cases, or a profile mutation during search | +| Drop a persona⟷agent conversation into an eval matrix as its dispatch | `runPersonaDispatch` → `runProfileMatrix({ dispatch })`: root `.` / `agent-eval/campaign` | a per-agent custom dispatch bridge | +| Best-of-N / parallel-research / map-reduce at equal compute | `fanout(items, opts)`: `/loops` | `Promise.all` over N calls + manual argmax/merge (bypasses the budget pool → breaks equal-k) | +| Produce-then-gate with a real checker | `verify(spec)`: `/loops` | "generate, then self-check with the same model, ship if ok" (collapses selector+judge) | +| Multi-judge review / rubric quorum over one artifact | `panel(spec)`: `/loops` | a judge ensemble that feeds one judge's score into another | +| Fixed sequential chain (plan→implement→…) | `pipeline(stages)`: `/loops` | hand-chained `await`s passing outputs along | +| Adaptive tree search / progressive widening | `widen(spec)` + `flatWidenGate()`: `/loops` | a best-first/MCTS that reads child *scores* to expand (selector=judge); keep `flatWidenGate()` until your gate is proven | +| Define the profile record for a personified run | `definePersona(input)`: `/loops` | a "profile-seam" / agent-config wrapper carrying model+prompt+tools+role | +| Make a worker self-verify / iterate / audit | a **hook / process / skill on its authored `AgentProfile`**: §1.5 | a per-round judge, a `while(!done)` loop, or a bash hill-climb (it's a profile lever) | +| Run an authored profile on a real harness | author the `AgentProfile`, hand it to the **sandbox substrate**: `@tangle-network/sandbox` (`defineAgentProfile`) | a `profile → opencode.json` realizer or any harness-specific config writer | +| Have the supervisor design its workers | author a **full `AgentProfile`** per sub-task (prompt+skills+tools+mcp+hooks+subagents): `/loops` | author a bare `systemPrompt` string (a worker can't act on levers it has no levers for) | +| Write a custom driver Agent and run it directly | `createSupervisor().run(root, task, opts)`: `/loops` | a bespoke orchestrator that spawns sub-agents and tallies cost (equal-compute claim breaks there) | +| Run depth-vs-breadth (or a custom strategy) over a stateful tool domain | `runAgentic({ surface, task, mode\|strategy, budget })`: `/loops` | a hand-rolled `Supervisor.run` + journal/registry, or a depth/breadth loop | +| Author a new topology/strategy compactly | `defineStrategy(name, body)` using `ctx.shot()`+`ctx.critique()`: `/loops` | a 70-line driver with `scope.spawn`/`scope.next` ceremony, or trusting a body-returned score | +| Compare strategies + get a significance report on a domain | `runBenchmark({ environment, tasks, worker, strategies })`: `/loops` | your own strategy-comparison loop / paired-bootstrap / Pareto math | +| Add a stateful tool-using domain | implement `AgenticSurface` (5 hooks: open/tools/call/score/close): `/loops` | a bespoke per-benchmark agent runner / tool-loop harness | +| Run a sandbox coding rollout, round-synchronous (fresh box per round) | `runLoop(options)`: `/loops` | a `new Sandbox()`+acquire+stream+parse+delete loop, or a 2nd winner-selector | +| Run **agent-eval fixture folders** through runtime `runLoop` | agent-eval fixture loading/planning, then `loopCampaignDispatch(...)`: `/loops` | a one-off `runCampaign` dispatch that hand-builds `ExecCtx`, drops loop traces, or forgets token/cost reporting | +| Run + **resume** ONE persistent box across turns | `openSandboxRun(client, opts, deliverable)`: `/loops` | a per-domain `new Sandbox`+`box.fs.read`+delete copy | +| Run **ONE agent turn** on any substrate: box (`streamPrompt`, or the sandbox SDK's autonomous-task `streamTask` via the `box-task` kind with per-task `TaskOptions`), cli-bridge/router `Executor`, or in-process chat backend: as ONE normalized `RuntimeStreamEvent` stream with a guaranteed terminal result+usage event; opt into in-stream `tool_call`/`tool_result` with `preserveToolParts`, or tap the raw sandbox events with `onRawEvent` | `streamAgentTurn(backend, prompt, { signal, timeoutMs, preserveToolParts?, onRawEvent? })` + `collectAgentTurn(stream)`: `/loops` | a per-provider stream→event mapper zoo, a bespoke `streamTask` parser, a hand-faked box around a non-box executor, or raw fetch leaking through the turn abstraction | +| Pick the **execution transport a driven loop runs on** (`sandbox` box / cli-bridge / router) from a product flag | `resolveSandboxClient({ backend })`: `/loops` | a per-product `if (backend === 'router') …` branch re-wiring `createExecutor` + `inlineSandboxClient` | +| Pick the **chat backend an in-process turn runs on** (`router`/`tcloud`/`cli-bridge`/`sandbox`) from a product flag | `resolveAgentBackend({ backend })`: root `.` | the copy-pasted `backend-name → createOpenAICompatibleBackend` branch every eval product hand-rolled (the copies drift) | +| Pick / register a leaf backend, or bring your own agent | `createExecutor({ backend })` / `createExecutorRegistry()` / implement `Executor`: `/loops` | a per-vendor adapter or closed `inline\|sandbox\|cli` switch (won't report through the `UsageEvent` channel) | +| Optimize text or named components with upstream GEPA | `officialGepa({ recipe, ... })`, passed as `improve(...).method` from root `.` | a local GEPA approximation, prompt mutation loop, or silent fallback when Python is unavailable | +| Optimize one text surface with Microsoft SkillOpt | `officialSkillOpt({ trainer, optimizer, ... })`, passed as `improve(...).method` from root `.` | Runtime-owned SkillOpt search or a silent local fallback | +| Improve one profile coordinate | `improve(profile, { surface, executionRef, method, trainScenarios, selectionScenarios, testScenarios, judges, agent, costCeiling })` from root `.`; `executionRef` binds saved work to executable behavior, `agent` receives the exact complete candidate profile, and `costCeiling` limits the whole run | an implicit per-surface optimizer, a method that sees final-test cases, an unmeasured profile mutation, or separate optimizer and final-test spend limits | | Compare complete optimization methods directly | `compareOptimizationMethods(...)` from `agent-eval/campaign` | comparing one method's training score to another method's final score | | Improve repository code | `improve(profile, { surface: 'code', code, scenarios, judge, agent, budget })` from root `.` | passing code through a text optimizer or managing candidate worktrees in product code | -| Decide ship/hold on a candidate (campaign context) | `defaultProductionGate({ holdoutScenarios, deltaThreshold })`; compose with `heldOutGate` / `composeGate` — `agent-eval/contract` | a raw `h1>h0` point comparison on the training set | -| Decide ship/hold from a **`BenchmarkReport`** (per-task cells) | `promotionGate({ report, incumbent, candidate })` — `/loops` | comparing two strategies' mean scores directly; re-deriving the bootstrap | -| Run the full multi-generation strategy flywheel + certify | `runStrategyEvolution(config)` — `/loops` | a bespoke gen0→author→gen1→holdout loop with hand-rolled champion selection | +| Decide ship/hold on a candidate (campaign context) | `defaultProductionGate({ holdoutScenarios, deltaThreshold })`; compose with `heldOutGate` / `composeGate`: `agent-eval/contract` | a raw `h1>h0` point comparison on the training set | +| Decide ship/hold from a **`BenchmarkReport`** (per-task cells) | `promotionGate({ report, incumbent, candidate })`: `/loops` | comparing two strategies' mean scores directly; re-deriving the bootstrap | +| Run the full multi-generation strategy flywheel + certify | `runStrategyEvolution(config)`: `/loops` | a bespoke gen0→author→gen1→holdout loop with hand-rolled champion selection | | Add or run a benchmark from the CLI/harness | `ADAPTERS` / `resolveAdapter(key)`, run via `bench/src/gate-cli.mts` | a per-script `switch(bench)` or a local benchmark-factory map | -| Wire a new benchmark | implement `BenchmarkAdapter` (5 methods) + feed to `runGate` — `bench` | a bespoke per-benchmark run script with its own (self-authored) scoring | -| Measure a topology on a benchmark at equal compute | `runGate(cfg)` (or `runAgentic`/`runBenchmark`) — equal-k holds via the conserved budget pool — `bench`/`/loops` | a batch-blind/batch-oracle/compare zoo, your own usage capture, or equal-k bookkeeping | -| Observe a run's full cost/time | `createWaterfallCollector()` → `anytimeReport()` — `/loops` | a per-step cost/token tally by inspecting events yourself (drifts from billed totals) | -| Meter **one `openSandboxRun` cell's token/cost usage** (the metering seam for bench cells) | `sumSandboxUsage(events)` — `/loops` (folds `extractLlmCallEvent` over the run's events) | a per-bench usage tally re-parsing raw sandbox events (misses usage → integrity-guard rejects the cell) | -| Stand up a **product eval leaderboard** (declare `cases` + `prompt` + `score` → harness×model matrix + ranked board) — START HERE (product leaderboards) | `defineLeaderboard(spec)` — `/loops` (every default overridable: `backends`/`dispatch`/`judges` seams; `runProfileMatrix` stays public as the escape floor; `toBenchmarkAdapter()` registers it into a benchmark registry) | the hand-rolled `expandProfileAxes` + `loopDispatch` + `runProfileMatrix` assembly (~650 lines/product) with its stale cell-cache, zero-token stub-cell, and missing-model-snapshot footguns | -| Render a **multi-profile × multi-axis benchmark leaderboard** (ranked board + score matrix + SVG/HTML charts) from an EXISTING fleet of matrix runs | `leaderboard(records)` + `renderLeaderboardMarkdown` / `renderLeaderboardSvg` / `renderLeaderboardHtml` — `/loops` (feed it `runProfileMatrix().records`, any domain; `defineLeaderboard` calls these for you) | a per-benchmark report/chart renderer; hand-rolled SVG/markdown tables; a curated subset of axes | -| Attach N observers to a running loop | `composeRuntimeHooks(...)` — root export | a second event-bus or callback-prop zoo (there is ONE stream) | -| Ship traces to an OTLP collector | `createOtelExporter()` + `buildLoopOtelSpans()` — root export | your own OTLP serializer or pulling the OTEL SDK | -| Know **what got mounted into a run** / **why a candidate won** | `result.provenance.mounts` / `result.provenance.selectionReceipts` (`MountManifestEntry`/`SelectionReceipt`/`RunProvenance`); declare mounts via the `recordMount` recorder in `prepareBox` — root export | re-reading box contents to reconstruct what was mounted, or re-deriving which candidate the selector picked | +| Wire a new benchmark | implement `BenchmarkAdapter` (5 methods) + feed to `runGate`: `bench` | a bespoke per-benchmark run script with its own (self-authored) scoring | +| Measure a topology on a benchmark at equal compute | `runGate(cfg)` (or `runAgentic`/`runBenchmark`): equal-k holds via the conserved budget pool: `bench`/`/loops` | a batch-blind/batch-oracle/compare zoo, your own usage capture, or equal-k bookkeeping | +| Observe a run's full cost/time | `createWaterfallCollector()` → `anytimeReport()`: `/loops` | a per-step cost/token tally by inspecting events yourself (drifts from billed totals) | +| Meter **one `openSandboxRun` cell's token/cost usage** (the metering seam for bench cells) | `sumSandboxUsage(events)`: `/loops` (folds `extractLlmCallEvent` over the run's events) | a per-bench usage tally re-parsing raw sandbox events (misses usage → integrity-guard rejects the cell) | +| Stand up a **product eval leaderboard** (declare `cases` + `prompt` + `score` → harness×model matrix + ranked board): START HERE (product leaderboards) | `defineLeaderboard(spec)`: `/loops` (every default overridable: `backends`/`dispatch`/`judges` seams; `runProfileMatrix` stays public as the escape floor; `toBenchmarkAdapter()` registers it into a benchmark registry) | the hand-rolled `expandProfileAxes` + `loopDispatch` + `runProfileMatrix` assembly (~650 lines/product) with its stale cell-cache, zero-token stub-cell, and missing-model-snapshot footguns | +| Render a **multi-profile × multi-axis benchmark leaderboard** (ranked board + score matrix + SVG/HTML charts) from an EXISTING fleet of matrix runs | `leaderboard(records)` + `renderLeaderboardMarkdown` / `renderLeaderboardSvg` / `renderLeaderboardHtml`: `/loops` (feed it `runProfileMatrix().records`, any domain; `defineLeaderboard` calls these for you) | a per-benchmark report/chart renderer; hand-rolled SVG/markdown tables; a curated subset of axes | +| Attach N observers to a running loop | `composeRuntimeHooks(...)`: root export | a second event-bus or callback-prop zoo (there is ONE stream) | +| Ship traces to an OTLP collector | `createOtelExporter()` + `buildLoopOtelSpans()`: root export | your own OTLP serializer or pulling the OTEL SDK | +| Know **what got mounted into a run** / **why a candidate won** | `result.provenance.mounts` / `result.provenance.selectionReceipts` (`MountManifestEntry`/`SelectionReceipt`/`RunProvenance`); declare mounts via the `recordMount` recorder in `prepareBox`: root export | re-reading box contents to reconstruct what was mounted, or re-deriving which candidate the selector picked | | State any benchmark/A-B claim | `pairedLift(...)` (bench) over `pairedBootstrap`/`heldoutSignificance` (substrate) | your own bootstrap loop/PRNG per gate; a point lift without `low/high/pairs` | -| Let an agent **delegate ONE generic INTENT** (no fixed coder/researcher type) and get the result + real spend SYNCHRONOUSLY | the **`delegate` tool** — `createDelegateHandler` via `createMcpServer({ delegateSupervisor })`; mount it over the `agent-runtime mcp` bin with `MCP_ENABLE_DELEGATE=1` (the bin authors a supervisor over a `sandbox` backend) — `/mcp` | a hardcoded coder/researcher profile, or task-specific `delegate_code`/`delegate_research` verbs (RETIRED) — `delegate` is the ONE delegation path and the only one with a cost channel | -| Run a coding task INSIDE the agent's OWN sandbox session (a sibling box, fresh branch, validated patch) | `detachedSessionDelegate({ sandboxClient \| executor, workerProfile? })` — `/mcp` (pass the worker `AgentProfile`; omit for a minimal model-only default) | a hardcoded coder profile baked into the delegate; `delegate()` (that spawns workers in a *chosen* backend, not the agent's own session) | +| Let an agent **delegate ONE generic INTENT** (no fixed coder/researcher type) and get the result + real spend SYNCHRONOUSLY | the **`delegate` tool**: `createDelegateHandler` via `createMcpServer({ delegateSupervisor })`; mount it over the `agent-runtime mcp` bin with `MCP_ENABLE_DELEGATE=1` (the bin authors a supervisor over a `sandbox` backend): `/mcp` | a hardcoded coder/researcher profile, or task-specific `delegate_code`/`delegate_research` verbs (RETIRED): `delegate` is the ONE delegation path and the only one with a cost channel | +| Run a coding task INSIDE the agent's OWN sandbox session (a sibling box, fresh branch, validated patch) | `detachedSessionDelegate({ sandboxClient \| executor, workerProfile? })`: `/mcp` (pass the worker `AgentProfile`; omit for a minimal model-only default) | a hardcoded coder profile baked into the delegate; `delegate()` (that spawns workers in a *chosen* backend, not the agent's own session) | | Have a **supervisor spawn + live-drive workers in a backend you choose** and observe or steer them while the coordinator is alive | the **coordination MCP** via `createCoordinationTools` / `serveCoordinationMcp` over a live `Scope`; each worker's leaf is `createExecutor({ backend })` | `detachedSessionDelegate`, which is own-sandbox-session only and one-shot. Supervised-tree restart recovery is not implemented. | -| Stand up a vertical agent in the eval loop | `defineAgent(manifest)` + `createSurfaceImprovementProposer` — `/agent` | a per-vertical manifest parser, surface-validator, or bespoke findings-to-patch mapper | -| Observe + deliver Intelligence on a live agent (send RunRecords + receive certified profile/diffs) | `withIntelligence(agent, { project, target })` — `/intelligence` (proposals surfaced, never auto-applied; `effort: 'off'` proves inference-only billing) | a custom trace-wrapper, a second receive path, or hand-rolled effort/tier config | +| Stand up a vertical agent in the eval loop | `defineAgent(manifest)` + `createSurfaceImprovementProposer`: `/agent` | a per-vertical manifest parser, surface-validator, or bespoke findings-to-patch mapper | +| Observe + deliver Intelligence on a live agent (send RunRecords + receive certified profile/diffs) | `withIntelligence(agent, { project, target })`: `/intelligence` (proposals surfaced, never auto-applied; `effort: 'off'` proves inference-only billing) | a custom trace-wrapper, a second receive path, or hand-rolled effort/tier config | | Turn trace evidence into one measured, review-only agent proposal | `proposeAgentImprovement({ analysis, profile, improvement, buildExperiment, placeCell })` in `/intelligence` (Runtime seals the optimizer ancestry) | manually joining analysis, optimizer ancestry, exact candidate execution, uncertainty, and candidate identity | -| Freeze a measured profile/diff plus a content-addressed code surface into one executable candidate | `buildAgentCandidateBundle(...)`, then `verifyAgentCandidateBundle(...)` at execution — `/candidate-execution` | a product callback that converts profile fields, reproduces Git diff flags, hashes bytes, or assembles `AgentCandidateBundle` by hand | -| Record approve/reject/change-request feedback against one exact proposal | `reviewAgentImprovementProposal(proposal, review)` — `/intelligence` | a mutable status row that is not bound to candidate bytes | +| Freeze a measured profile/diff plus a content-addressed code surface into one executable candidate | `buildAgentCandidateBundle(...)`, then `verifyAgentCandidateBundle(...)` at execution: `/candidate-execution` | a product callback that converts profile fields, reproduces Git diff flags, hashes bytes, or assembles `AgentCandidateBundle` by hand | +| Record approve/reject/change-request feedback against one exact proposal | `reviewAgentImprovementProposal(proposal, review)`: `/intelligence` | a mutable status row that is not bound to candidate bytes | | Run and grade the exact signed baseline-versus-candidate matrix before review | `runAgentCandidateExperiment({ experiment, placeCell })` in `/intelligence`; use `createProtectedExactProcessCandidateExperimentExecutor(...)` for any exact-process provider with protected model grants and pass the remaining product ports as `hostPorts` | product-local pairing, retry, isolation, receipt, and comparison code | | Authorize and execute writes only for the exact measured and approved candidate | `createAgentImprovementActivation(...)`, then `executeAgentImprovementActivation(...)` with one idempotent transition in `/intelligence`; use `createKnowledgeImprovementActivationExecutor(...)` from `/knowledge` for one local KB | a mutable approval flag, a second per-surface approval path, or a write that does not persist an exact result | -| Capture and restore exact task, candidate, or memory workspace bytes | `captureAgentCandidateWorkspace(...)` + `createAgentCandidateWorkspacePort(...)` — `/candidate-execution` | a product-specific archive format, ambient `git checkout`, or a materializer that skips byte/path/mode verification | -| Fold **certified prompt additions into a system prompt you assemble yourself** (product chat routes) | `createCertifiedPromptSource({ target })` → `source.compose(base)` — `/intelligence` (cached, coalesced, fail-closed; `withIntelligence` rides the same source) | a module-scope cache + refresh-window + keep-last-known loop around `pullCertified` in product wiring | +| Capture and restore exact task, candidate, or memory workspace bytes | `captureAgentCandidateWorkspace(...)` + `createAgentCandidateWorkspacePort(...)`: `/candidate-execution` | a product-specific archive format, ambient `git checkout`, or a materializer that skips byte/path/mode verification | +| Fold **certified prompt additions into a system prompt you assemble yourself** (product chat routes) | `createCertifiedPromptSource({ target })` → `source.compose(base)`: `/intelligence` (cached, coalesced, fail-closed; `withIntelligence` rides the same source) | a module-scope cache + refresh-window + keep-last-known loop around `pullCertified` in product wiring | | Produce a frozen KB candidate with runtime agents, readiness checks, and measured supervised spend | `runKnowledgeImprovementJob(options)` from `/knowledge`, then the shared activation path above after review | hand-wiring `improveKnowledgeBase` + a supervised updater, or letting candidate search write live knowledge | -For the full export inventory (every primitive, its import path, its summary — generated, never stale), see `docs/api/primitive-catalog.md`; for per-symbol signatures, the per-module `docs/api/` pages. For the recursive atom (recursion · isolated-or-collaborative artifact · conserved budget · analysts) and the two-timescale architecture, see `docs/architecture.md`. For the profile→run→optimize→ship spine in depth, `docs/concepts.md` + `docs/learning-flywheel.md`. For the Intelligence SDK (Observe + the provable-OFF billing boundary), `docs/intelligence-sdk.md`. +For the full export inventory (every primitive, its import path, its summary: generated, never stale), see `docs/api/primitive-catalog.md`; for per-symbol signatures, the per-module `docs/api/` pages. For the recursive atom (recursion · isolated-or-collaborative artifact · conserved budget · analysts) and the two-timescale architecture, see `docs/architecture.md`. For the profile→run→optimize→ship spine in depth, `docs/concepts.md` + `docs/learning-flywheel.md`. For the Intelligence SDK (Observe + the provable-OFF billing boundary), `docs/intelligence-sdk.md`. -## 2.1 Which front door do I use? — the four public verbs +## 2.1 Which front door do I use?: the four public verbs §2 maps a fine-grained intent to a primitive; this is the coarse router one level up. Pick a front door by **what you hand in**. Each bottoms out at ONE function; the §2 rows above carry each one's "do NOT build" twin. Exact per-symbol signatures + line anchors live in the generated `docs/api/` pages (never stale); the file paths below are what the freshness gate protects. | You hand in… | Front door | Bottoms out at | What it is | |---|---|---|---| -| a **string intent** ("fix the failing auth test") — you don't care HOW | the `delegate` tool | `delegate(intent, opts)` — `src/runtime/supervise/delegate.ts` (MCP handler `createDelegateHandler`, `src/mcp/tools/delegate.ts`) | a default authoring supervisor decomposes the intent and writes the worker profile per sub-task; synchronous, returns the delivered output + `spentTotal`. The ONE delegation path. | +| a **string intent** ("fix the failing auth test"): you don't care HOW | the `delegate` tool | `delegate(intent, opts)`: `src/runtime/supervise/delegate.ts` (MCP handler `createDelegateHandler`, `src/mcp/tools/delegate.ts`) | a default authoring supervisor decomposes the intent and writes the worker profile per sub-task; synchronous, returns the delivered output + `spentTotal`. The ONE delegation path. | | an **authored supervisor `AgentProfile`** + a task | `supervise(profile, task, opts)` | `src/runtime/supervise/supervise.ts` | the one-call LLM-brain driver over the keystone `Supervisor`, scaffolding defaulted. START HERE when you wrote the driver. | -| a **deterministic shot grammar** over a stateful tool domain | `runAgentic(opts)` | `src/runtime/strategy.ts` | runs a `Strategy` (depth/breadth/custom) through the `Supervisor` — programmatic, no LLM picking the shape. | -| a **deterministic topology combinator** (`loopUntil`/`fanout`/`verify`/`panel`/`pipeline`) over a persona | `runPersonified(options)` | `src/runtime/personify/persona.ts` | composes a persona + a `CombinatorShape` over the `Supervisor` — programmatic. | +| a **deterministic shot grammar** over a stateful tool domain | `runAgentic(opts)` | `src/runtime/strategy.ts` | runs a `Strategy` (depth/breadth/custom) through the `Supervisor`: programmatic, no LLM picking the shape. | +| a **deterministic topology combinator** (`loopUntil`/`fanout`/`verify`/`panel`/`pipeline`) over a persona | `runPersonified(options)` | `src/runtime/personify/persona.ts` | composes a persona + a `CombinatorShape` over the `Supervisor`: programmatic. | Rule of thumb: `delegate` = "I don't care how"; `supervise` = "I authored the driver"; `runAgentic`/`runPersonified` = "I want a deterministic topology, no LLM choosing the shape." All four run over the one `Executor` port on the conserved budget pool, so equal-compute holds by construction. -**Two-agent patterns — compose a shape, don't hand-roll a turn loop:** +**Two-agent patterns: compose a shape, don't hand-roll a turn loop:** | Pattern | Use | Bottoms out at | |---|---|---| -| **researcher → engineer** (gather, then build) | `defineStrategy(name, body)` — both agents in one body via `ctx.shot()` + `ctx.critique()` | `src/runtime/strategy.ts:789` | -| **implement → verify** (build, then a SEPARATE checker gates it — selector ≠ judge) | `verify(spec)` as the `shape` | `src/runtime/personify/combinators.ts:333` | +| **researcher → engineer** (gather, then build) | `defineStrategy(name, body)`: both agents in one body via `ctx.shot()` + `ctx.critique()` | `src/runtime/strategy.ts:789` | +| **implement → verify** (build, then a SEPARATE checker gates it: selector ≠ judge) | `verify(spec)` as the `shape` | `src/runtime/personify/combinators.ts:333` | | **N-judge panel** (fan judges out, merge verdicts) | `panel(spec)` as the `shape` | `src/runtime/personify/combinators.ts:273` | diff --git a/examples/ablation-suite/gepa-driver-prompt.ts b/examples/ablation-suite/gepa-driver-prompt.ts index d8ca3359..201cdb65 100644 --- a/examples/ablation-suite/gepa-driver-prompt.ts +++ b/examples/ablation-suite/gepa-driver-prompt.ts @@ -12,14 +12,9 @@ * check — there is no model in the scoring loop to flatter it. */ -import type { - DispatchContext, - JudgeConfig, - MutableSurface, - Scenario, -} from '@tangle-network/agent-eval/contract' -import type { AgentProfile } from '@tangle-network/agent-interface' -import { improve, officialGepa } from '@tangle-network/agent-runtime' +import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract' +import { type AgentProfile, canonicalCandidateDigest } from '@tangle-network/agent-interface' +import { improve, officialGepa, type ReadonlyAgentProfile } from '@tangle-network/agent-runtime' import { type AgenticSurface, type AgenticTask, @@ -71,8 +66,6 @@ export async function optimizeDriverPrompt(opts: { * inference). Defaults to the worker's router + model when omitted. */ supervisorRouter?: { baseUrl: string; apiKey: string; model: string } reflectionModel?: string - /** Change when execution or scoring behavior changes outside the listed runtime knobs. */ - evaluationId?: string maxEvaluations?: number maxProposerCostUsd?: number maxConcurrency?: number @@ -118,21 +111,18 @@ export async function optimizeDriverPrompt(opts: { const selectionScenarios = mapScenarios(selectionTasks) const testScenarios = mapScenarios(testTasks) - // The agent under improvement: it receives the CURRENT candidate driver prompt (the surface string) - // and runs a REAL supervised rollout with that prompt as the supervisor brain's standing instruction. + // The agent under improvement receives the complete candidate profile and runs a real supervised + // rollout with its prompt as the supervisor brain's standing instruction. // The returned artifact is the harness-verified deployable outcome — `resolved`/`score` come from the // surface's completion oracle, not a self-report, so the candidate cannot fabricate a win. const agent = async ( - candidate: MutableSurface, + candidate: ReadonlyAgentProfile, scenario: DriverPromptScenario, ctx: DispatchContext, ): Promise => { - // The candidate is the driver prompt. A `CodeSurface` is not a prompt — this loop only optimizes the - // string driver prompt, so a non-string candidate is a wiring error that must fail loud. - if (typeof candidate !== 'string') { - throw new Error( - `optimizeDriverPrompt: candidate surface is a CodeSurface, not a driver prompt — this loop optimizes the string driver prompt only`, - ) + const systemPrompt = candidate.prompt?.systemPrompt + if (systemPrompt === undefined) { + throw new Error('optimizeDriverPrompt: candidate profile has no system prompt') } const paid = await ctx.cost.runPaidCall({ channel: 'agent', @@ -140,7 +130,7 @@ export async function optimizeDriverPrompt(opts: { model: supervisorRouter.model, signal: ctx.signal, execute: () => - superviseSurface({ name: 'driver', systemPrompt: candidate }, scenario.task, { + superviseSurface({ name: 'driver', systemPrompt }, scenario.task, { surface, worker, // A small conserved pool: enough for the driver's turns plus several worker spawns so the @@ -210,19 +200,19 @@ export async function optimizeDriverPrompt(opts: { }) const result = await improve(profile, { surface: 'prompt', + executionRef: canonicalCandidateDigest({ + callback: 'examples/ablation-suite/gepa-driver-prompt', + workerModel: worker.model, + workerMaxTokens: worker.maxTokens ?? null, + workerTurns: worker.innerTurns ?? null, + workerShots: worker.budget ?? null, + supervisorModel: supervisorRouter.model, + workerEndpoint: new URL(worker.routerBaseUrl).origin, + supervisorEndpoint: new URL(supervisorRouter.baseUrl).origin, + }), method: officialGepa({ objective: 'Improve the complete standing prompt used by a supervisor that spawns, steers, and verifies coding workers.', - evaluationId: - opts.evaluationId ?? - [ - 'ablation-driver-prompt', - `worker=${worker.model}`, - `supervisor=${supervisorRouter.model}`, - `tokens=${worker.maxTokens ?? 'default'}`, - `turns=${worker.innerTurns ?? 'default'}`, - `shots=${worker.budget ?? 'default'}`, - ].join('|'), background: [ 'Return only the complete supervisor prompt.', 'The supervisor must verify settled workers, target subsequent workers at observed failures, and stop only after the task check passes.', diff --git a/examples/improve/README.md b/examples/improve/README.md index 5823a255..7bd6c027 100644 --- a/examples/improve/README.md +++ b/examples/improve/README.md @@ -14,10 +14,11 @@ Runs offline, no credentials. ## What it does, step by step 1. Runtime extracts the exact profile field selected by `surface`. -2. The supplied `OptimizationMethod` generates and selects a candidate using train and selection cases. -3. Runtime scores the baseline and candidate on the untouched final-test cases. -4. Runtime returns `ship` only when the paired confidence interval clears the required lift. -5. Approval and activation remain separate operations. +2. Runtime binds saved work to `executionRef` plus the complete baseline profile. +3. The supplied `OptimizationMethod` generates and selects a candidate using train and selection cases. +4. Runtime scores the baseline and candidate on the untouched final-test cases. +5. Runtime returns `ship` only when the paired confidence interval clears the required lift. +6. Approval and activation remain separate operations. ## What you'll see diff --git a/examples/improve/improve.ts b/examples/improve/improve.ts index b5ca8dec..5e44f73d 100644 --- a/examples/improve/improve.ts +++ b/examples/improve/improve.ts @@ -2,14 +2,13 @@ import { makeFinding } from '@tangle-network/agent-eval' import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' -import type { - DispatchContext, - JudgeConfig, - MutableSurface, - Scenario, -} from '@tangle-network/agent-eval/contract' -import type { AgentProfile } from '@tangle-network/agent-interface' -import { type ImproveMethodFactory, improve } from '@tangle-network/agent-runtime' +import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract' +import { type AgentProfile, canonicalCandidateDigest } from '@tangle-network/agent-interface' +import { + type ImproveMethodFactory, + improve, + type ReadonlyAgentProfile, +} from '@tangle-network/agent-runtime' export interface DemoScenario extends Scenario { kind: 'demo' @@ -26,7 +25,7 @@ export const testScenarios = scenarios.slice(8) // The agent returns the surface verbatim as the artifact AND reports usage, so the backend-integrity // guard sees a real backend rather than a stub-zero cell. No LLM. export const agent = async ( - surface: MutableSurface, + candidate: ReadonlyAgentProfile, _scenario: DemoScenario, ctx: DispatchContext, ): Promise => { @@ -35,7 +34,7 @@ export const agent = async ( actor: 'example', model: 'deterministic-example', maximumCharge: { externallyEnforcedMaximumUsd: 0.0001 }, - execute: async () => String(surface), + execute: async () => candidate.prompt?.systemPrompt ?? '', receipt: () => ({ model: 'deterministic-example', inputTokens: 1, @@ -83,10 +82,15 @@ const findings = [ ] export const profile: AgentProfile = { name: 'demo', prompt: { systemPrompt: 'BASELINE' } } +export const executionRef = canonicalCandidateDigest({ + callback: 'examples/improve/agent', + model: 'deterministic-example', +}) async function main(): Promise { const out = await improve(profile, { surface: 'prompt', + executionRef, method: scriptedWinner, findings, trainScenarios, @@ -103,7 +107,7 @@ async function main(): Promise { 'improve() proposed a detached prompt candidate and measured it on final-test scenarios (offline complete method, deterministic judge):', ) console.log(`decision: ${out.decision} lift: ${out.lift.toFixed(3)}`) - console.log(`candidate prompt: ${out.candidate.profile?.prompt?.systemPrompt}`) + console.log(`candidate prompt: ${out.candidate.profile.prompt?.systemPrompt}`) console.log(`live prompt unchanged: ${profile.prompt?.systemPrompt}`) } diff --git a/examples/intelligence-recommend/intelligence-recommend.ts b/examples/intelligence-recommend/intelligence-recommend.ts index 6f8809cd..9b42f2be 100644 --- a/examples/intelligence-recommend/intelligence-recommend.ts +++ b/examples/intelligence-recommend/intelligence-recommend.ts @@ -23,6 +23,7 @@ import { createIntelligenceClient } from '@tangle-network/agent-runtime/intellig import type { LoopTraceEvent } from '@tangle-network/agent-runtime/loops' import { agent, + executionRef, judge, profile, scriptedWinner, @@ -79,6 +80,7 @@ const findings = [ async function main(): Promise { const out = await improve(profile, { surface: 'prompt', + executionRef, method: scriptedWinner, findings, trainScenarios, @@ -94,7 +96,7 @@ async function main(): Promise { console.log(`trace recorded: ${traceId}`) console.log(`findings derived: ${findings.length}`) console.log(`candidate decision: ${out.decision}`) - console.log(`candidate prompt: ${out.candidate.profile?.prompt?.systemPrompt}`) + console.log(`candidate prompt: ${out.candidate.profile.prompt?.systemPrompt}`) console.log(`live prompt unchanged: ${profile.prompt?.systemPrompt}`) } diff --git a/package.json b/package.json index b874bec5..ac535e2e 100644 --- a/package.json +++ b/package.json @@ -122,7 +122,7 @@ }, "devDependencies": { "@biomejs/biome": "^2.4.15", - "@tangle-network/agent-eval": "0.126.0", + "@tangle-network/agent-eval": "0.126.1", "@tangle-network/agent-interface": "0.32.0", "@tangle-network/sandbox": "^0.11.1", "@types/node": "^25.9.3", @@ -154,7 +154,7 @@ "license": "MIT", "packageManager": "pnpm@10.28.0", "peerDependencies": { - "@tangle-network/agent-eval": ">=0.126.0 <0.127.0", + "@tangle-network/agent-eval": ">=0.126.1 <0.127.0", "@tangle-network/agent-interface": ">=0.32.0 <0.33.0", "@tangle-network/sandbox": ">=0.11.1 <1.0.0", "playwright": "^1.40.0" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index db9e0db7..dfd8309c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -22,8 +22,8 @@ importers: specifier: ^2.4.15 version: 2.4.15 '@tangle-network/agent-eval': - specifier: 0.126.0 - version: 0.126.0(typescript@5.9.3) + specifier: 0.126.1 + version: 0.126.1(typescript@5.9.3) '@tangle-network/agent-interface': specifier: 0.32.0 version: 0.32.0 @@ -61,8 +61,8 @@ importers: bench: dependencies: '@tangle-network/agent-eval': - specifier: 0.126.0 - version: 0.126.0(typescript@5.9.3) + specifier: 0.126.1 + version: 0.126.1(typescript@5.9.3) '@tangle-network/agent-interface': specifier: 0.32.0 version: 0.32.0 @@ -691,8 +691,8 @@ packages: engines: {node: '>=20'} hasBin: true - '@tangle-network/agent-eval@0.126.0': - resolution: {integrity: sha512-IY2GJ+DmZcTzWLYj+87sz+agf3TZ/pQyYD38NUZifCIMh7pgO7le/rebPJFII9mNHMek+ojCH8edUm9q90TydA==} + '@tangle-network/agent-eval@0.126.1': + resolution: {integrity: sha512-Tnrw12ecQF2d3IRU1mWSQFEq5wPCO/Xx11tQnF7bmxgAZjU3XgZGBBPoz9CH4YLndlnPre2ybzsslp6iZgQjyw==} engines: {node: '>=20'} hasBin: true @@ -1800,7 +1800,7 @@ snapshots: - typescript - utf-8-validate - '@tangle-network/agent-eval@0.126.0(typescript@5.9.3)': + '@tangle-network/agent-eval@0.126.1(typescript@5.9.3)': dependencies: '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) '@ax-llm/ax': 23.0.1(zod@4.4.3) diff --git a/scripts/verify-package-exports.mjs b/scripts/verify-package-exports.mjs index a13aa7af..1da80a2e 100644 --- a/scripts/verify-package-exports.mjs +++ b/scripts/verify-package-exports.mjs @@ -347,7 +347,7 @@ try { '--eval', ` const runtime = await import('@tangle-network/agent-runtime') - for (const name of ['improve']) { + for (const name of ['improve', 'officialGepa', 'officialSkillOpt']) { if (typeof runtime[name] !== 'function') throw new Error('missing improvement export ' + name) } if ('loadAgentImprovementProposalFixture' in runtime) { diff --git a/src/candidate-execution/profile.ts b/src/candidate-execution/profile.ts index f7e72479..349c2903 100644 --- a/src/candidate-execution/profile.ts +++ b/src/candidate-execution/profile.ts @@ -181,7 +181,7 @@ function candidateProfileFromGenericProfile(profile: AgentProfile): AgentCandida /** Prove the measured generic profile and sealed candidate profile describe the same behavior. */ export function assertCandidateProfileBinding( - measuredInput: AgentProfile, + measuredInput: unknown, bundled: AgentCandidateProfile, ): void { const measured = parseExactAgentProfile(measuredInput, 'measured agent profile') diff --git a/src/improvement/improve.test.ts b/src/improvement/improve.test.ts index 06694aa5..78ca3fea 100644 --- a/src/improvement/improve.test.ts +++ b/src/improvement/improve.test.ts @@ -16,10 +16,11 @@ import type { MutableSurface, Scenario, } from '@tangle-network/agent-eval/contract' -import type { AgentProfile } from '@tangle-network/agent-interface' +import { type AgentProfile, canonicalCandidateDigest } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { ConfigError } from '../errors' import { improve } from './improve' +import type { ReadonlyAgentProfile } from './profile-types' interface TestScenario extends Scenario { kind: 'fixture' @@ -36,6 +37,7 @@ const testScenarios: TestScenario[] = [ { id: 'test-b', kind: 'fixture' }, ] const allScenarios = [...trainScenarios, ...selectionScenarios, ...testScenarios] +const executionRef = canonicalCandidateDigest({ fixture: 'improve-method' }) const improvementJudge: JudgeConfig = { name: 'improvement', @@ -46,8 +48,8 @@ const improvementJudge: JudgeConfig = { }, } -async function paidText( - surface: MutableSurface, +async function paidArtifact( + text: string, _scenario: TestScenario, ctx: DispatchContext, ): Promise { @@ -56,7 +58,7 @@ async function paidText( actor: 'test-agent', model: 'deterministic-test', maximumCharge: { externallyEnforcedMaximumUsd: 0.0001 }, - execute: async () => ({ text: String(surface) }), + execute: async () => ({ text }), receipt: () => ({ model: 'deterministic-test', inputTokens: 1, @@ -68,6 +70,14 @@ async function paidText( return paid.value } +async function paidProfile( + profile: ReadonlyAgentProfile, + scenario: TestScenario, + ctx: DispatchContext, +): Promise { + return paidArtifact(profile.prompt?.systemPrompt ?? '', scenario, ctx) +} + function fixedMethod( winnerSurface: MutableSurface, inspect?: (input: OptimizationMethodInput) => void, @@ -87,11 +97,12 @@ function fixedMethod( function methodOptions(method: OptimizationMethod): { method: OptimizationMethod + executionRef: typeof executionRef trainScenarios: TestScenario[] selectionScenarios: TestScenario[] testScenarios: TestScenario[] judges: JudgeConfig[] - agent: typeof paidText + agent: typeof paidProfile runDir: string storage: ReturnType resamples: number @@ -99,11 +110,12 @@ function methodOptions(method: OptimizationMethod): } { return { method, + executionRef, trainScenarios, selectionScenarios, testScenarios, judges: [improvementJudge], - agent: paidText, + agent: paidProfile, runDir: `mem://improve-method-${Math.random()}`, storage: inMemoryCampaignStorage(), resamples: 40, @@ -120,6 +132,11 @@ describe('improve method execution', () => { it('runs a complete method without exposing final-test cases and materializes its prompt', async () => { let observed: OptimizationMethodInput | undefined const profile = promptProfile() + const expectedEvaluationRef = canonicalCandidateDigest({ + executionRef, + baselineProfileDigest: canonicalCandidateDigest(profile), + surface: 'prompt', + }) const result = await improve(profile, { ...methodOptions(fixedMethod('improved prompt', (input) => (observed = input))), surface: 'prompt', @@ -127,6 +144,15 @@ describe('improve method execution', () => { expect(observed?.trainScenarios.map((scenario) => scenario.id)).toEqual(['train']) expect(observed?.selectionScenarios.map((scenario) => scenario.id)).toEqual(['selection']) + expect(observed?.runOptions.dispatchRef).toBe(`improve:${expectedEvaluationRef}`) + expect(observed?.judges[0]?.judgeVersion).toBe( + canonicalCandidateDigest({ + evaluationRef: expectedEvaluationRef, + name: improvementJudge.name, + dimensions: improvementJudge.dimensions, + declaredJudgeVersion: null, + }), + ) expect(JSON.stringify(observed)).not.toContain('test-a') expect(result.mode).toBe('method') expect(result.method).toBe('fixed-method') @@ -138,6 +164,105 @@ describe('improve method execution', () => { expect(Object.isFrozen(result.candidate)).toBe(true) }) + it('resumes an identical profile run without dispatching another agent call', async () => { + const storage = inMemoryCampaignStorage() + let agentCalls = 0 + const options = { + ...methodOptions(fixedMethod('improved prompt')), + storage, + runDir: 'mem://improve-exact-resume', + agent: async ( + candidate: ReadonlyAgentProfile, + scenario: TestScenario, + ctx: DispatchContext, + ) => { + agentCalls += 1 + return paidProfile(candidate, scenario, ctx) + }, + } + + const first = await improve(promptProfile(), options) + const callsAfterFirst = agentCalls + const second = await improve(promptProfile(), options) + + expect(callsAfterFirst).toBeGreaterThan(0) + expect(agentCalls).toBe(callsAfterFirst) + expect(second.cost).toEqual(first.cost) + expect(second.candidate.profile).toEqual(first.candidate.profile) + }) + + it('does not reuse saved measurements after executable or baseline identity changes', async () => { + const storage = inMemoryCampaignStorage() + const runDir = 'mem://improve-identity-change' + let agentCalls = 0 + const options = { + ...methodOptions(fixedMethod('improved prompt')), + storage, + runDir, + agent: async ( + candidate: ReadonlyAgentProfile, + scenario: TestScenario, + ctx: DispatchContext, + ) => { + agentCalls += 1 + return paidProfile(candidate, scenario, ctx) + }, + } + await improve(promptProfile(), options) + const callsAfterBaseline = agentCalls + + await improve(promptProfile(), { + ...options, + executionRef: canonicalCandidateDigest({ fixture: 'changed-execution' }), + }) + expect(agentCalls).toBeGreaterThan(callsAfterBaseline) + const callsAfterExecutionChange = agentCalls + + await improve( + { + ...promptProfile(), + description: 'changed complete profile', + }, + options, + ) + expect(agentCalls).toBeGreaterThan(callsAfterExecutionChange) + }) + + it('requires a content-addressed executable identity', async () => { + await expect( + improve(promptProfile(), { + ...methodOptions(fixedMethod('improved prompt')), + executionRef: undefined as never, + }), + ).rejects.toThrow(/executionRef must be a lowercase sha256/) + await expect( + improve(promptProfile(), { + ...methodOptions(fixedMethod('improved prompt')), + executionRef: 'callback-v1' as never, + }), + ).rejects.toThrow(/executionRef must be a lowercase sha256/) + }) + + it('propagates caller cancellation into the optimization method', async () => { + const controller = new AbortController() + const reason = new Error('caller stopped improvement') + controller.abort(reason) + let observedAbortedSignal = false + const method = fixedMethod('improved prompt') + method.optimize = async (input) => { + observedAbortedSignal = input.runOptions.signal?.aborted === true + throw input.runOptions.signal?.reason + } + + await expect( + improve(promptProfile(), { + ...methodOptions(method), + signal: controller.signal, + }), + ).rejects.toThrow(reason.message) + expect(observedAbortedSignal).toBe(true) + }) + it('passes current findings to a method factory', async () => { const findings = [{ claim: 'answers omit citations' }] let observedFindings: readonly unknown[] | undefined @@ -298,13 +423,7 @@ describe('improve method execution', () => { tools: JSON.parse(components.tools ?? '{}') as Record, }), }, - agent: async (surface, scenario, ctx) => { - const text = - typeof surface === 'object' && surface.kind === 'components' - ? (surface.components.prompt ?? '') - : String(surface) - return paidText(text, scenario, ctx) - }, + agent: paidProfile, }) expect(observedBaseline).toEqual({ @@ -359,18 +478,113 @@ describe('improve method execution', () => { read: (current) => ({ prompt: current.prompt?.systemPrompt ?? '' }), apply: (current) => ({ ...current }), }, - agent: async (surface, scenario, ctx) => - paidText( - typeof surface === 'object' && surface.kind === 'components' - ? (surface.components.prompt ?? '') - : String(surface), - scenario, - ctx, - ), + agent: paidProfile, }), ).rejects.toThrow(/round-trip every winning component/) }) + it('rejects a component adapter that changes unmeasured baseline fields', async () => { + await expect( + improve(promptProfile(), { + ...methodOptions( + fixedMethod({ + kind: 'components', + components: { prompt: 'improved prompt' }, + }), + ), + surface: 'agent-profile', + profileComponents: { + read: (current) => ({ prompt: current.prompt?.systemPrompt ?? '' }), + apply: (current, components) => ({ + ...current, + description: 'unmeasured adapter side effect', + prompt: { ...current.prompt, systemPrompt: components.prompt }, + }), + }, + }), + ).rejects.toThrow(/must reproduce the complete baseline profile exactly/) + }) + + it('scores and returns the same complete profile produced by a component mapping', async () => { + const observed: ReadonlyAgentProfile[] = [] + const result = await improve( + { + name: 'fixture-agent', + prompt: { systemPrompt: 'baseline' }, + tools: { Bash: false }, + }, + { + ...methodOptions( + fixedMethod({ + kind: 'components', + components: { prompt: 'improved prompt' }, + }), + ), + surface: 'agent-profile', + profileComponents: { + read: (current) => ({ prompt: current.prompt?.systemPrompt ?? '' }), + apply: (current, components) => ({ + ...current, + prompt: { ...current.prompt, systemPrompt: components.prompt }, + tools: { Bash: components.prompt === 'improved prompt' }, + }), + }, + agent: async (candidate, scenario, ctx) => { + observed.push(candidate) + return paidProfile(candidate, scenario, ctx) + }, + }, + ) + + const measuredWinner = observed.find( + (candidate) => + candidate.prompt?.systemPrompt === 'improved prompt' && candidate.tools?.Bash === true, + ) + expect(result.candidate.profile?.tools).toEqual({ Bash: true }) + expect(measuredWinner).toBeDefined() + expect(result.candidate.profile).toBe(measuredWinner) + expect(observed.every((candidate) => Object.isFrozen(candidate))).toBe(true) + expect(observed.every((candidate) => Object.isFrozen(candidate.prompt))).toBe(true) + expect(observed.every((candidate) => Object.isFrozen(candidate.tools))).toBe(true) + }) + + it('holds an apparent win when any run cost is incompletely accounted', async () => { + const method = fixedMethod('improved prompt') + method.optimize = async () => ({ + winnerSurface: 'improved prompt', + cost: { + totalCostUsd: 0, + accountingComplete: false, + incompleteReasons: ['optimizer model cost unavailable'], + }, + }) + + const result = await improve(promptProfile(), methodOptions(method)) + + expect(result.liftInterval.low).toBeGreaterThan(0) + expect(result.cost.accountingComplete).toBe(false) + expect(result.decision).toBe('hold') + }) + + it('rejects method-reported spend above the configured total limit', async () => { + const method = fixedMethod('improved prompt') + method.optimize = async () => ({ + winnerSurface: 'improved prompt', + cost: { + totalCostUsd: 2, + accountingComplete: true, + incompleteReasons: [], + }, + }) + + await expect( + improve(promptProfile(), { + ...methodOptions(method), + costCeiling: 1, + }), + ).rejects.toThrow(/reported total cost \$2\.\d+ exceeds costCeiling \$1/) + }) + it('rejects an invalid method and malformed profile output', async () => { await expect( improve(promptProfile(), { @@ -450,7 +664,7 @@ async function paidCodeText( if (typeof surface !== 'object' || surface === null || !('worktreeRef' in surface)) { throw new Error('expected a code surface') } - return paidText( + return paidArtifact( readFileSync(join(String(surface.worktreeRef), 'module.txt'), 'utf8'), _scenario, ctx, diff --git a/src/improvement/improve.ts b/src/improvement/improve.ts index 6cf9c3fe..05b80edd 100644 --- a/src/improvement/improve.ts +++ b/src/improvement/improve.ts @@ -36,6 +36,7 @@ import { type AgentProfileResourceRef, agentProfileSchema, type Sha256Digest, + sha256DigestSchema, } from '@tangle-network/agent-interface' import { canonicalCandidateDigest, immutableCandidateValue } from '../candidate-execution/digest' import { ConfigError } from '../errors' @@ -47,6 +48,7 @@ import { improvementDriver, type ManagedImprovementDriver, } from './improvement-driver' +import type { ReadonlyAgentProfile } from './profile-types' import { rawTraceDistiller } from './raw-trace-distiller' import { applyRolloutPolicyToProfile, @@ -76,11 +78,15 @@ export type ImproveProfileSurface = Exclude export interface ImproveMethodContext { /** Validated baseline profile. */ - readonly profile: Readonly + readonly profile: ReadonlyAgentProfile + /** Runtime-derived identity for upstream optimizer resume state. */ + readonly evaluationRef: Sha256Digest /** Exact profile coordinate being optimized. */ readonly surface: ImproveProfileSurface /** Exact bytes supplied to the optimization method. */ readonly baselineSurface: MutableSurface + /** Structured value represented by `baselineSurface`, before serialization. */ + readonly baselineValue: unknown /** Findings produced before this search, if any. */ readonly findings: readonly unknown[] } @@ -94,23 +100,41 @@ export type ImproveMethodSource = | OptimizationMethod | ImproveMethodFactory +/** Runs one exact materialized profile on one scenario. */ +export type ImproveProfileAgent = ( + profile: ReadonlyAgentProfile, + scenario: TScenario, + ctx: Parameters< + CompareOptimizationMethodsOptions['dispatchWithSurface'] + >[2], +) => Promise + +export type ImproveOptimizationRunOptions = Omit< + NonNullable['optimizationRunOptions']>, + 'dispatchRef' +> + /** Complete-method configuration for every non-code profile surface. */ export type ImproveMethodOptions = Omit< CompareOptimizationMethodsOptions, - 'baselineSurface' | 'dispatchWithSurface' | 'methods' | 'optimizationConcurrency' + | 'baselineSurface' + | 'dispatchRef' + | 'dispatchWithSurface' + | 'methods' + | 'optimizationConcurrency' + | 'optimizationRunOptions' > & { /** Exact profile coordinate optimized by `method`. Default `'prompt'`. */ surface?: ImproveProfileSurface + /** + * Immutable digest of `agent`, profile component mapping, models, tools, and + * every closure or external setting that can change measured behavior. + */ + executionRef: Sha256Digest /** A complete optimizer or a factory that can incorporate current findings. */ method: ImproveMethodSource - /** Runs one candidate surface on one scenario. */ - agent: ( - surface: MutableSurface, - scenario: TScenario, - ctx: Parameters< - CompareOptimizationMethodsOptions['dispatchWithSurface'] - >[2], - ) => Promise + /** Runs the exact complete profile materialized from one candidate surface. */ + agent: ImproveProfileAgent /** Trace or analyst findings available to a method factory. */ findings?: readonly unknown[] /** Select the exact inline skill document for `surface: 'skills'`. */ @@ -120,6 +144,8 @@ export type ImproveMethodOptions = Omit< * Valid only with `surface: 'agent-profile'`. */ profileComponents?: ImproveProfileComponents + /** Shared settings for method train and selection calls. */ + optimizationRunOptions?: ImproveOptimizationRunOptions /** Ship only when the paired final-test interval is entirely above this lift. Default `0`. */ minimumLift?: number } @@ -174,9 +200,12 @@ export interface ImproveSkillsOptions { /** Caller-owned mapping for optimizing several profile fields as one candidate. */ export interface ImproveProfileComponents { /** Extract the exact named text components optimized together. */ - read(profile: Readonly): Readonly> + read(profile: ReadonlyAgentProfile): Readonly> /** Apply a complete winning component map to a detached profile. */ - apply(profile: Readonly, components: Readonly>): AgentProfile + apply( + profile: ReadonlyAgentProfile, + components: Readonly>, + ): ReadonlyAgentProfile } export interface ImproveCodeOptions { @@ -201,15 +230,23 @@ export interface ImproveCodeOptions { generator?: CandidateGenerator } -export interface ImprovementCandidate { +export interface ImprovementProfileCandidate { /** Surface searched by this run. */ - surface: ImproveSurface + surface: ImproveProfileSurface /** Exact winning value returned by agent-eval. */ value: MutableSurface - /** Detached profile candidate when the surface maps directly to AgentProfile. */ - profile?: AgentProfile + /** Exact complete profile instance measured on the final cases. */ + profile: ReadonlyAgentProfile } +export interface ImprovementCodeCandidate { + surface: 'code' + value: MutableSurface + profile?: never +} + +export type ImprovementCandidate = ImprovementProfileCandidate | ImprovementCodeCandidate + /** Normalized spend reported for one Runtime improvement run. */ export interface ImproveCost { totalCostUsd: number @@ -223,11 +260,15 @@ export interface ImproveLineage { runId: string /** Exact train-plus-selection scenario payloads exposed to candidate selection. */ developmentSplitDigest: Sha256Digest + /** Complete callback, materializer, model, tool, and closure identity for a profile run. */ + executionRef?: Sha256Digest + /** Complete baseline profile identity for a profile run. */ + baselineProfileDigest?: Sha256Digest } -interface ImproveResultBase { +interface ImproveResultBase { /** Frozen candidate only. Live state is changed through an approved activation. */ - candidate: ImprovementCandidate + candidate: TCandidate /** Final-test decision for this search result. */ decision: SelfImproveResult['gateDecision'] /** Final-test lift when one was measured. */ @@ -247,7 +288,7 @@ interface ImproveResultBase { dispose(): Promise } -export interface ImproveMethodResult extends ImproveResultBase { +export interface ImproveMethodResult extends ImproveResultBase { mode: 'method' method: string /** External optimizer package and resumable run identity, when reported. */ @@ -259,7 +300,7 @@ export interface ImproveMethodResult extends ImproveResultBase { } export interface ImproveCodeResult - extends ImproveResultBase { + extends ImproveResultBase { mode: 'code' raw: SelfImproveResult } @@ -268,34 +309,59 @@ export type ImproveResult = | ImproveMethodResult | ImproveCodeResult -/** Extract the baseline optimized by a method. Prompt, skills, and memory are - * text; config fields are JSON; a caller may map the full profile to named - * components explicitly. */ -function baselineSurfaceFor( - profile: AgentProfile, +interface PreparedProfileSurface { + surface: MutableSurface + value: unknown +} + +/** Extract the baseline optimized by a method and retain its structured value + * so external optimizers can inspect it for private fields before serialization. */ +function prepareProfileSurface( + profile: ReadonlyAgentProfile, surface: ImproveSurface, skills?: ImproveSkillsOptions, profileComponents?: ImproveProfileComponents, -): MutableSurface { +): PreparedProfileSurface { switch (surface) { case 'prompt': - return profile.prompt?.systemPrompt ?? '' - case 'skills': - return inlineSkill(profile, skills).content - case 'tools': - return canonicalJson(profile.tools ?? {}) - case 'mcp': - return canonicalJson(profile.mcp ?? {}) - case 'hooks': - return canonicalJson(profile.hooks ?? {}) - case 'subagents': - return canonicalJson(profile.subagents ?? {}) - case 'agent-profile': - return profileComponents - ? componentSurface(profileComponents.read(profile), 'profileComponents.read') - : canonicalJson(profile) - case 'memory': - return profileInstructions(profile) + return { + surface: profile.prompt?.systemPrompt ?? '', + value: profile.prompt?.systemPrompt ?? '', + } + case 'skills': { + const value = inlineSkill(profile, skills).content + return { surface: value, value } + } + case 'tools': { + const value = profile.tools ?? {} + return { surface: canonicalJson(value), value } + } + case 'mcp': { + const value = profile.mcp ?? {} + return { surface: canonicalJson(value), value } + } + case 'hooks': { + const value = profile.hooks ?? {} + return { surface: canonicalJson(value), value } + } + case 'subagents': { + const value = profile.subagents ?? {} + return { surface: canonicalJson(value), value } + } + case 'agent-profile': { + if (profileComponents) { + const value = profileComponents.read(profile) + return { + surface: componentSurface(value, 'profileComponents.read'), + value, + } + } + return { surface: canonicalJson(profile), value: profile } + } + case 'memory': { + const value = profileInstructions(profile) + return { surface: value, value } + } case 'rollout-policy': { const policy = structuralRolloutPolicyFromProfile(profile) if (!policy) { @@ -303,7 +369,7 @@ function baselineSurfaceFor( "improve(): surface 'rollout-policy' requires an existing structural rollout policy", ) } - return serializeRolloutPolicy(policy) + return { surface: serializeRolloutPolicy(policy), value: policy } } case 'code': throw new ConfigError( @@ -593,11 +659,11 @@ function assertCandidateSurfaceKind( /** Materialize a detached profile candidate without changing the baseline. */ function materializeImprovementProfileCandidate( profile: AgentProfile, - surface: ImproveSurface, + surface: ImproveProfileSurface, winner: MutableSurface, skills?: ImproveSkillsOptions, profileComponents?: ImproveProfileComponents, -): AgentProfile | undefined { +): AgentProfile { let candidate: AgentProfile if (isComponentSurface(winner)) { if (surface !== 'agent-profile' || !profileComponents) { @@ -606,8 +672,8 @@ function materializeImprovementProfileCandidate( ) } const winnerComponents = immutableCandidateValue({ ...winner.components }) - candidate = profileComponents.apply(profile, winnerComponents) - const validated = validateProfileCandidate(candidate, surface) + const applied = profileComponents.apply(profile, winnerComponents) + const validated = validateProfileCandidate(applied, surface) const materializedComponents = Object.fromEntries( validateComponents(profileComponents.read(validated), 'profileComponents.read after apply'), ) @@ -622,7 +688,9 @@ function materializeImprovementProfileCandidate( } return validated } - if (typeof winner !== 'string') return undefined + if (typeof winner !== 'string') { + throw new ConfigError(`improve(): the '${surface}' candidate cannot form an AgentProfile`) + } switch (surface) { case 'prompt': candidate = { ...profile, prompt: { ...profile.prompt, systemPrompt: winner } } @@ -675,13 +743,11 @@ function materializeImprovementProfileCandidate( candidate = applyRolloutPolicyToProfile(profile, policy) break } - case 'code': - return undefined } return validateProfileCandidate(candidate, surface) } -function validateProfileCandidate(candidate: AgentProfile, surface: ImproveSurface): AgentProfile { +function validateProfileCandidate(candidate: unknown, surface: ImproveSurface): AgentProfile { const parsed = agentProfileSchema.safeParse(candidate) if (!parsed.success) { throw new ConfigError( @@ -691,10 +757,50 @@ function validateProfileCandidate(candidate: AgentProfile, surface: ImproveSurfa return immutableCandidateValue(parsed.data) } -function inlineSkill( +function createProfileCandidateMaterializer( profile: AgentProfile, + surface: ImproveProfileSurface, + baselineSurface: MutableSurface, + skills?: ImproveSkillsOptions, + profileComponents?: ImproveProfileComponents, +): (candidateSurface: MutableSurface) => AgentProfile { + const baselineDigest = canonicalCandidateDigest(baselineSurface) + if (profileComponents) { + const reappliedBaseline = materializeImprovementProfileCandidate( + profile, + surface, + baselineSurface, + skills, + profileComponents, + ) + if (canonicalCandidateDigest(reappliedBaseline) !== canonicalCandidateDigest(profile)) { + throw new ConfigError( + 'improve(): profileComponents.apply(profile, profileComponents.read(profile)) must reproduce the complete baseline profile exactly', + ) + } + } + const candidates = new Map([[baselineDigest, profile]]) + return (candidateSurface) => { + assertCandidateSurfaceKind(surface, baselineSurface, candidateSurface) + const digest = canonicalCandidateDigest(candidateSurface) + const existing = candidates.get(digest) + if (existing) return existing + const candidate = materializeImprovementProfileCandidate( + profile, + surface, + immutableCandidateValue(candidateSurface), + skills, + profileComponents, + ) + candidates.set(digest, candidate) + return candidate + } +} + +function inlineSkill( + profile: ReadonlyAgentProfile, options: ImproveSkillsOptions | undefined, -): Extract { +): Readonly> { const resourceName = options?.resourceName.trim() if (!resourceName) { throw new ConfigError( @@ -713,7 +819,7 @@ function inlineSkill( return matches[0] } -function profileInstructions(profile: AgentProfile): string { +function profileInstructions(profile: ReadonlyAgentProfile): string { assertFailClosedResources(profile, 'memory') const instructions = profile.resources?.instructions if (instructions === undefined) return '' @@ -738,7 +844,10 @@ function replaceProfileInstructions( return { ...instructions, content } } -function assertFailClosedResources(profile: AgentProfile, surface: 'skills' | 'memory'): void { +function assertFailClosedResources( + profile: ReadonlyAgentProfile, + surface: 'skills' | 'memory', +): void { if (profile.resources?.failOnError !== true) { throw new ConfigError( `improve(): surface '${surface}' requires profile.resources.failOnError: true`, @@ -788,6 +897,14 @@ function copyProvenance( } } +function validateExecutionRef(value: unknown): Sha256Digest { + const parsed = sha256DigestSchema.safeParse(value) + if (!parsed.success) { + throw new ConfigError('improve(): executionRef must be a lowercase sha256:<64 hex> digest') + } + return parsed.data +} + function developmentSplitDigest( trainScenarios: readonly TScenario[], selectionScenarios: readonly TScenario[], @@ -805,11 +922,13 @@ async function runMethodImprovement( ): Promise { const { surface = 'prompt', + executionRef: inputExecutionRef, method: methodSource, agent, findings: inputFindings = [], skills, profileComponents, + optimizationRunOptions, minimumLift = 0, ...comparisonOptions } = opts @@ -821,59 +940,106 @@ async function runMethodImprovement( if (profileComponents && surface !== 'agent-profile') { throw new ConfigError("improve(): profileComponents is valid only with surface 'agent-profile'") } + const executionRef = validateExecutionRef(inputExecutionRef) const findings = [...inputFindings] - const baselineSurface = baselineSurfaceFor(profile, surface, skills, profileComponents) + const preparedSurface = prepareProfileSurface(profile, surface, skills, profileComponents) + const baselineSurface = preparedSurface.surface + const baselineValue = immutableCandidateValue(preparedSurface.value) + const baselineProfileDigest = canonicalCandidateDigest(profile) + const evaluationRef = canonicalCandidateDigest({ + executionRef, + baselineProfileDigest, + surface, + }) + const dispatchRef = `improve:${evaluationRef}` + const identifiedJudges = comparisonOptions.judges.map((judge) => + Object.freeze({ + ...judge, + judgeVersion: canonicalCandidateDigest({ + evaluationRef, + name: judge.name, + dimensions: judge.dimensions, + declaredJudgeVersion: judge.judgeVersion ?? null, + }), + }), + ) + const materializeProfile = createProfileCandidateMaterializer( + profile, + surface, + baselineSurface, + skills, + profileComponents, + ) const runtimeRunId = `runtime-optimization:${randomUUID()}` const splitDigest = developmentSplitDigest( comparisonOptions.trainScenarios, comparisonOptions.selectionScenarios, - comparisonOptions.optimizationRunOptions?.reps ?? 1, + optimizationRunOptions?.reps ?? 1, ) const method = resolveOptimizationMethod(methodSource, { profile, + evaluationRef, surface, baselineSurface, + baselineValue, findings, }) + const measuredMethod: OptimizationMethod = { + ...method, + async optimize(input) { + const result = await method.optimize(input) + materializeProfile(result.winnerSurface) + return result + }, + } const startedAt = Date.now() const raw = await compareOptimizationMethods({ ...comparisonOptions, - methods: [method], + judges: identifiedJudges, + dispatchRef, + optimizationRunOptions: { + ...(optimizationRunOptions ?? {}), + dispatchRef, + }, + methods: [measuredMethod], baselineSurface, - dispatchWithSurface: agent, + dispatchWithSurface: (candidateSurface, scenario, ctx) => + agent(materializeProfile(candidateSurface), scenario, ctx), }) + if ( + comparisonOptions.costCeiling !== undefined && + raw.totalCost.totalCostUsd > comparisonOptions.costCeiling + ) { + throw new ConfigError( + `improve(): reported total cost $${raw.totalCost.totalCostUsd} exceeds costCeiling $${comparisonOptions.costCeiling}`, + ) + } const score = raw.best - const winnerSurface = score.winnerSurface + const winnerSurface = immutableCandidateValue(score.winnerSurface) assertCandidateSurfaceKind(surface, baselineSurface, winnerSurface) - const candidateProfile = materializeImprovementProfileCandidate( - profile, - surface, - winnerSurface, - skills, - profileComponents, - ) - if (!candidateProfile) { - throw new ConfigError(`improve(): the '${surface}' method produced no profile candidate`) - } - const candidate = immutableCandidateValue({ + const candidateProfile = materializeProfile(winnerSurface) + const candidate: ImprovementProfileCandidate = Object.freeze({ surface, value: winnerSurface, profile: candidateProfile, }) + const cost = copyCost(raw.totalCost) return { mode: 'method', method: method.name, ...(score.provenance ? { provenance: copyProvenance(score.provenance) } : {}), candidate, - decision: score.liftCi.low > minimumLift ? 'ship' : 'hold', + decision: cost.accountingComplete && score.liftCi.low > minimumLift ? 'ship' : 'hold', lift: score.lift, liftInterval: { ...score.liftCi }, - cost: copyCost(raw.totalCost), + cost, durationMs: Date.now() - startedAt, lineage: Object.freeze({ runId: score.provenance?.runId ?? runtimeRunId, developmentSplitDigest: splitDigest, + executionRef, + baselineProfileDigest, }), raw, async dispose() {}, @@ -942,7 +1108,7 @@ async function runCodeImprovement( ) } const dispose = idempotentDispose(async () => preparedCode.cleanup()) - const candidate = immutableCandidateValue({ + const candidate = immutableCandidateValue({ surface: 'code', value: winnerSurface, }) diff --git a/src/improvement/index.ts b/src/improvement/index.ts index d7d329a1..303e501f 100644 --- a/src/improvement/index.ts +++ b/src/improvement/index.ts @@ -41,7 +41,11 @@ export { type ImproveMethodResult, type ImproveMethodSource, type ImprovementCandidate, + type ImprovementCodeCandidate, + type ImprovementProfileCandidate, + type ImproveOptimizationRunOptions, type ImproveOptions, + type ImproveProfileAgent, type ImproveProfileComponents, type ImproveProfileSurface, type ImproveResult, @@ -65,6 +69,7 @@ export { researchDriverNote, strategyAuthorMethod, } from './optimizer-prompt' +export type { DeepReadonly, ReadonlyAgentProfile } from './profile-types' export { type RawTraceDistillerOptions, rawTraceDistiller, diff --git a/src/improvement/official-optimizers.test.ts b/src/improvement/official-optimizers.test.ts index 4fb58c6e..ba1adb4d 100644 --- a/src/improvement/official-optimizers.test.ts +++ b/src/improvement/official-optimizers.test.ts @@ -1,13 +1,8 @@ import { mkdtempSync, readFileSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' -import type { - DispatchContext, - JudgeConfig, - MutableSurface, - Scenario, -} from '@tangle-network/agent-eval/contract' -import type { AgentProfile } from '@tangle-network/agent-interface' +import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract' +import { type AgentProfile, canonicalCandidateDigest } from '@tangle-network/agent-interface' import { afterEach, describe, expect, it } from 'vitest' import { improve } from './improve' import { @@ -15,6 +10,7 @@ import { officialGepa, officialSkillOpt, } from './official-optimizers' +import type { ReadonlyAgentProfile } from './profile-types' interface OptimizerScenario extends Scenario { kind: 'fixture' @@ -24,6 +20,7 @@ interface OptimizerScenario extends Scenario { interface Artifact { text: string + privateDetail?: string } const profile: AgentProfile = { @@ -41,7 +38,7 @@ const judge: JudgeConfig = { } async function agent( - surface: MutableSurface, + candidate: ReadonlyAgentProfile, _scenario: OptimizerScenario, ctx: DispatchContext, ): Promise { @@ -50,7 +47,7 @@ async function agent( actor: 'official-optimizer-test', model: 'deterministic-test', maximumCharge: { externallyEnforcedMaximumUsd: 0.0001 }, - execute: async () => ({ text: String(surface) }), + execute: async () => ({ text: candidate.prompt?.systemPrompt ?? '' }), receipt: () => ({ model: 'deterministic-test', inputTokens: 1, @@ -77,6 +74,12 @@ const testCases: OptimizerScenario[] = [ { id: 'test-a', kind: 'fixture', prompt: 'private test a', privateNote: 'TEST_SECRET_A' }, { id: 'test-b', kind: 'fixture', prompt: 'private test b', privateNote: 'TEST_SECRET_B' }, ] +const executionRef = canonicalCandidateDigest({ fixture: 'official-optimizer-method' }) +const evaluationRef = canonicalCandidateDigest({ + executionRef, + baselineProfileDigest: canonicalCandidateDigest(profile), + surface: 'prompt', +}) const runDirs: string[] = [] @@ -92,7 +95,14 @@ function runDir(): string { return value } -function fakeRunner(optimizer: 'gepa' | 'skillopt', observedInputPath: string) { +function fakeRunner( + optimizer: 'gepa' | 'skillopt', + observedInputPath: string, + callback?: { + exampleId: string + responsePath: string + }, +) { const optimizerSource = { package: optimizer, version: optimizer === 'gepa' ? 'test' : '0.2.0', @@ -104,21 +114,23 @@ function fakeRunner(optimizer: 'gepa' | 'skillopt', observedInputPath: string) { ? [ ' bestCandidate: "improved prompt",', ' bestScore: 1,', - ' totalEvaluations: 0,', + ` totalEvaluations: ${callback ? 1 : 0},`, ' recipeKind: input.recipe.kind,', + ' tokenUsage: { inputTokens: 0, outputTokens: 0, totalTokens: 0, calls: 0 },', ' proposerCostAccounting: "unavailable",', ' upstream: optimizerSource,', ] : [ ' bestCandidate: "improved prompt",', ' bestScore: 1,', - ' totalEvaluations: 0,', + ` totalEvaluations: ${callback ? 1 : 0},`, ' totalSteps: 0,', ' tokenUsage: { inputTokens: 0, outputTokens: 0, totalTokens: 0, calls: 0 },', ' upstream: optimizerSource,', ] const source = [ "const fs = require('node:fs')", + ';(async () => {', "const inputPath = process.argv[process.argv.indexOf('--input') + 1]", "const outputPath = process.argv[process.argv.indexOf('--output') + 1]", 'const input = JSON.parse(fs.readFileSync(inputPath, "utf8"))', @@ -126,18 +138,34 @@ function fakeRunner(optimizer: 'gepa' | 'skillopt', observedInputPath: string) { 'if (input.operation === "inspect") {', ' fs.writeFileSync(outputPath, JSON.stringify({ runtime: {', ' python: { implementation: "CPython", version: "3.12.0" },', - ' bridge: { package: "agent-eval-rpc", version: "0.126.0", sourceSha256: "a".repeat(64) },', + ' bridge: { package: "agent-eval-rpc", version: "0.126.1", sourceSha256: "a".repeat(64) },', ' optimizer: optimizerSource,', ' engineModules: [],', ' } }))', ' process.exit(0)', '}', `fs.writeFileSync(${JSON.stringify(observedInputPath)}, JSON.stringify(input))`, + ...(callback + ? [ + 'const callbackResponse = await fetch(input.callbackUrl, {', + ' method: "POST",', + ' headers: {', + ' authorization: "Bearer " + input.callbackToken,', + ' "content-type": "application/json",', + ' },', + ` body: JSON.stringify({ candidate: "improved prompt", exampleId: ${JSON.stringify(callback.exampleId)} }),`, + '})', + 'if (!callbackResponse.ok) throw new Error("callback failed: " + callbackResponse.status)', + 'const callbackBody = await callbackResponse.json()', + `fs.writeFileSync(${JSON.stringify(callback.responsePath)}, JSON.stringify(callbackBody))`, + ] + : []), 'fs.writeFileSync(outputPath, JSON.stringify({', ...output, ' runId: input.runId,', ' resumed: false,', '}))', + '})().catch((error) => { process.stderr.write(String(error?.stack ?? error)); process.exit(1) })', ].join('\n') return { command: process.execPath, @@ -172,6 +200,7 @@ const testOptimizer = { function commonOptions(method: ReturnType>) { return { surface: 'prompt' as const, + executionRef, method, findings: [{ claim: 'answers omit citations' }], trainScenarios: train, @@ -193,7 +222,6 @@ describe('official optimizer methods', () => { ...commonOptions( officialGepa({ objective: 'Improve the agent prompt.', - evaluationId: 'prompt-eval', recipe: { kind: 'engine', run: { @@ -202,6 +230,7 @@ describe('official optimizer methods', () => { maxProposerCostUsd: 1, }, }, + optimizer: testOptimizer, resume: 'if-compatible', trustResumeState: true, describeScenario: (scenario: OptimizerScenario) => ({ prompt: scenario.prompt }), @@ -213,7 +242,7 @@ describe('official optimizer methods', () => { const observed = JSON.parse(readFileSync(observedInputPath, 'utf8')) as Record expect(observed).toMatchObject({ - evaluationId: 'prompt-eval', + evaluationId: evaluationRef, resume: 'if-compatible', trustedResumeState: true, recipe: { @@ -263,7 +292,6 @@ describe('official optimizer methods', () => { ...commonOptions( officialGepa({ objective: 'Improve the agent prompt.', - evaluationId: 'prompt-eval', recipe, describeScenario: (scenario: OptimizerScenario) => ({ prompt: scenario.prompt }), runner: fakeRunner('gepa', observedInputPath), @@ -286,7 +314,6 @@ describe('official optimizer methods', () => { ...commonOptions( officialSkillOpt({ objective: 'Improve the agent prompt.', - evaluationId: 'prompt-eval', trainer: { epochs: 1, batchSize: 1, @@ -303,7 +330,7 @@ describe('official optimizer methods', () => { const observed = JSON.parse(readFileSync(observedInputPath, 'utf8')) as Record expect(observed).toMatchObject({ - evaluationId: 'prompt-eval', + evaluationId: evaluationRef, resume: 'if-compatible', trainer: { epochs: 1, @@ -335,7 +362,6 @@ describe('official optimizer methods', () => { () => officialGepa({ objective: 'Improve the agent prompt.', - evaluationId: 'prompt-eval', recipe: { kind: 'engine', run: { engine: 'gepa', maxEvaluations: 1, maxProposerCostUsd: 1 }, @@ -349,7 +375,6 @@ describe('official optimizer methods', () => { () => officialSkillOpt({ objective: 'Improve the agent prompt.', - evaluationId: 'prompt-eval', trainer: { epochs: 1, batchSize: 1, @@ -380,7 +405,6 @@ describe('official optimizer methods', () => { it('does not relabel an unrelated optimizer process failure as a missing dependency', async () => { const method = officialGepa({ objective: 'Improve the agent prompt.', - evaluationId: 'prompt-eval', recipe: { kind: 'engine', run: { engine: 'gepa', maxEvaluations: 1, maxProposerCostUsd: 1 }, @@ -400,4 +424,101 @@ describe('official optimizer methods', () => { expect(failure).not.toBeInstanceOf(OfficialOptimizerUnavailableError) expect((failure as Error).message).toContain('application-owned callback failed') }) + + it('redacts common private values from findings before starting the optimizer', async () => { + const root = runDir() + const observedInputPath = join(root, 'observed-redacted-findings.json') + const privateValue = 'sk-private-finding-value-123456789' + const result = await improve(profile, { + ...commonOptions( + officialGepa({ + objective: 'Improve the agent prompt.', + recipe: { + kind: 'engine', + run: { engine: 'gepa', maxEvaluations: 1, maxProposerCostUsd: 1 }, + }, + optimizer: testOptimizer, + runner: fakeRunner('gepa', observedInputPath), + }), + ), + findings: [{ claim: 'answers omit citations', apiKey: privateValue }], + runDir: join(root, 'run'), + }) + + const observed = readFileSync(observedInputPath, 'utf8') + expect(observed).not.toContain(privateValue) + expect(observed).toContain('[redacted]') + expect(result.decision).toBe('ship') + }) + + it('redacts artifact details and judge notes at the optimizer callback boundary', async () => { + const root = runDir() + const observedInputPath = join(root, 'observed-redacted-evidence-input.json') + const observedResponsePath = join(root, 'observed-redacted-evidence-response.json') + const privateArtifact = 'sk-private-artifact-value-123456789' + const privateNote = 'sk-private-judge-note-123456789' + const evidenceJudge: JudgeConfig = { + ...judge, + score: ({ artifact }) => ({ + dimensions: { quality: artifact.text === 'improved prompt' ? 1 : 0 }, + composite: artifact.text === 'improved prompt' ? 1 : 0, + notes: `Internal judge detail: ${privateNote}`, + }), + } + const result = await improve(profile, { + ...commonOptions( + officialGepa({ + objective: 'Improve the agent prompt.', + recipe: { + kind: 'engine', + run: { engine: 'gepa', maxEvaluations: 1, maxProposerCostUsd: 1 }, + }, + optimizer: testOptimizer, + describeArtifact: (artifact) => artifact, + runner: fakeRunner('gepa', observedInputPath, { + exampleId: 'train', + responsePath: observedResponsePath, + }), + }), + ), + judges: [evidenceJudge], + agent: async (candidate, scenario, ctx) => ({ + ...(await agent(candidate, scenario, ctx)), + privateDetail: privateArtifact, + }), + runDir: join(root, 'run'), + }) + + const observed = readFileSync(observedResponsePath, 'utf8') + expect(observed).not.toContain(privateArtifact) + expect(observed).not.toContain(privateNote) + expect(observed).toContain('[redacted]') + expect(result.provenance?.evaluationCount).toBe(1) + expect(result.decision).toBe('ship') + }) + + it('rejects a private profile surface before starting an external optimizer', async () => { + const privateValue = 'sk-private-profile-value-123456789' + await expect( + improve( + { + name: 'optimizer-fixture', + prompt: { systemPrompt: `Use ${privateValue}` }, + }, + { + ...commonOptions( + officialGepa({ + objective: 'Improve the agent prompt.', + recipe: { + kind: 'engine', + run: { engine: 'gepa', maxEvaluations: 1, maxProposerCostUsd: 1 }, + }, + optimizer: testOptimizer, + runner: failingRunner('runner must not start'), + }), + ), + }, + ), + ).rejects.toThrow(/selected profile surface contains a common credential or private value/) + }) }) diff --git a/src/improvement/official-optimizers.ts b/src/improvement/official-optimizers.ts index 0794e5f8..ca92e9e6 100644 --- a/src/improvement/official-optimizers.ts +++ b/src/improvement/official-optimizers.ts @@ -1,20 +1,24 @@ +import { isDeepStrictEqual } from 'node:util' import { canonicalJson } from '@tangle-network/agent-eval' import { type GepaOptimizationMethodConfig, gepaOptimizationMethod, + type JudgeScore, type OptimizationMethod, type SkillOptOptimizationMethodConfig, skillOptOptimizationMethod, } from '@tangle-network/agent-eval/campaign' +import { canonicalCandidateDigest } from '../candidate-execution/digest' import { ConfigError } from '../errors' +import { defaultRedactor } from '../redact' import type { ImproveMethodContext, ImproveMethodFactory } from './improve' -const DEFAULT_MAX_FINDINGS_CHARS = 50_000 -const PYTHON_CLIENT_DOCS = 'https://github.com/tangle-network/agent-eval/tree/main/clients/python' -const GEPA_INSTALL = +const defaultMaxFindingsChars = 50_000 +const pythonClientDocs = 'https://github.com/tangle-network/agent-eval/tree/main/clients/python' +const gepaInstall = '`python -m pip install agent-eval-rpc`, then ' + '`python -m pip install "gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f"`' -const SKILLOPT_INSTALL = +const skillOptInstall = '`python -m pip install agent-eval-rpc`, then ' + '`python -m pip install "skillopt @ git+https://github.com/microsoft/SkillOpt.git@61735e3922efc2b90c6d6cab561e62e98452ca90"`' @@ -32,14 +36,14 @@ export interface OfficialOptimizerContextOptions { export type OfficialGepaOptions< TScenario extends { id: string; kind: string }, TArtifact = unknown, -> = Omit, 'background'> & +> = Omit, 'background' | 'evaluationId'> & OfficialOptimizerContextOptions /** Official SkillOpt configuration plus bounded Runtime findings context. */ export type OfficialSkillOptOptions< TScenario extends { id: string; kind: string }, TArtifact = unknown, -> = Omit, 'background'> & +> = Omit, 'background' | 'evaluationId'> & OfficialOptimizerContextOptions /** Missing optional Python dependencies for an official optimizer. */ @@ -50,14 +54,14 @@ export class OfficialOptimizerUnavailableError extends ConfigError { const detail = cause instanceof Error ? cause.message : String(cause) const install = optimizer === 'gepa' - ? `Install the Python bridge and pinned GEPA source: ${GEPA_INSTALL}.` - : `Install Microsoft SkillOpt: ${SKILLOPT_INSTALL}.` + ? `Install the Python bridge and pinned GEPA source: ${gepaInstall}.` + : `Install Microsoft SkillOpt: ${skillOptInstall}.` super( [ `Official ${optimizer === 'gepa' ? 'GEPA' : 'SkillOpt'} could not start.`, 'Runtime did not use a local fallback.', install, - `Setup: ${PYTHON_CLIENT_DOCS}.`, + `Setup: ${pythonClientDocs}.`, `Cause: ${detail}`, ].join(' '), { cause }, @@ -75,13 +79,24 @@ export class OfficialOptimizerUnavailableError extends ConfigError { export function officialGepa( options: OfficialGepaOptions, ): ImproveMethodFactory { - const { background, includeFindings = true, maxFindingsChars, ...config } = options + const { + background, + includeFindings = true, + maxFindingsChars, + describeScenario, + describeArtifact, + ...config + } = options assertMaxFindingsChars('officialGepa', maxFindingsChars) - return (context) => - withDependencyHelp( + assertSafeCallerText('officialGepa', 'objective', config.objective) + return (context) => { + assertSafeOptimizerSurface('officialGepa', context) + return withDependencyHelp( 'gepa', + context.evaluationRef, gepaOptimizationMethod({ ...config, + evaluationId: context.evaluationRef, background: methodBackground({ context, background, @@ -89,8 +104,15 @@ export function officialGepa + redactOptimizerEvidence(describeScenario ? describeScenario(scenario) : scenario), + describeArtifact: (artifact, scenario) => + redactOptimizerEvidence( + describeArtifact ? describeArtifact(artifact, scenario) : artifact, + ), }), ) + } } /** Build a complete method backed by Microsoft's official SkillOpt trainer. */ @@ -100,13 +122,24 @@ export function officialSkillOpt< >( options: OfficialSkillOptOptions, ): ImproveMethodFactory { - const { background, includeFindings = true, maxFindingsChars, ...config } = options + const { + background, + includeFindings = true, + maxFindingsChars, + describeScenario, + describeArtifact, + ...config + } = options assertMaxFindingsChars('officialSkillOpt', maxFindingsChars) - return (context) => - withDependencyHelp( + assertSafeCallerText('officialSkillOpt', 'objective', config.objective) + return (context) => { + assertSafeOptimizerSurface('officialSkillOpt', context) + return withDependencyHelp( 'skillopt', + context.evaluationRef, skillOptOptimizationMethod({ ...config, + evaluationId: context.evaluationRef, background: methodBackground({ context, background, @@ -114,8 +147,15 @@ export function officialSkillOpt< maxFindingsChars, label: 'officialSkillOpt', }), + describeScenario: (scenario) => + redactOptimizerEvidence(describeScenario ? describeScenario(scenario) : scenario), + describeArtifact: (artifact, scenario) => + redactOptimizerEvidence( + describeArtifact ? describeArtifact(artifact, scenario) : artifact, + ), }), ) + } } function assertMaxFindingsChars(label: string, value: number | undefined): void { @@ -135,9 +175,11 @@ function methodBackground(options: { context, background, includeFindings, - maxFindingsChars = DEFAULT_MAX_FINDINGS_CHARS, + maxFindingsChars = defaultMaxFindingsChars, label, } = options + assertSafeCallerText(label, 'background', background) + assertSafeCallerText(label, 'profile name', context.profile.name) const sections = [ background?.trim(), `Agent profile: ${context.profile.name}. Surface: ${context.surface}.`, @@ -145,7 +187,7 @@ function methodBackground(options: { if (includeFindings && context.findings.length > 0) { let serialized: string try { - serialized = canonicalJson(context.findings) + serialized = canonicalJson(defaultRedactor(context.findings)) } catch (cause) { throw new ConfigError(`${label}: findings must be JSON-serializable`, { cause }) } @@ -159,15 +201,68 @@ function methodBackground(options: { return sections.join('\n\n') } +function assertSafeOptimizerSurface(label: string, context: ImproveMethodContext): void { + const redactedValue = defaultRedactor(context.baselineValue) + const redactedSurface = defaultRedactor(context.baselineSurface) + if ( + !isDeepStrictEqual(context.baselineValue, redactedValue) || + !isDeepStrictEqual(context.baselineSurface, redactedSurface) + ) { + throw new ConfigError( + `${label}: the selected profile surface contains a common credential or private value. ` + + 'Store live credentials as provider references, or remove private data before starting an external optimizer.', + ) + } +} + +function redactOptimizerEvidence(value: unknown): unknown { + return defaultRedactor(value) +} + +function redactJudgeScore(score: JudgeScore): JudgeScore { + const notes = defaultRedactor(score.notes) + return { + ...score, + notes: typeof notes === 'string' ? notes : '[redacted]', + } +} + +function assertSafeCallerText(label: string, field: string, value: string | undefined): void { + if (value !== undefined && defaultRedactor(value) !== value) { + throw new ConfigError( + `${label}: ${field} contains a common credential or private value. Sanitize it before starting an external optimizer.`, + ) + } +} + function withDependencyHelp( optimizer: 'gepa' | 'skillopt', + evaluationRef: ImproveMethodContext['evaluationRef'], method: OptimizationMethod, ): OptimizationMethod { return { ...method, async optimize(input) { try { - return await method.optimize(input) + const judges = input.judges.map((judge) => + Object.freeze({ + ...judge, + judgeVersion: canonicalCandidateDigest({ + evaluationRef, + name: judge.name, + dimensions: judge.dimensions, + judgeVersion: judge.judgeVersion ?? null, + outwardEvidence: 'default-redactor', + }), + async score(scoreInput: Parameters[0]) { + return redactJudgeScore(await judge.score(scoreInput)) + }, + }), + ) + return await method.optimize({ + ...input, + judges: Object.freeze(judges), + }) } catch (cause) { if (isMissingDependency(optimizer, cause)) { throw new OfficialOptimizerUnavailableError(optimizer, cause) diff --git a/src/improvement/profile-types.ts b/src/improvement/profile-types.ts new file mode 100644 index 00000000..545a56b6 --- /dev/null +++ b/src/improvement/profile-types.ts @@ -0,0 +1,12 @@ +import type { AgentProfile } from '@tangle-network/agent-interface' + +export type DeepReadonly = T extends (...args: never[]) => unknown + ? T + : T extends readonly (infer TItem)[] + ? readonly DeepReadonly[] + : T extends object + ? { readonly [TKey in keyof T]: DeepReadonly } + : T + +/** Complete immutable profile value used during measured execution. */ +export type ReadonlyAgentProfile = DeepReadonly diff --git a/src/improvement/rollout-policy.test.ts b/src/improvement/rollout-policy.test.ts index e1d8ff8c..41248609 100644 --- a/src/improvement/rollout-policy.test.ts +++ b/src/improvement/rollout-policy.test.ts @@ -3,7 +3,7 @@ import { type OptimizationMethod, } from '@tangle-network/agent-eval/campaign' import type { JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract' -import type { AgentProfile } from '@tangle-network/agent-interface' +import { type AgentProfile, canonicalCandidateDigest } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import type { StructuralRolloutPolicy } from '../runtime/structural-rollout' import { improve } from './improve' @@ -69,6 +69,7 @@ const testCases: PolicyScenario[] = [ { id: 'test-a', kind: 'fixture' }, { id: 'test-b', kind: 'fixture' }, ] +const executionRef = canonicalCandidateDigest({ fixture: 'rollout-policy-method' }) const judge: JudgeConfig = { name: 'k', @@ -103,12 +104,13 @@ describe("improve surface 'rollout-policy'", () => { } const result = await improve(profile, { surface: 'rollout-policy', + executionRef, method: policyMethod({ k: 7, repairRounds: 2, testgen: 6 }), trainScenarios: train, selectionScenarios: selection, testScenarios: testCases, judges: [judge], - agent: async (surface) => ({ policy: parseRolloutPolicy(surface) }), + agent: async (candidate) => ({ policy: structuralRolloutPolicyFromProfile(candidate) }), runDir: 'mem://rollout-policy-method', storage: inMemoryCampaignStorage(), resamples: 40, @@ -136,12 +138,13 @@ describe("improve surface 'rollout-policy'", () => { { name: 'fixture-agent' }, { surface: 'rollout-policy', + executionRef, method: policyMethod({ k: 7, repairRounds: 2, testgen: 6 }), trainScenarios: train, selectionScenarios: selection, testScenarios: testCases, judges: [judge], - agent: async (surface) => ({ policy: parseRolloutPolicy(surface) }), + agent: async (candidate) => ({ policy: structuralRolloutPolicyFromProfile(candidate) }), runDir: 'mem://rollout-policy-missing', storage: inMemoryCampaignStorage(), resamples: 40, diff --git a/src/improvement/rollout-policy.ts b/src/improvement/rollout-policy.ts index 9a11cac9..b598f982 100644 --- a/src/improvement/rollout-policy.ts +++ b/src/improvement/rollout-policy.ts @@ -6,6 +6,7 @@ import { defaultStructuralRolloutPolicy, type StructuralRolloutPolicy, } from '../runtime/structural-rollout' +import type { ReadonlyAgentProfile } from './profile-types' /** The profile extensions namespace the policy persists under. */ export const ROLLOUT_POLICY_EXTENSION = 'structural-rollout' @@ -64,7 +65,7 @@ export function serializeRolloutPolicy(policy: StructuralRolloutPolicy): string /** Read the persisted policy off the profile. `undefined` when the profile does * not opt into structural rollout. */ export function structuralRolloutPolicyFromProfile( - profile: AgentProfile, + profile: ReadonlyAgentProfile, ): StructuralRolloutPolicy | undefined { const bag = profile.extensions?.[ROLLOUT_POLICY_EXTENSION] if (bag === undefined) return undefined @@ -73,9 +74,10 @@ export function structuralRolloutPolicyFromProfile( /** Persist a detached policy under the profile extension without mutating the input. */ export function applyRolloutPolicyToProfile( - profile: AgentProfile, + profile: ReadonlyAgentProfile, policy: StructuralRolloutPolicy, ): AgentProfile { + const candidate = structuredClone(profile) as AgentProfile const bag: Record = { k: policy.k, repairRounds: policy.repairRounds, @@ -84,7 +86,7 @@ export function applyRolloutPolicyToProfile( ...(policy.temperature !== undefined ? { temperature: policy.temperature } : {}), } return { - ...profile, - extensions: { ...profile.extensions, [ROLLOUT_POLICY_EXTENSION]: bag }, + ...candidate, + extensions: { ...candidate.extensions, [ROLLOUT_POLICY_EXTENSION]: bag }, } } diff --git a/src/intelligence/improvement-cycle.ts b/src/intelligence/improvement-cycle.ts index 93bada42..2d4df1fb 100644 --- a/src/intelligence/improvement-cycle.ts +++ b/src/intelligence/improvement-cycle.ts @@ -384,9 +384,6 @@ function assertImprovementCandidateBinding { cyclic.self = cyclic expect(() => defaultRedactor(cyclic)).not.toThrow() }) + + it('redacts secret assignments embedded in serialized text', () => { + const redacted = defaultRedactor('judge note: {"password":"hunter2"} token=plain-secret') + expect(redacted).toBe('judge note: {"password":[redacted]} token=[redacted]') + }) }) describe('createIntelligenceClient / traceRun — Observe', () => { diff --git a/src/intelligence/redact.ts b/src/redact.ts similarity index 81% rename from src/intelligence/redact.ts rename to src/redact.ts index 44220b52..7cfb31bc 100644 --- a/src/intelligence/redact.ts +++ b/src/redact.ts @@ -1,10 +1,9 @@ /** * - * Redaction for Intelligence trace export. The trace carries the customer's - * real input/output; before any of it leaves the process it passes through a - * `Redactor`. The default scrubs the obvious leak classes (API keys, bearer - * tokens, emails, private keys) from strings and walks nested objects/arrays, - * but a customer with domain-specific PII supplies their own `redact` hook. + * Redaction for values that may leave the Runtime process. The default scrubs + * common leak classes (API keys, bearer tokens, emails, private keys) from + * strings and walks nested objects and arrays. A customer with domain-specific + * PII supplies their own `redact` hook. * * This is intentionally narrower than `src/sanitize.ts` (which redacts the * runtime's *event envelope* field-by-field): here the value is opaque @@ -39,9 +38,14 @@ const valuePatterns: ReadonlyArray = [ /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g, ] +/** Secret assignments embedded inside an otherwise opaque string, including + * serialized JSON and log-style `key=value` text. */ +const secretAssignmentPattern = + /((?:"|')?(?:api[-_]?key|secret|token|password|passwd|authorization|auth|private[-_]?key|credential|access[-_]?key|client[-_]?secret|session[-_]?(?:id|token)|cookie|bearer)(?:"|')?\s*[:=]\s*)(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|[^\s,;}]+)/gi + /** Scrub a single string of in-value secret/PII patterns. */ function scrubString(input: string): string { - let out = input + let out = input.replace(secretAssignmentPattern, `$1${redactedMarker}`) for (const pattern of valuePatterns) { out = out.replace(pattern, redactedMarker) } diff --git a/tests/profile-improvement-stack.test.ts b/tests/profile-improvement-stack.test.ts index c2f0e7e8..8a807c82 100644 --- a/tests/profile-improvement-stack.test.ts +++ b/tests/profile-improvement-stack.test.ts @@ -4,17 +4,13 @@ import { type OptimizationMethod, skillOptOptimizationMethod, } from '@tangle-network/agent-eval/campaign' -import type { - DispatchContext, - JudgeConfig, - MutableSurface, - Scenario, -} from '@tangle-network/agent-eval/contract' -import type { AgentProfile } from '@tangle-network/agent-interface' +import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract' +import { type AgentProfile, canonicalCandidateDigest } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../src/durable/spawn-journal' import * as runtimeImprovement from '../src/improvement' import { improve, officialGepa, officialSkillOpt } from '../src/improvement' +import type { ReadonlyAgentProfile } from '../src/improvement/profile-types' import { loopUntil } from '../src/runtime/personify/combinators' import { definePersona, runPersonified } from '../src/runtime/personify/persona' import type { Outcome } from '../src/runtime/personify/wave-types' @@ -39,6 +35,7 @@ const scenarios: ProfileScenario[] = Array.from({ length: 12 }, (_, i) => ({ const trainScenarios = scenarios.slice(0, 4) const selectionScenarios = scenarios.slice(4, 8) const testScenarios = scenarios.slice(8) +const executionRef = canonicalCandidateDigest({ fixture: 'profile-improvement-stack' }) const baseProfile = (): AgentProfile => ({ name: 'incident-responder', @@ -66,7 +63,7 @@ const promptJudge: JudgeConfig<{ prompt: string }, ProfileScenario> = { } async function promptAgent( - surface: MutableSurface, + profile: ReadonlyAgentProfile, _scenario: ProfileScenario, ctx: DispatchContext, ): Promise<{ prompt: string }> { @@ -75,7 +72,7 @@ async function promptAgent( actor: 'profile-stack-test', model: 'deterministic-test', maximumCharge: { externallyEnforcedMaximumUsd: 0.0001 }, - execute: async () => ({ prompt: String(surface) }), + execute: async () => ({ prompt: profile.prompt?.systemPrompt ?? '' }), receipt: () => ({ model: 'deterministic-test', inputTokens: 1, @@ -181,6 +178,7 @@ describe('profile improvement stack', () => { it('runs a detached profile candidate through loopUntil()', async () => { const improved = await improve(baseProfile(), { surface: 'prompt', + executionRef, method: improvingMethod, findings: [{ failure: 'single draft gets stuck' }], trainScenarios, From 6d460a99070061452e1a7df3705a0e62fe17cf65 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 24 Jul 2026 11:22:15 -0600 Subject: [PATCH 08/14] refactor(improvement): split optimizer execution modules --- src/improvement/code-execution.ts | 292 +++++++ src/improvement/improve-result.ts | 13 + src/improvement/improve-types.ts | 270 +++++++ src/improvement/improve.ts | 1156 +-------------------------- src/improvement/method-execution.ts | 207 +++++ src/improvement/profile-surface.ts | 383 +++++++++ 6 files changed, 1202 insertions(+), 1119 deletions(-) create mode 100644 src/improvement/code-execution.ts create mode 100644 src/improvement/improve-result.ts create mode 100644 src/improvement/improve-types.ts create mode 100644 src/improvement/method-execution.ts create mode 100644 src/improvement/profile-surface.ts diff --git a/src/improvement/code-execution.ts b/src/improvement/code-execution.ts new file mode 100644 index 00000000..cc3bc0e6 --- /dev/null +++ b/src/improvement/code-execution.ts @@ -0,0 +1,292 @@ +import { makeFinding } from '@tangle-network/agent-eval' +import { + gitWorktreeAdapter, + type Worktree, + type WorktreeAdapter, +} from '@tangle-network/agent-eval/campaign' +import { + type CodeSurface, + type MutableSurface, + type Scenario, + type SelfImproveBudget, + type SelfImproveOptions, + type SelfImproveResult, + type SurfaceProposer, + selfImprove, +} from '@tangle-network/agent-eval/contract' +import { immutableCandidateValue } from '../candidate-execution/digest' +import { agenticGenerator } from './agentic-generator' +import { rethrowAfterCleanup } from './cleanup' +import { copyImproveCost } from './improve-result' +import type { + ImproveCodeOptions, + ImproveCodeResult, + ImproveCodeRunOptions, + ImprovementCodeCandidate, +} from './improve-types' +import { improvementDriver, type ManagedImprovementDriver } from './improvement-driver' +import { assertCandidateSurfaceKind, isCodeSurface } from './profile-surface' +import { rawTraceDistiller } from './raw-trace-distiller' + +/** Slice bound for distilled judge notes: wide enough that a real traceback or + * failing assertion survives intact. */ +const distilledNotesMaxChars = 1500 + +/** Slice bound for a cell's error string. Full text remains on the raw cell. */ +const distilledErrorMaxChars = 500 + +/** Distill failing cells into typed findings for the next proposal round. */ +function generationFailureDistiller( + staticFindings: unknown[], +): NonNullable['analyzeGeneration']> { + const CAP = 12 + return async (input) => { + const failures: Array<{ + scenario: string + composite: number + notes: string + claim?: string + error?: string + }> = [] + for (const candidate of input.candidates) { + for (const rawCell of candidate.campaign.cells) { + const cell = rawCell as unknown as Record + const scenario = String(cell.scenarioId ?? 'unknown') + const error = typeof cell.error === 'string' ? cell.error : undefined + const judgeScores = + cell.judgeScores && typeof cell.judgeScores === 'object' + ? Object.values( + cell.judgeScores as Record, + ) + : [] + const composite = + judgeScores.length === 0 + ? 0 + : judgeScores.reduce((sum, judge) => sum + (judge.composite ?? 0), 0) / + judgeScores.length + if (!error && composite >= 0.999) continue + const notes = judgeScores + .map((judge) => judge.notes) + .filter((note): note is string => typeof note === 'string' && note.length > 0) + .join('; ') + .slice(0, distilledNotesMaxChars) + const claim = + notes || + (error ? `Scenario ${scenario} failed: ${error.slice(0, distilledErrorMaxChars)}` : '') + failures.push({ + scenario, + composite: Number(composite.toFixed(3)), + notes, + ...(claim ? { claim } : {}), + ...(error ? { error: error.slice(0, distilledErrorMaxChars) } : {}), + }) + } + } + if (failures.length === 0) return staticFindings + failures.sort((left, right) => left.composite - right.composite) + return failures.slice(0, CAP).map((failure) => + makeFinding({ + analyst_id: 'generation-failure-distiller', + severity: failure.error !== undefined || failure.composite < 0.5 ? 'high' : 'medium', + area: 'generation-failure', + confidence: 1, + subject: failure.scenario, + claim: `Scenario ${failure.scenario} scored composite ${failure.composite}${ + failure.notes ? `: ${failure.notes}` : '' + }${failure.error ? ` (error: ${failure.error})` : ''}`, + evidence_refs: [], + metadata: { + scenario: failure.scenario, + composite: failure.composite, + ...(failure.notes ? { notes: failure.notes } : {}), + ...(failure.error !== undefined ? { error: failure.error } : {}), + }, + }), + ) + } +} + +/** Default code-run analysis: raw trace paths for durable runs, otherwise a + * bounded digest of failed cells. */ +function defaultDistillerFor( + opts: ImproveCodeRunOptions, + findings: unknown[], +): NonNullable['analyzeGeneration']> { + const durableRun = opts.runDir !== undefined && !opts.runDir.startsWith('mem://') + const useRawTraces = opts.rawTraceContext ?? durableRun + if (useRawTraces) return rawTraceDistiller({ fallbackFindings: findings }) + return generationFailureDistiller(findings) +} + +interface PreparedCodeRun { + baseline: CodeSurface + proposer: SurfaceProposer + cleanup(retainedWinner?: MutableSurface): Promise +} + +async function discardPreparedBaseline( + worktree: WorktreeAdapter, + baselineWorktree: Worktree, + cause: unknown, +): Promise { + return rethrowAfterCleanup( + cause, + () => worktree.discard(baselineWorktree), + 'improve(): code preparation failed', + ) +} + +/** Create a clean incumbent checkout and the candidate producer for a code run. */ +async function prepareCodeRun(code: ImproveCodeOptions): Promise { + const baseRef = code.baseRef ?? 'main' + const worktree = + code.worktree ?? + gitWorktreeAdapter({ + repoRoot: code.repoRoot, + ...(code.worktreeDir ? { worktreeDir: code.worktreeDir } : {}), + }) + const baselineWorktree = await worktree.create({ baseRef, label: 'incumbent-baseline' }) + try { + const baseline = await worktree.finalize(baselineWorktree, 'Incumbent code checkout') + let baselineDiscarded = false + const generator = + code.generator ?? + agenticGenerator({ + ...(code.harness ? { harness: code.harness } : {}), + ...(code.verify ? { verify: code.verify } : {}), + ...(code.timeoutMs ? { timeoutMs: code.timeoutMs } : {}), + }) + const managed: ManagedImprovementDriver = improvementDriver({ worktree, generator, baseRef }) + + return { + baseline, + proposer: managed, + async cleanup(retainedWinner) { + const errors: unknown[] = [] + const retainedWorktreeRef = isCodeSurface(retainedWinner) + ? retainedWinner.worktreeRef + : undefined + try { + await managed?.cleanup(retainedWorktreeRef ? [retainedWorktreeRef] : []) + } catch (cause) { + errors.push(cause) + } + if (!baselineDiscarded && retainedWorktreeRef !== baseline.worktreeRef) { + try { + await worktree.discard(baselineWorktree) + baselineDiscarded = true + } catch (cause) { + errors.push(cause) + } + } + if (errors.length > 0) { + throw new AggregateError(errors, 'improve(): failed to clean code improvement worktrees') + } + }, + } + } catch (cause) { + return discardPreparedBaseline(worktree, baselineWorktree, cause) + } +} + +function idempotentDispose(dispose: () => Promise): () => Promise { + let disposed = false + let inFlight: Promise | undefined + return async () => { + if (disposed) return + if (inFlight) return inFlight + inFlight = (async () => { + await dispose() + disposed = true + })() + try { + await inFlight + } finally { + inFlight = undefined + } + } +} + +export async function runCodeImprovement( + opts: ImproveCodeRunOptions, +): Promise> { + const { + gate = 'holdout', + findings: inputFindings = [], + rawTraceContext: _rawTraceContext, + code, + promotionGate, + analyzeGeneration, + surface: _surface, + ...sharedOptions + } = opts + const findings = [...inputFindings] + const preparedCode = await prepareCodeRun(code) + + const budget: SelfImproveBudget = + gate === 'none' ? { ...sharedOptions.budget, generations: 0 } : { ...sharedOptions.budget } + + let raw: SelfImproveResult + try { + raw = await selfImprove({ + ...sharedOptions, + baselineSurface: preparedCode.baseline, + proposer: preparedCode.proposer, + budget, + findings, + ...(promotionGate !== undefined ? { gate: promotionGate } : {}), + ...(analyzeGeneration === null + ? {} + : { + analyzeGeneration: + analyzeGeneration ?? defaultDistillerFor(opts, findings), + }), + }) + } catch (cause) { + return rethrowAfterCleanup( + cause, + () => preparedCode.cleanup(), + 'improve(): code improvement failed', + ) + } + + const winnerSurface = raw.winner.surface + assertCandidateSurfaceKind('code', preparedCode.baseline, winnerSurface) + try { + await preparedCode.cleanup(winnerSurface) + } catch (cleanupCause) { + try { + await preparedCode.cleanup() + } catch (finalCleanupCause) { + throw new AggregateError( + [cleanupCause, finalCleanupCause], + 'improve(): code result cleanup failed, including the final all-worktree retry', + ) + } + throw new AggregateError( + [cleanupCause], + 'improve(): code result cleanup failed; the final all-worktree retry succeeded', + ) + } + const dispose = idempotentDispose(async () => preparedCode.cleanup()) + const candidate = immutableCandidateValue({ + surface: 'code', + value: winnerSurface, + }) + + return { + mode: 'code', + candidate, + decision: raw.gateDecision, + ...(raw.lift !== undefined ? { lift: raw.lift } : {}), + cost: copyImproveCost(raw.cost), + durationMs: raw.durationMs, + lineage: Object.freeze({ + runId: raw.provenance.runId, + developmentSplitDigest: raw.provenance.evidence.search.splitDigest, + }), + generationsExplored: raw.generationsExplored, + raw, + dispose, + } +} diff --git a/src/improvement/improve-result.ts b/src/improvement/improve-result.ts new file mode 100644 index 00000000..b996dbfb --- /dev/null +++ b/src/improvement/improve-result.ts @@ -0,0 +1,13 @@ +import type { ImproveCost } from './improve-types' + +export function copyImproveCost(cost: { + totalCostUsd: number + accountingComplete: boolean + incompleteReasons: readonly string[] +}): ImproveCost { + return { + totalCostUsd: cost.totalCostUsd, + accountingComplete: cost.accountingComplete, + incompleteReasons: [...cost.incompleteReasons], + } +} diff --git a/src/improvement/improve-types.ts b/src/improvement/improve-types.ts new file mode 100644 index 00000000..8d09291e --- /dev/null +++ b/src/improvement/improve-types.ts @@ -0,0 +1,270 @@ +import type { + CompareOptimizationMethodsOptions, + OptimizationMethod, + OptimizationMethodComparison, + WorktreeAdapter, +} from '@tangle-network/agent-eval/campaign' +import type { + MutableSurface, + Scenario, + SelfImproveBudget, + SelfImproveOptions, + SelfImproveResult, +} from '@tangle-network/agent-eval/contract' +import type { Sha256Digest } from '@tangle-network/agent-interface' +import type { LocalHarness } from '../mcp/local-harness' +import type { Verifier } from './agentic-generator' +import type { CandidateGenerator } from './improvement-driver' +import type { ReadonlyAgentProfile } from './profile-types' + +/** The executable agent lever `improve` optimizes. Profile fields remain + * portable AgentProfile coordinates; implementation and orchestration files + * use the code surface so a winner can be sealed into an exact candidate. + * `rollout-policy` is the inference-time structuralRollout dials + * (`profile.extensions['structural-rollout']`). */ +export type ImproveSurface = + | 'prompt' + | 'skills' + | 'tools' + | 'mcp' + | 'hooks' + | 'subagents' + | 'agent-profile' + | 'memory' + | 'code' + | 'rollout-policy' + +export type ImproveProfileSurface = Exclude + +export interface ImproveMethodContext { + /** Validated baseline profile. */ + readonly profile: ReadonlyAgentProfile + /** Runtime-derived identity for upstream optimizer resume state. */ + readonly evaluationRef: Sha256Digest + /** Exact profile coordinate being optimized. */ + readonly surface: ImproveProfileSurface + /** Exact bytes supplied to the optimization method. */ + readonly baselineSurface: MutableSurface + /** Structured value represented by `baselineSurface`, before serialization. */ + readonly baselineValue: unknown + /** Findings produced before this search, if any. */ + readonly findings: readonly unknown[] +} + +/** Build a complete method after trace findings are available. */ +export type ImproveMethodFactory = ( + context: ImproveMethodContext, +) => OptimizationMethod + +export type ImproveMethodSource = + | OptimizationMethod + | ImproveMethodFactory + +/** Runs one exact materialized profile on one scenario. */ +export type ImproveProfileAgent = ( + profile: ReadonlyAgentProfile, + scenario: TScenario, + ctx: Parameters< + CompareOptimizationMethodsOptions['dispatchWithSurface'] + >[2], +) => Promise + +export type ImproveOptimizationRunOptions = Omit< + NonNullable['optimizationRunOptions']>, + 'dispatchRef' +> + +/** Complete-method configuration for every non-code profile surface. */ +export type ImproveMethodOptions = Omit< + CompareOptimizationMethodsOptions, + | 'baselineSurface' + | 'dispatchRef' + | 'dispatchWithSurface' + | 'methods' + | 'optimizationConcurrency' + | 'optimizationRunOptions' +> & { + /** Exact profile coordinate optimized by `method`. Default `'prompt'`. */ + surface?: ImproveProfileSurface + /** + * Immutable digest of `agent`, profile component mapping, models, tools, and + * every closure or external setting that can change measured behavior. + */ + executionRef: Sha256Digest + /** A complete optimizer or a factory that can incorporate current findings. */ + method: ImproveMethodSource + /** Runs the exact complete profile materialized from one candidate surface. */ + agent: ImproveProfileAgent + /** Trace or analyst findings available to a method factory. */ + findings?: readonly unknown[] + /** Select the exact inline skill document for `surface: 'skills'`. */ + skills?: ImproveSkillsOptions + /** + * Map a profile to named text components and apply the winning components. + * Valid only with `surface: 'agent-profile'`. + */ + profileComponents?: ImproveProfileComponents + /** Shared settings for method train and selection calls. */ + optimizationRunOptions?: ImproveOptimizationRunOptions + /** Ship only when the paired final-test interval is entirely above this lift. Default `0`. */ + minimumLift?: number +} + +/** Runtime-owned code search in isolated git worktrees. */ +export type ImproveCodeRunOptions = Omit< + SelfImproveOptions, + | 'analyzeGeneration' + | 'baselineSurface' + | 'budget' + | 'findings' + | 'gate' + | 'llm' + | 'method' + | 'mutationPrimitives' + | 'proposer' + | 'proposerTarget' + | 'selectionScenarios' +> & { + surface: 'code' + /** Local code-search budget. Method-only selection controls do not apply. */ + budget?: Omit + /** Findings supplied to Runtime's code candidate driver. */ + findings?: readonly unknown[] + /** Gate mode. `'holdout'` (default) runs the held-out promotion gate; + * `'none'` is a baseline-only run (`budget.generations = 0`). */ + gate?: 'holdout' | 'none' + /** Per-generation findings producer for Runtime's code search. + * Pass your own producer to replace the code-trace distiller; pass `null` + * to keep the static findings for every generation. */ + analyzeGeneration?: SelfImproveOptions['analyzeGeneration'] | null + /** Feed code candidates paths to prior raw traces instead of a failure digest. + * Defaults to true for durable runs and false for in-memory runs. */ + rawTraceContext?: boolean + /** Isolated repository and candidate generator settings. */ + code: ImproveCodeOptions + /** Custom held-back-exam decision. The string `gate` above controls whether + * the exam runs; this callback controls how its evidence decides promotion. */ + promotionGate?: SelfImproveOptions['gate'] +} + +/** The canonical improvement API: complete methods for profiles, worktrees for code. */ +export type ImproveOptions = + | ImproveMethodOptions + | ImproveCodeRunOptions + +export interface ImproveSkillsOptions { + /** `name` of one inline entry in `profile.resources.skills`. */ + resourceName: string +} + +/** Caller-owned mapping for optimizing several profile fields as one candidate. */ +export interface ImproveProfileComponents { + /** Extract the exact named text components optimized together. */ + read(profile: ReadonlyAgentProfile): Readonly> + /** Apply a complete winning component map to a detached profile. */ + apply( + profile: ReadonlyAgentProfile, + components: Readonly>, + ): ReadonlyAgentProfile +} + +export interface ImproveCodeOptions { + /** Repo root candidate worktrees fork from. */ + repoRoot: string + /** Base ref candidates fork from. Default `main`. */ + baseRef?: string + /** Directory worktrees are created under. Default `/.worktrees`. */ + worktreeDir?: string + /** Git-compatible adapter override, primarily for tests. Candidate advancement + * still requires normal Git worktree and commit semantics. */ + worktree?: WorktreeAdapter + /** Coding harness the agentic generator runs in each worktree. Default `claude`. */ + harness?: LocalHarness + /** Verify a candidate worktree before it becomes a measurable surface; failures + * feed the next shot (see `agenticGenerator.verify` / `commandVerifier`). */ + verify?: Verifier + /** Per-shot wall-clock timeout for the harness (ms). */ + timeoutMs?: number + /** Byte-producer override, used for tests and custom candidate production. + * When set, `harness`, `verify`, and `timeoutMs` are unused. */ + generator?: CandidateGenerator +} + +export interface ImprovementProfileCandidate { + /** Surface searched by this run. */ + surface: ImproveProfileSurface + /** Exact winning value returned by agent-eval. */ + value: MutableSurface + /** Exact complete profile instance measured on the final cases. */ + profile: ReadonlyAgentProfile +} + +export interface ImprovementCodeCandidate { + surface: 'code' + value: MutableSurface + profile?: never +} + +export type ImprovementCandidate = ImprovementProfileCandidate | ImprovementCodeCandidate + +/** Normalized spend reported for one Runtime improvement run. */ +export interface ImproveCost { + totalCostUsd: number + accountingComplete: boolean + incompleteReasons: string[] +} + +/** Optimizer ancestry sealed into downstream candidate experiments. */ +export interface ImproveLineage { + /** Upstream optimizer run when reported, otherwise this Runtime optimization invocation. */ + runId: string + /** Exact train-plus-selection scenario payloads exposed to candidate selection. */ + developmentSplitDigest: Sha256Digest + /** Complete callback, materializer, model, tool, and closure identity for a profile run. */ + executionRef?: Sha256Digest + /** Complete baseline profile identity for a profile run. */ + baselineProfileDigest?: Sha256Digest +} + +interface ImproveResultBase { + /** Frozen candidate only. Live state is changed through an approved activation. */ + candidate: TCandidate + /** Final-test decision for this search result. */ + decision: SelfImproveResult['gateDecision'] + /** Final-test lift when one was measured. */ + lift?: number + /** Paired final-test confidence interval for method-based profile runs. */ + liftInterval?: { low: number; high: number } + /** Full search and final-test spend. */ + cost: ImproveCost + /** Full wall-clock duration. */ + durationMs: number + /** Optimizer ancestry used when sealing a candidate experiment. */ + lineage: ImproveLineage + /** Number of generations explored by Runtime's code path. */ + generationsExplored?: number + /** Release resources owned by this result. Idempotent; currently disposes + * the returned code worktree and is a no-op for profile-only surfaces. */ + dispose(): Promise +} + +export interface ImproveMethodResult extends ImproveResultBase { + mode: 'method' + method: string + /** External optimizer package and resumable run identity, when reported. */ + provenance?: OptimizationMethodComparison['best']['provenance'] + decision: 'ship' | 'hold' + lift: number + liftInterval: { low: number; high: number } + raw: OptimizationMethodComparison +} + +export interface ImproveCodeResult + extends ImproveResultBase { + mode: 'code' + raw: SelfImproveResult +} + +export type ImproveResult = + | ImproveMethodResult + | ImproveCodeResult diff --git a/src/improvement/improve.ts b/src/improvement/improve.ts index 05b80edd..ea01514d 100644 --- a/src/improvement/improve.ts +++ b/src/improvement/improve.ts @@ -9,1126 +9,44 @@ * @experimental */ -import { randomUUID } from 'node:crypto' -import { canonicalJson, makeFinding } from '@tangle-network/agent-eval' -import { - type CompareOptimizationMethodsOptions, - campaignSplitDigest, - compareOptimizationMethods, - gitWorktreeAdapter, - type OptimizationMethod, - type OptimizationMethodComparison, - type Worktree, - type WorktreeAdapter, -} from '@tangle-network/agent-eval/campaign' -import { - type CodeSurface, - type MutableSurface, - type Scenario, - type SelfImproveBudget, - type SelfImproveOptions, - type SelfImproveResult, - type SurfaceProposer, - selfImprove, -} from '@tangle-network/agent-eval/contract' -import { - type AgentProfile, - type AgentProfileResourceRef, - agentProfileSchema, - type Sha256Digest, - sha256DigestSchema, -} from '@tangle-network/agent-interface' -import { canonicalCandidateDigest, immutableCandidateValue } from '../candidate-execution/digest' +import type { Scenario } from '@tangle-network/agent-eval/contract' +import { type AgentProfile, agentProfileSchema } from '@tangle-network/agent-interface' +import { immutableCandidateValue } from '../candidate-execution/digest' import { ConfigError } from '../errors' -import type { LocalHarness } from '../mcp/local-harness' -import { agenticGenerator, type Verifier } from './agentic-generator' -import { rethrowAfterCleanup } from './cleanup' -import { - type CandidateGenerator, - improvementDriver, - type ManagedImprovementDriver, -} from './improvement-driver' -import type { ReadonlyAgentProfile } from './profile-types' -import { rawTraceDistiller } from './raw-trace-distiller' -import { - applyRolloutPolicyToProfile, - normalizeRolloutPolicy, - serializeRolloutPolicy, - structuralRolloutPolicyFromProfile, -} from './rollout-policy' - -/** The executable agent lever `improve` optimizes. Profile fields remain - * portable AgentProfile coordinates; implementation and orchestration files - * use the code surface so a winner can be sealed into an exact candidate. - * `rollout-policy` is the inference-time structuralRollout dials - * (`profile.extensions['structural-rollout']`). */ -export type ImproveSurface = - | 'prompt' - | 'skills' - | 'tools' - | 'mcp' - | 'hooks' - | 'subagents' - | 'agent-profile' - | 'memory' - | 'code' - | 'rollout-policy' - -export type ImproveProfileSurface = Exclude - -export interface ImproveMethodContext { - /** Validated baseline profile. */ - readonly profile: ReadonlyAgentProfile - /** Runtime-derived identity for upstream optimizer resume state. */ - readonly evaluationRef: Sha256Digest - /** Exact profile coordinate being optimized. */ - readonly surface: ImproveProfileSurface - /** Exact bytes supplied to the optimization method. */ - readonly baselineSurface: MutableSurface - /** Structured value represented by `baselineSurface`, before serialization. */ - readonly baselineValue: unknown - /** Findings produced before this search, if any. */ - readonly findings: readonly unknown[] -} - -/** Build a complete method after trace findings are available. */ -export type ImproveMethodFactory = ( - context: ImproveMethodContext, -) => OptimizationMethod - -export type ImproveMethodSource = - | OptimizationMethod - | ImproveMethodFactory - -/** Runs one exact materialized profile on one scenario. */ -export type ImproveProfileAgent = ( - profile: ReadonlyAgentProfile, - scenario: TScenario, - ctx: Parameters< - CompareOptimizationMethodsOptions['dispatchWithSurface'] - >[2], -) => Promise - -export type ImproveOptimizationRunOptions = Omit< - NonNullable['optimizationRunOptions']>, - 'dispatchRef' -> - -/** Complete-method configuration for every non-code profile surface. */ -export type ImproveMethodOptions = Omit< - CompareOptimizationMethodsOptions, - | 'baselineSurface' - | 'dispatchRef' - | 'dispatchWithSurface' - | 'methods' - | 'optimizationConcurrency' - | 'optimizationRunOptions' -> & { - /** Exact profile coordinate optimized by `method`. Default `'prompt'`. */ - surface?: ImproveProfileSurface - /** - * Immutable digest of `agent`, profile component mapping, models, tools, and - * every closure or external setting that can change measured behavior. - */ - executionRef: Sha256Digest - /** A complete optimizer or a factory that can incorporate current findings. */ - method: ImproveMethodSource - /** Runs the exact complete profile materialized from one candidate surface. */ - agent: ImproveProfileAgent - /** Trace or analyst findings available to a method factory. */ - findings?: readonly unknown[] - /** Select the exact inline skill document for `surface: 'skills'`. */ - skills?: ImproveSkillsOptions - /** - * Map a profile to named text components and apply the winning components. - * Valid only with `surface: 'agent-profile'`. - */ - profileComponents?: ImproveProfileComponents - /** Shared settings for method train and selection calls. */ - optimizationRunOptions?: ImproveOptimizationRunOptions - /** Ship only when the paired final-test interval is entirely above this lift. Default `0`. */ - minimumLift?: number -} - -/** Runtime-owned code search in isolated git worktrees. */ -export type ImproveCodeRunOptions = Omit< - SelfImproveOptions, - | 'analyzeGeneration' - | 'baselineSurface' - | 'budget' - | 'findings' - | 'gate' - | 'llm' - | 'method' - | 'mutationPrimitives' - | 'proposer' - | 'proposerTarget' - | 'selectionScenarios' -> & { - surface: 'code' - /** Local code-search budget. Method-only selection controls do not apply. */ - budget?: Omit - /** Findings supplied to Runtime's code candidate driver. */ - findings?: readonly unknown[] - /** Gate mode. `'holdout'` (default) runs the held-out promotion gate; - * `'none'` is a baseline-only run (`budget.generations = 0`). */ - gate?: 'holdout' | 'none' - /** Per-generation findings producer for Runtime's code search. - * Pass your own producer to replace the code-trace distiller; pass `null` - * to keep the static findings for every generation. */ - analyzeGeneration?: SelfImproveOptions['analyzeGeneration'] | null - /** Feed code candidates paths to prior raw traces instead of a failure digest. - * Defaults to true for durable runs and false for in-memory runs. */ - rawTraceContext?: boolean - /** Isolated repository and candidate generator settings. */ - code: ImproveCodeOptions - /** Custom held-back-exam decision. The string `gate` above controls whether - * the exam runs; this callback controls how its evidence decides promotion. */ - promotionGate?: SelfImproveOptions['gate'] -} - -/** The canonical improvement API: complete methods for profiles, worktrees for code. */ -export type ImproveOptions = - | ImproveMethodOptions - | ImproveCodeRunOptions - -export interface ImproveSkillsOptions { - /** `name` of one inline entry in `profile.resources.skills`. */ - resourceName: string -} - -/** Caller-owned mapping for optimizing several profile fields as one candidate. */ -export interface ImproveProfileComponents { - /** Extract the exact named text components optimized together. */ - read(profile: ReadonlyAgentProfile): Readonly> - /** Apply a complete winning component map to a detached profile. */ - apply( - profile: ReadonlyAgentProfile, - components: Readonly>, - ): ReadonlyAgentProfile -} - -export interface ImproveCodeOptions { - /** Repo root candidate worktrees fork from. */ - repoRoot: string - /** Base ref candidates fork from. Default `main`. */ - baseRef?: string - /** Directory worktrees are created under. Default `/.worktrees`. */ - worktreeDir?: string - /** Git-compatible adapter override, primarily for tests. Candidate advancement - * still requires normal Git worktree and commit semantics. */ - worktree?: WorktreeAdapter - /** Coding harness the agentic generator runs in each worktree. Default `claude`. */ - harness?: LocalHarness - /** Verify a candidate worktree before it becomes a measurable surface; failures - * feed the next shot (see `agenticGenerator.verify` / `commandVerifier`). */ - verify?: Verifier - /** Per-shot wall-clock timeout for the harness (ms). */ - timeoutMs?: number - /** Byte-producer override — the test seam and the escape hatch for custom - * candidate production. When set, `harness`/`verify`/`timeoutMs` are unused. */ - generator?: CandidateGenerator -} - -export interface ImprovementProfileCandidate { - /** Surface searched by this run. */ - surface: ImproveProfileSurface - /** Exact winning value returned by agent-eval. */ - value: MutableSurface - /** Exact complete profile instance measured on the final cases. */ - profile: ReadonlyAgentProfile -} - -export interface ImprovementCodeCandidate { - surface: 'code' - value: MutableSurface - profile?: never -} - -export type ImprovementCandidate = ImprovementProfileCandidate | ImprovementCodeCandidate - -/** Normalized spend reported for one Runtime improvement run. */ -export interface ImproveCost { - totalCostUsd: number - accountingComplete: boolean - incompleteReasons: string[] -} - -/** Optimizer ancestry sealed into downstream candidate experiments. */ -export interface ImproveLineage { - /** Upstream optimizer run when reported, otherwise this Runtime optimization invocation. */ - runId: string - /** Exact train-plus-selection scenario payloads exposed to candidate selection. */ - developmentSplitDigest: Sha256Digest - /** Complete callback, materializer, model, tool, and closure identity for a profile run. */ - executionRef?: Sha256Digest - /** Complete baseline profile identity for a profile run. */ - baselineProfileDigest?: Sha256Digest -} - -interface ImproveResultBase { - /** Frozen candidate only. Live state is changed through an approved activation. */ - candidate: TCandidate - /** Final-test decision for this search result. */ - decision: SelfImproveResult['gateDecision'] - /** Final-test lift when one was measured. */ - lift?: number - /** Paired final-test confidence interval for method-based profile runs. */ - liftInterval?: { low: number; high: number } - /** Full search and final-test spend. */ - cost: ImproveCost - /** Full wall-clock duration. */ - durationMs: number - /** Optimizer ancestry used when sealing a candidate experiment. */ - lineage: ImproveLineage - /** Number of generations explored by Runtime's code path. */ - generationsExplored?: number - /** Release resources owned by this result. Idempotent; currently disposes - * the returned code worktree and is a no-op for profile-only surfaces. */ - dispose(): Promise -} - -export interface ImproveMethodResult extends ImproveResultBase { - mode: 'method' - method: string - /** External optimizer package and resumable run identity, when reported. */ - provenance?: OptimizationMethodComparison['best']['provenance'] - decision: 'ship' | 'hold' - lift: number - liftInterval: { low: number; high: number } - raw: OptimizationMethodComparison -} - -export interface ImproveCodeResult - extends ImproveResultBase { - mode: 'code' - raw: SelfImproveResult -} - -export type ImproveResult = - | ImproveMethodResult - | ImproveCodeResult - -interface PreparedProfileSurface { - surface: MutableSurface - value: unknown -} - -/** Extract the baseline optimized by a method and retain its structured value - * so external optimizers can inspect it for private fields before serialization. */ -function prepareProfileSurface( - profile: ReadonlyAgentProfile, - surface: ImproveSurface, - skills?: ImproveSkillsOptions, - profileComponents?: ImproveProfileComponents, -): PreparedProfileSurface { - switch (surface) { - case 'prompt': - return { - surface: profile.prompt?.systemPrompt ?? '', - value: profile.prompt?.systemPrompt ?? '', - } - case 'skills': { - const value = inlineSkill(profile, skills).content - return { surface: value, value } - } - case 'tools': { - const value = profile.tools ?? {} - return { surface: canonicalJson(value), value } - } - case 'mcp': { - const value = profile.mcp ?? {} - return { surface: canonicalJson(value), value } - } - case 'hooks': { - const value = profile.hooks ?? {} - return { surface: canonicalJson(value), value } - } - case 'subagents': { - const value = profile.subagents ?? {} - return { surface: canonicalJson(value), value } - } - case 'agent-profile': { - if (profileComponents) { - const value = profileComponents.read(profile) - return { - surface: componentSurface(value, 'profileComponents.read'), - value, - } - } - return { surface: canonicalJson(profile), value: profile } - } - case 'memory': { - const value = profileInstructions(profile) - return { surface: value, value } - } - case 'rollout-policy': { - const policy = structuralRolloutPolicyFromProfile(profile) - if (!policy) { - throw new ConfigError( - "improve(): surface 'rollout-policy' requires an existing structural rollout policy", - ) - } - return { surface: serializeRolloutPolicy(policy), value: policy } - } - case 'code': - throw new ConfigError( - 'improve(): code requires the isolated baseline created from opts.code.repoRoot', - ) - } -} - -/** Slice bound for distilled judge notes: wide enough that a real traceback / - * failing-assertion note survives intact — clipping mid-traceback would leave - * the proposer trace-blind. Referenced by the `rawTraceContext` docs. */ -const DISTILLED_NOTES_MAX_CHARS = 1500 - -/** Slice bound for a cell's error string — tighter than notes (errors are - * usually one line; the full text stays on the raw cell). Also bounds the - * error-derived `claim` fallback. */ -const DISTILLED_ERROR_MAX_CHARS = 500 - -/** The in-memory-run `analyzeGeneration`: distill each generation's failing cells - * into TYPED `AnalystFinding`s for the next proposal round — the same envelope - * `rawTraceDistiller` and the analyst registry emit, so consumers never branch on - * a second ad-hoc digest shape. Judge notes and errors are already the domain's - * own diagnosis (executable gates put their reasons there); a trace-analyst can - * replace this wholesale via `opts.analyzeGeneration`. Falls back to the static - * seed findings when the generation had no failures, so a clean round never wipes - * the seed context. */ -function generationFailureDistiller( - staticFindings: unknown[], -): NonNullable['analyzeGeneration']> { - const CAP = 12 - return async (input) => { - const failures: Array<{ - scenario: string - composite: number - notes: string - claim?: string - error?: string - }> = [] - for (const candidate of input.candidates) { - for (const rawCell of candidate.campaign.cells) { - const cell = rawCell as unknown as Record - const scenario = String(cell.scenarioId ?? 'unknown') - const error = typeof cell.error === 'string' ? cell.error : undefined - const judgeScores = - cell.judgeScores && typeof cell.judgeScores === 'object' - ? Object.values( - cell.judgeScores as Record, - ) - : [] - const composite = - judgeScores.length === 0 - ? 0 - : judgeScores.reduce((sum, j) => sum + (j.composite ?? 0), 0) / judgeScores.length - if (!error && composite >= 0.999) continue - const notes = judgeScores - .map((j) => j.notes) - .filter((n): n is string => typeof n === 'string' && n.length > 0) - .join('; ') - .slice(0, DISTILLED_NOTES_MAX_CHARS) - const claim = - notes || - (error ? `Scenario ${scenario} failed: ${error.slice(0, DISTILLED_ERROR_MAX_CHARS)}` : '') - failures.push({ - scenario, - composite: Number(composite.toFixed(3)), - notes, - ...(claim ? { claim } : {}), - ...(error ? { error: error.slice(0, DISTILLED_ERROR_MAX_CHARS) } : {}), - }) - } - } - if (failures.length === 0) return staticFindings - failures.sort((a, b) => a.composite - b.composite) - return failures.slice(0, CAP).map((f) => - makeFinding({ - analyst_id: 'generation-failure-distiller', - severity: f.error !== undefined || f.composite < 0.5 ? 'high' : 'medium', - area: 'generation-failure', - confidence: 1, - subject: f.scenario, - claim: `Scenario ${f.scenario} scored composite ${f.composite}${ - f.notes ? `: ${f.notes}` : '' - }${f.error ? ` (error: ${f.error})` : ''}`, - evidence_refs: [], - metadata: { - scenario: f.scenario, - composite: f.composite, - ...(f.notes ? { notes: f.notes } : {}), - ...(f.error !== undefined ? { error: f.error } : {}), - }, - }), - ) - } -} - -/** Default code-run analysis: raw trace paths for durable runs, otherwise a - * bounded digest of failed cells. */ -function defaultDistillerFor( - opts: ImproveCodeRunOptions, - findings: unknown[], -): NonNullable['analyzeGeneration']> { - const durableRun = opts.runDir !== undefined && !opts.runDir.startsWith('mem://') - const useRawTraces = opts.rawTraceContext ?? durableRun - if (useRawTraces) return rawTraceDistiller({ fallbackFindings: findings }) - return generationFailureDistiller(findings) -} - -interface PreparedCodeRun { - baseline: CodeSurface - proposer: SurfaceProposer - cleanup(retainedWinner?: MutableSurface): Promise -} - -async function discardPreparedBaseline( - worktree: WorktreeAdapter, - baselineWorktree: Worktree, - cause: unknown, -): Promise { - return rethrowAfterCleanup( - cause, - () => worktree.discard(baselineWorktree), - 'improve(): code preparation failed', - ) -} - -function isCodeSurface(surface: MutableSurface | undefined): surface is CodeSurface { - return typeof surface === 'object' && surface !== null && surface.kind === 'code' -} - -type ComponentSurface = Extract - -function isComponentSurface(surface: MutableSurface | undefined): surface is ComponentSurface { - return typeof surface === 'object' && surface !== null && surface.kind === 'components' -} - -function componentSurface( - components: Readonly>, - source: string, -): ComponentSurface { - const entries = validateComponents(components, source) - return immutableCandidateValue({ - kind: 'components', - components: Object.fromEntries(entries), - } as ComponentSurface) -} - -function validateComponents( - components: Readonly>, - source: string, -): Array<[string, string]> { - if (typeof components !== 'object' || components === null || Array.isArray(components)) { - throw new ConfigError(`improve(): ${source} must return a component record`) - } - const entries = Object.entries(components) - if (entries.length === 0) { - throw new ConfigError(`improve(): ${source} must return at least one component`) - } - for (const [name, value] of entries) { - if (!name || name.trim() !== name || typeof value !== 'string') { - throw new ConfigError( - `improve(): ${source} must return trimmed component names with string values`, - ) - } - } - return entries -} - -/** Create a clean incumbent checkout and the candidate producer for a code run. */ -async function prepareCodeRun(code: ImproveCodeOptions): Promise { - const baseRef = code.baseRef ?? 'main' - const worktree = - code.worktree ?? - gitWorktreeAdapter({ - repoRoot: code.repoRoot, - ...(code.worktreeDir ? { worktreeDir: code.worktreeDir } : {}), - }) - const baselineWorktree = await worktree.create({ baseRef, label: 'incumbent-baseline' }) - try { - const baseline = await worktree.finalize(baselineWorktree, 'Incumbent code checkout') - let baselineDiscarded = false - const generator = - code.generator ?? - agenticGenerator({ - ...(code.harness ? { harness: code.harness } : {}), - ...(code.verify ? { verify: code.verify } : {}), - ...(code.timeoutMs ? { timeoutMs: code.timeoutMs } : {}), - }) - const managed: ManagedImprovementDriver = improvementDriver({ worktree, generator, baseRef }) - - return { - baseline, - proposer: managed, - async cleanup(retainedWinner) { - const errors: unknown[] = [] - const retainedWorktreeRef = isCodeSurface(retainedWinner) - ? retainedWinner.worktreeRef - : undefined - try { - await managed?.cleanup(retainedWorktreeRef ? [retainedWorktreeRef] : []) - } catch (cause) { - errors.push(cause) - } - if (!baselineDiscarded && retainedWorktreeRef !== baseline.worktreeRef) { - try { - await worktree.discard(baselineWorktree) - baselineDiscarded = true - } catch (cause) { - errors.push(cause) - } - } - if (errors.length > 0) { - throw new AggregateError(errors, 'improve(): failed to clean code improvement worktrees') - } - }, - } - } catch (cause) { - return discardPreparedBaseline(worktree, baselineWorktree, cause) - } -} - -function idempotentDispose(dispose: () => Promise): () => Promise { - let disposed = false - let inFlight: Promise | undefined - return async () => { - if (disposed) return - if (inFlight) return inFlight - inFlight = (async () => { - await dispose() - disposed = true - })() - try { - await inFlight - } finally { - inFlight = undefined - } - } -} - -/** Parse a JSON winner surface with a typed, contextual error. */ -function parseWinnerJson(winner: string, surface: ImproveSurface): T { - try { - return JSON.parse(winner) as T - } catch (cause) { - throw new ConfigError( - `improve(): the '${surface}' candidate is not valid JSON, so it cannot form a profile candidate: ${ - (cause as Error).message - }`, - ) - } -} - -function assertCandidateSurfaceKind( - surface: ImproveSurface, - baseline: MutableSurface, - winner: MutableSurface, -): asserts winner is MutableSurface { - if (surface === 'code') { - if (isCodeSurface(winner)) return - throw new ConfigError( - `improve(): the '${surface}' candidate returned an incompatible surface value`, - ) - } - if (typeof baseline === 'string') { - if (typeof winner === 'string') return - throw new ConfigError( - `improve(): the '${surface}' candidate changed from a text surface to an incompatible surface value`, - ) - } - if (!isComponentSurface(baseline) || !isComponentSurface(winner)) { - throw new ConfigError( - `improve(): the '${surface}' candidate returned an incompatible surface value`, - ) - } - validateComponents(winner.components, `the '${surface}' candidate`) - const baselineNames = Object.keys(baseline.components).sort() - const winnerNames = Object.keys(winner.components).sort() - if ( - baselineNames.length !== winnerNames.length || - baselineNames.some((name, index) => name !== winnerNames[index]) - ) { - throw new ConfigError( - `improve(): the '${surface}' candidate must preserve the exact component names`, - ) - } -} - -/** Materialize a detached profile candidate without changing the baseline. */ -function materializeImprovementProfileCandidate( - profile: AgentProfile, - surface: ImproveProfileSurface, - winner: MutableSurface, - skills?: ImproveSkillsOptions, - profileComponents?: ImproveProfileComponents, -): AgentProfile { - let candidate: AgentProfile - if (isComponentSurface(winner)) { - if (surface !== 'agent-profile' || !profileComponents) { - throw new ConfigError( - `improve(): the '${surface}' candidate has no profile component mapping`, - ) - } - const winnerComponents = immutableCandidateValue({ ...winner.components }) - const applied = profileComponents.apply(profile, winnerComponents) - const validated = validateProfileCandidate(applied, surface) - const materializedComponents = Object.fromEntries( - validateComponents(profileComponents.read(validated), 'profileComponents.read after apply'), - ) - const names = Object.keys(winnerComponents) - if ( - names.length !== Object.keys(materializedComponents).length || - names.some((name) => materializedComponents[name] !== winnerComponents[name]) - ) { - throw new ConfigError( - 'improve(): profileComponents.apply must round-trip every winning component exactly', - ) - } - return validated - } - if (typeof winner !== 'string') { - throw new ConfigError(`improve(): the '${surface}' candidate cannot form an AgentProfile`) - } - switch (surface) { - case 'prompt': - candidate = { ...profile, prompt: { ...profile.prompt, systemPrompt: winner } } - break - case 'skills': { - const selectedSkill = inlineSkill(profile, skills) - candidate = { - ...profile, - resources: { - ...profile.resources, - skills: profile.resources?.skills?.map((resource) => - resource === selectedSkill ? { ...resource, content: winner } : resource, - ), - }, - } - break - } - case 'tools': - candidate = { ...profile, tools: parseWinnerJson(winner, surface) } - break - case 'mcp': - candidate = { ...profile, mcp: parseWinnerJson(winner, surface) } - break - case 'hooks': - candidate = { ...profile, hooks: parseWinnerJson(winner, surface) } - break - case 'subagents': - candidate = { ...profile, subagents: parseWinnerJson(winner, surface) } - break - case 'agent-profile': - candidate = parseWinnerJson(winner, surface) - break - case 'memory': - candidate = { - ...profile, - resources: { - ...profile.resources, - instructions: replaceProfileInstructions(profile, winner), - }, - } - break - case 'rollout-policy': { - const policy = normalizeRolloutPolicy(parseWinnerJson(winner, surface)) - if (!policy) { - throw new ConfigError( - `improve(): the shipped 'rollout-policy' winner is not a valid StructuralRolloutPolicy ` + - `(integer k >= 1, repairRounds >= 0, testgen >= 0), so it cannot be applied: ${winner}`, - ) - } - candidate = applyRolloutPolicyToProfile(profile, policy) - break - } - } - return validateProfileCandidate(candidate, surface) -} - -function validateProfileCandidate(candidate: unknown, surface: ImproveSurface): AgentProfile { - const parsed = agentProfileSchema.safeParse(candidate) - if (!parsed.success) { - throw new ConfigError( - `improve(): the '${surface}' candidate does not produce a valid AgentProfile: ${parsed.error.message}`, - ) - } - return immutableCandidateValue(parsed.data) -} - -function createProfileCandidateMaterializer( - profile: AgentProfile, - surface: ImproveProfileSurface, - baselineSurface: MutableSurface, - skills?: ImproveSkillsOptions, - profileComponents?: ImproveProfileComponents, -): (candidateSurface: MutableSurface) => AgentProfile { - const baselineDigest = canonicalCandidateDigest(baselineSurface) - if (profileComponents) { - const reappliedBaseline = materializeImprovementProfileCandidate( - profile, - surface, - baselineSurface, - skills, - profileComponents, - ) - if (canonicalCandidateDigest(reappliedBaseline) !== canonicalCandidateDigest(profile)) { - throw new ConfigError( - 'improve(): profileComponents.apply(profile, profileComponents.read(profile)) must reproduce the complete baseline profile exactly', - ) - } - } - const candidates = new Map([[baselineDigest, profile]]) - return (candidateSurface) => { - assertCandidateSurfaceKind(surface, baselineSurface, candidateSurface) - const digest = canonicalCandidateDigest(candidateSurface) - const existing = candidates.get(digest) - if (existing) return existing - const candidate = materializeImprovementProfileCandidate( - profile, - surface, - immutableCandidateValue(candidateSurface), - skills, - profileComponents, - ) - candidates.set(digest, candidate) - return candidate - } -} - -function inlineSkill( - profile: ReadonlyAgentProfile, - options: ImproveSkillsOptions | undefined, -): Readonly> { - const resourceName = options?.resourceName.trim() - if (!resourceName) { - throw new ConfigError( - "improve(): surface 'skills' requires opts.skills.resourceName for one inline profile skill", - ) - } - assertFailClosedResources(profile, 'skills') - const matches = (profile.resources?.skills ?? []).filter( - (resource) => resource.name === resourceName, - ) - if (matches.length !== 1 || matches[0]?.kind !== 'inline') { - throw new ConfigError( - `improve(): skill '${resourceName}' must identify exactly one inline profile resource`, - ) - } - return matches[0] -} - -function profileInstructions(profile: ReadonlyAgentProfile): string { - assertFailClosedResources(profile, 'memory') - const instructions = profile.resources?.instructions - if (instructions === undefined) return '' - if (typeof instructions === 'string') return instructions - if (instructions.kind === 'inline') return instructions.content - throw new ConfigError( - "improve(): surface 'memory' requires inline profile instructions so candidate bytes are exact", - ) -} - -function replaceProfileInstructions( - profile: AgentProfile, - content: string, -): string | AgentProfileResourceRef { - const instructions = profile.resources?.instructions - if (typeof instructions !== 'object') return content - if (instructions.kind !== 'inline') { - throw new ConfigError( - "improve(): surface 'memory' requires inline profile instructions so candidate bytes are exact", - ) - } - return { ...instructions, content } -} - -function assertFailClosedResources( - profile: ReadonlyAgentProfile, - surface: 'skills' | 'memory', -): void { - if (profile.resources?.failOnError !== true) { - throw new ConfigError( - `improve(): surface '${surface}' requires profile.resources.failOnError: true`, - ) - } -} - -function resolveOptimizationMethod( - source: ImproveMethodSource, - context: ImproveMethodContext, -): OptimizationMethod { - const method = typeof source === 'function' ? source(context) : source - if ( - !method || - typeof method !== 'object' || - typeof method.name !== 'string' || - method.name.trim() !== method.name || - method.name.length === 0 || - typeof method.optimize !== 'function' - ) { - throw new ConfigError( - 'improve(): method must be a complete OptimizationMethod with a trimmed name and optimize(input)', - ) - } - return method -} - -function copyCost(cost: { - totalCostUsd: number - accountingComplete: boolean - incompleteReasons: readonly string[] -}): ImproveCost { - return { - totalCostUsd: cost.totalCostUsd, - accountingComplete: cost.accountingComplete, - incompleteReasons: [...cost.incompleteReasons], - } -} - -function copyProvenance( - provenance: NonNullable, -): NonNullable { - return { - ...provenance, - source: { ...provenance.source }, - ...(provenance.tokenUsage ? { tokenUsage: { ...provenance.tokenUsage } } : {}), - } -} - -function validateExecutionRef(value: unknown): Sha256Digest { - const parsed = sha256DigestSchema.safeParse(value) - if (!parsed.success) { - throw new ConfigError('improve(): executionRef must be a lowercase sha256:<64 hex> digest') - } - return parsed.data -} - -function developmentSplitDigest( - trainScenarios: readonly TScenario[], - selectionScenarios: readonly TScenario[], - reps: number, -): Sha256Digest { - return canonicalCandidateDigest({ - train: campaignSplitDigest(trainScenarios, reps), - selection: campaignSplitDigest(selectionScenarios, reps), - }) -} - -async function runMethodImprovement( - profile: AgentProfile, - opts: ImproveMethodOptions, -): Promise { - const { - surface = 'prompt', - executionRef: inputExecutionRef, - method: methodSource, - agent, - findings: inputFindings = [], - skills, - profileComponents, - optimizationRunOptions, - minimumLift = 0, - ...comparisonOptions - } = opts - if (!Number.isFinite(minimumLift) || minimumLift < 0) { - throw new ConfigError( - 'improve(): minimumLift must be a finite number greater than or equal to 0', - ) - } - if (profileComponents && surface !== 'agent-profile') { - throw new ConfigError("improve(): profileComponents is valid only with surface 'agent-profile'") - } - const executionRef = validateExecutionRef(inputExecutionRef) - const findings = [...inputFindings] - const preparedSurface = prepareProfileSurface(profile, surface, skills, profileComponents) - const baselineSurface = preparedSurface.surface - const baselineValue = immutableCandidateValue(preparedSurface.value) - const baselineProfileDigest = canonicalCandidateDigest(profile) - const evaluationRef = canonicalCandidateDigest({ - executionRef, - baselineProfileDigest, - surface, - }) - const dispatchRef = `improve:${evaluationRef}` - const identifiedJudges = comparisonOptions.judges.map((judge) => - Object.freeze({ - ...judge, - judgeVersion: canonicalCandidateDigest({ - evaluationRef, - name: judge.name, - dimensions: judge.dimensions, - declaredJudgeVersion: judge.judgeVersion ?? null, - }), - }), - ) - const materializeProfile = createProfileCandidateMaterializer( - profile, - surface, - baselineSurface, - skills, - profileComponents, - ) - const runtimeRunId = `runtime-optimization:${randomUUID()}` - const splitDigest = developmentSplitDigest( - comparisonOptions.trainScenarios, - comparisonOptions.selectionScenarios, - optimizationRunOptions?.reps ?? 1, - ) - const method = resolveOptimizationMethod(methodSource, { - profile, - evaluationRef, - surface, - baselineSurface, - baselineValue, - findings, - }) - const measuredMethod: OptimizationMethod = { - ...method, - async optimize(input) { - const result = await method.optimize(input) - materializeProfile(result.winnerSurface) - return result - }, - } - const startedAt = Date.now() - const raw = await compareOptimizationMethods({ - ...comparisonOptions, - judges: identifiedJudges, - dispatchRef, - optimizationRunOptions: { - ...(optimizationRunOptions ?? {}), - dispatchRef, - }, - methods: [measuredMethod], - baselineSurface, - dispatchWithSurface: (candidateSurface, scenario, ctx) => - agent(materializeProfile(candidateSurface), scenario, ctx), - }) - if ( - comparisonOptions.costCeiling !== undefined && - raw.totalCost.totalCostUsd > comparisonOptions.costCeiling - ) { - throw new ConfigError( - `improve(): reported total cost $${raw.totalCost.totalCostUsd} exceeds costCeiling $${comparisonOptions.costCeiling}`, - ) - } - const score = raw.best - const winnerSurface = immutableCandidateValue(score.winnerSurface) - assertCandidateSurfaceKind(surface, baselineSurface, winnerSurface) - const candidateProfile = materializeProfile(winnerSurface) - const candidate: ImprovementProfileCandidate = Object.freeze({ - surface, - value: winnerSurface, - profile: candidateProfile, - }) - - const cost = copyCost(raw.totalCost) - return { - mode: 'method', - method: method.name, - ...(score.provenance ? { provenance: copyProvenance(score.provenance) } : {}), - candidate, - decision: cost.accountingComplete && score.liftCi.low > minimumLift ? 'ship' : 'hold', - lift: score.lift, - liftInterval: { ...score.liftCi }, - cost, - durationMs: Date.now() - startedAt, - lineage: Object.freeze({ - runId: score.provenance?.runId ?? runtimeRunId, - developmentSplitDigest: splitDigest, - executionRef, - baselineProfileDigest, - }), - raw, - async dispose() {}, - } -} - -async function runCodeImprovement( - opts: ImproveCodeRunOptions, -): Promise> { - const { - gate = 'holdout', - findings: inputFindings = [], - rawTraceContext: _rawTraceContext, - code, - promotionGate, - analyzeGeneration, - surface: _surface, - ...sharedOptions - } = opts - const findings = [...inputFindings] - const preparedCode = await prepareCodeRun(code) - - const budget: SelfImproveBudget = - gate === 'none' ? { ...sharedOptions.budget, generations: 0 } : { ...sharedOptions.budget } - - let raw: SelfImproveResult - try { - raw = await selfImprove({ - ...sharedOptions, - baselineSurface: preparedCode.baseline, - proposer: preparedCode.proposer, - budget, - findings, - ...(promotionGate !== undefined ? { gate: promotionGate } : {}), - ...(analyzeGeneration === null - ? {} - : { - analyzeGeneration: - analyzeGeneration ?? defaultDistillerFor(opts, findings), - }), - }) - } catch (cause) { - return rethrowAfterCleanup( - cause, - () => preparedCode.cleanup(), - 'improve(): code improvement failed', - ) - } - - const winnerSurface = raw.winner.surface - assertCandidateSurfaceKind('code', preparedCode.baseline, winnerSurface) - try { - await preparedCode.cleanup(winnerSurface) - } catch (cleanupCause) { - try { - await preparedCode.cleanup() - } catch (finalCleanupCause) { - throw new AggregateError( - [cleanupCause, finalCleanupCause], - 'improve(): code result cleanup failed, including the final all-worktree retry', - ) - } - throw new AggregateError( - [cleanupCause], - 'improve(): code result cleanup failed; the final all-worktree retry succeeded', - ) - } - const dispose = idempotentDispose(async () => preparedCode.cleanup()) - const candidate = immutableCandidateValue({ - surface: 'code', - value: winnerSurface, - }) - - return { - mode: 'code', - candidate, - decision: raw.gateDecision, - ...(raw.lift !== undefined ? { lift: raw.lift } : {}), - cost: copyCost(raw.cost), - durationMs: raw.durationMs, - lineage: Object.freeze({ - runId: raw.provenance.runId, - developmentSplitDigest: raw.provenance.evidence.search.splitDigest, - }), - generationsExplored: raw.generationsExplored, - raw, - dispose, - } -} +import { runCodeImprovement } from './code-execution' +import type { + ImproveCodeResult, + ImproveCodeRunOptions, + ImproveMethodOptions, + ImproveMethodResult, + ImproveOptions, + ImproveResult, +} from './improve-types' +import { runMethodImprovement } from './method-execution' + +export type { + ImproveCodeOptions, + ImproveCodeResult, + ImproveCodeRunOptions, + ImproveCost, + ImproveLineage, + ImproveMethodContext, + ImproveMethodFactory, + ImproveMethodOptions, + ImproveMethodResult, + ImproveMethodSource, + ImprovementCandidate, + ImprovementCodeCandidate, + ImprovementProfileCandidate, + ImproveOptimizationRunOptions, + ImproveOptions, + ImproveProfileAgent, + ImproveProfileComponents, + ImproveProfileSurface, + ImproveResult, + ImproveSkillsOptions, + ImproveSurface, +} from './improve-types' /** * Optimize one exact profile surface with a complete method, or optimize code diff --git a/src/improvement/method-execution.ts b/src/improvement/method-execution.ts new file mode 100644 index 00000000..f70c886e --- /dev/null +++ b/src/improvement/method-execution.ts @@ -0,0 +1,207 @@ +import { randomUUID } from 'node:crypto' +import { + campaignSplitDigest, + compareOptimizationMethods, + type OptimizationMethod, + type OptimizationMethodComparison, +} from '@tangle-network/agent-eval/campaign' +import type { Scenario } from '@tangle-network/agent-eval/contract' +import { + type AgentProfile, + type Sha256Digest, + sha256DigestSchema, +} from '@tangle-network/agent-interface' +import { canonicalCandidateDigest, immutableCandidateValue } from '../candidate-execution/digest' +import { ConfigError } from '../errors' +import { copyImproveCost } from './improve-result' +import type { + ImproveMethodContext, + ImproveMethodOptions, + ImproveMethodResult, + ImproveMethodSource, + ImprovementProfileCandidate, +} from './improve-types' +import { + assertCandidateSurfaceKind, + createProfileCandidateMaterializer, + prepareProfileSurface, +} from './profile-surface' + +function resolveOptimizationMethod( + source: ImproveMethodSource, + context: ImproveMethodContext, +): OptimizationMethod { + const method = typeof source === 'function' ? source(context) : source + if ( + !method || + typeof method !== 'object' || + typeof method.name !== 'string' || + method.name.trim() !== method.name || + method.name.length === 0 || + typeof method.optimize !== 'function' + ) { + throw new ConfigError( + 'improve(): method must be a complete OptimizationMethod with a trimmed name and optimize(input)', + ) + } + return method +} + +function copyProvenance( + provenance: NonNullable, +): NonNullable { + return { + ...provenance, + source: { ...provenance.source }, + ...(provenance.tokenUsage ? { tokenUsage: { ...provenance.tokenUsage } } : {}), + } +} + +function validateExecutionRef(value: unknown): Sha256Digest { + const parsed = sha256DigestSchema.safeParse(value) + if (!parsed.success) { + throw new ConfigError('improve(): executionRef must be a lowercase sha256:<64 hex> digest') + } + return parsed.data +} + +function developmentSplitDigest( + trainScenarios: readonly TScenario[], + selectionScenarios: readonly TScenario[], + reps: number, +): Sha256Digest { + return canonicalCandidateDigest({ + train: campaignSplitDigest(trainScenarios, reps), + selection: campaignSplitDigest(selectionScenarios, reps), + }) +} + +export async function runMethodImprovement( + profile: AgentProfile, + opts: ImproveMethodOptions, +): Promise { + const { + surface = 'prompt', + executionRef: inputExecutionRef, + method: methodSource, + agent, + findings: inputFindings = [], + skills, + profileComponents, + optimizationRunOptions, + minimumLift = 0, + ...comparisonOptions + } = opts + if (!Number.isFinite(minimumLift) || minimumLift < 0) { + throw new ConfigError( + 'improve(): minimumLift must be a finite number greater than or equal to 0', + ) + } + if (profileComponents && surface !== 'agent-profile') { + throw new ConfigError("improve(): profileComponents is valid only with surface 'agent-profile'") + } + const executionRef = validateExecutionRef(inputExecutionRef) + const findings = [...inputFindings] + const preparedSurface = prepareProfileSurface(profile, surface, skills, profileComponents) + const baselineSurface = preparedSurface.surface + const baselineValue = immutableCandidateValue(preparedSurface.value) + const baselineProfileDigest = canonicalCandidateDigest(profile) + const evaluationRef = canonicalCandidateDigest({ + executionRef, + baselineProfileDigest, + surface, + }) + const dispatchRef = `improve:${evaluationRef}` + const identifiedJudges = comparisonOptions.judges.map((judge) => + Object.freeze({ + ...judge, + judgeVersion: canonicalCandidateDigest({ + evaluationRef, + name: judge.name, + dimensions: judge.dimensions, + declaredJudgeVersion: judge.judgeVersion ?? null, + }), + }), + ) + const materializeProfile = createProfileCandidateMaterializer( + profile, + surface, + baselineSurface, + skills, + profileComponents, + ) + const runtimeRunId = `runtime-optimization:${randomUUID()}` + const splitDigest = developmentSplitDigest( + comparisonOptions.trainScenarios, + comparisonOptions.selectionScenarios, + optimizationRunOptions?.reps ?? 1, + ) + const method = resolveOptimizationMethod(methodSource, { + profile, + evaluationRef, + surface, + baselineSurface, + baselineValue, + findings, + }) + const measuredMethod: OptimizationMethod = { + ...method, + async optimize(input) { + const result = await method.optimize(input) + materializeProfile(result.winnerSurface) + return result + }, + } + const startedAt = Date.now() + const raw = await compareOptimizationMethods({ + ...comparisonOptions, + judges: identifiedJudges, + dispatchRef, + optimizationRunOptions: { + ...(optimizationRunOptions ?? {}), + dispatchRef, + }, + methods: [measuredMethod], + baselineSurface, + dispatchWithSurface: (candidateSurface, scenario, ctx) => + agent(materializeProfile(candidateSurface), scenario, ctx), + }) + if ( + comparisonOptions.costCeiling !== undefined && + raw.totalCost.totalCostUsd > comparisonOptions.costCeiling + ) { + throw new ConfigError( + `improve(): reported total cost $${raw.totalCost.totalCostUsd} exceeds costCeiling $${comparisonOptions.costCeiling}`, + ) + } + const score = raw.best + const winnerSurface = immutableCandidateValue(score.winnerSurface) + assertCandidateSurfaceKind(surface, baselineSurface, winnerSurface) + const candidateProfile = materializeProfile(winnerSurface) + const candidate: ImprovementProfileCandidate = Object.freeze({ + surface, + value: winnerSurface, + profile: candidateProfile, + }) + + const cost = copyImproveCost(raw.totalCost) + return { + mode: 'method', + method: method.name, + ...(score.provenance ? { provenance: copyProvenance(score.provenance) } : {}), + candidate, + decision: cost.accountingComplete && score.liftCi.low > minimumLift ? 'ship' : 'hold', + lift: score.lift, + liftInterval: { ...score.liftCi }, + cost, + durationMs: Date.now() - startedAt, + lineage: Object.freeze({ + runId: score.provenance?.runId ?? runtimeRunId, + developmentSplitDigest: splitDigest, + executionRef, + baselineProfileDigest, + }), + raw, + async dispose() {}, + } +} diff --git a/src/improvement/profile-surface.ts b/src/improvement/profile-surface.ts new file mode 100644 index 00000000..f9262bc4 --- /dev/null +++ b/src/improvement/profile-surface.ts @@ -0,0 +1,383 @@ +import { canonicalJson } from '@tangle-network/agent-eval' +import type { MutableSurface } from '@tangle-network/agent-eval/contract' +import { + type AgentProfile, + type AgentProfileResourceRef, + agentProfileSchema, + type Sha256Digest, +} from '@tangle-network/agent-interface' +import { canonicalCandidateDigest, immutableCandidateValue } from '../candidate-execution/digest' +import { ConfigError } from '../errors' +import type { + ImproveProfileComponents, + ImproveProfileSurface, + ImproveSkillsOptions, + ImproveSurface, +} from './improve-types' +import type { ReadonlyAgentProfile } from './profile-types' +import { + applyRolloutPolicyToProfile, + normalizeRolloutPolicy, + serializeRolloutPolicy, + structuralRolloutPolicyFromProfile, +} from './rollout-policy' + +export interface PreparedProfileSurface { + surface: MutableSurface + value: unknown +} + +/** Extract the baseline optimized by a method and retain its structured value + * so external optimizers can inspect it for private fields before serialization. */ +export function prepareProfileSurface( + profile: ReadonlyAgentProfile, + surface: ImproveSurface, + skills?: ImproveSkillsOptions, + profileComponents?: ImproveProfileComponents, +): PreparedProfileSurface { + switch (surface) { + case 'prompt': + return { + surface: profile.prompt?.systemPrompt ?? '', + value: profile.prompt?.systemPrompt ?? '', + } + case 'skills': { + const value = inlineSkill(profile, skills).content + return { surface: value, value } + } + case 'tools': { + const value = profile.tools ?? {} + return { surface: canonicalJson(value), value } + } + case 'mcp': { + const value = profile.mcp ?? {} + return { surface: canonicalJson(value), value } + } + case 'hooks': { + const value = profile.hooks ?? {} + return { surface: canonicalJson(value), value } + } + case 'subagents': { + const value = profile.subagents ?? {} + return { surface: canonicalJson(value), value } + } + case 'agent-profile': { + if (profileComponents) { + const value = profileComponents.read(profile) + return { + surface: componentSurface(value, 'profileComponents.read'), + value, + } + } + return { surface: canonicalJson(profile), value: profile } + } + case 'memory': { + const value = profileInstructions(profile) + return { surface: value, value } + } + case 'rollout-policy': { + const policy = structuralRolloutPolicyFromProfile(profile) + if (!policy) { + throw new ConfigError( + "improve(): surface 'rollout-policy' requires an existing structural rollout policy", + ) + } + return { surface: serializeRolloutPolicy(policy), value: policy } + } + case 'code': + throw new ConfigError( + 'improve(): code requires the isolated baseline created from opts.code.repoRoot', + ) + } +} + +export function isCodeSurface( + surface: MutableSurface | undefined, +): surface is Extract { + return typeof surface === 'object' && surface !== null && surface.kind === 'code' +} + +type ComponentSurface = Extract + +function isComponentSurface(surface: MutableSurface | undefined): surface is ComponentSurface { + return typeof surface === 'object' && surface !== null && surface.kind === 'components' +} + +function componentSurface( + components: Readonly>, + source: string, +): ComponentSurface { + const entries = validateComponents(components, source) + return immutableCandidateValue({ + kind: 'components', + components: Object.fromEntries(entries), + } as ComponentSurface) +} + +function validateComponents( + components: Readonly>, + source: string, +): Array<[string, string]> { + if (typeof components !== 'object' || components === null || Array.isArray(components)) { + throw new ConfigError(`improve(): ${source} must return a component record`) + } + const entries = Object.entries(components) + if (entries.length === 0) { + throw new ConfigError(`improve(): ${source} must return at least one component`) + } + for (const [name, value] of entries) { + if (!name || name.trim() !== name || typeof value !== 'string') { + throw new ConfigError( + `improve(): ${source} must return trimmed component names with string values`, + ) + } + } + return entries +} + +/** Parse a JSON winner surface with a typed, contextual error. */ +function parseWinnerJson(winner: string, surface: ImproveSurface): T { + try { + return JSON.parse(winner) as T + } catch (cause) { + throw new ConfigError( + `improve(): the '${surface}' candidate is not valid JSON, so it cannot form a profile candidate: ${ + (cause as Error).message + }`, + ) + } +} + +export function assertCandidateSurfaceKind( + surface: ImproveSurface, + baseline: MutableSurface, + winner: MutableSurface, +): asserts winner is MutableSurface { + if (surface === 'code') { + if (isCodeSurface(winner)) return + throw new ConfigError( + `improve(): the '${surface}' candidate returned an incompatible surface value`, + ) + } + if (typeof baseline === 'string') { + if (typeof winner === 'string') return + throw new ConfigError( + `improve(): the '${surface}' candidate changed from a text surface to an incompatible surface value`, + ) + } + if (!isComponentSurface(baseline) || !isComponentSurface(winner)) { + throw new ConfigError( + `improve(): the '${surface}' candidate returned an incompatible surface value`, + ) + } + validateComponents(winner.components, `the '${surface}' candidate`) + const baselineNames = Object.keys(baseline.components).sort() + const winnerNames = Object.keys(winner.components).sort() + if ( + baselineNames.length !== winnerNames.length || + baselineNames.some((name, index) => name !== winnerNames[index]) + ) { + throw new ConfigError( + `improve(): the '${surface}' candidate must preserve the exact component names`, + ) + } +} + +/** Materialize a detached profile candidate without changing the baseline. */ +function materializeImprovementProfileCandidate( + profile: AgentProfile, + surface: ImproveProfileSurface, + winner: MutableSurface, + skills?: ImproveSkillsOptions, + profileComponents?: ImproveProfileComponents, +): AgentProfile { + let candidate: AgentProfile + if (isComponentSurface(winner)) { + if (surface !== 'agent-profile' || !profileComponents) { + throw new ConfigError( + `improve(): the '${surface}' candidate has no profile component mapping`, + ) + } + const winnerComponents = immutableCandidateValue({ ...winner.components }) + const applied = profileComponents.apply(profile, winnerComponents) + const validated = validateProfileCandidate(applied, surface) + const materializedComponents = Object.fromEntries( + validateComponents(profileComponents.read(validated), 'profileComponents.read after apply'), + ) + const names = Object.keys(winnerComponents) + if ( + names.length !== Object.keys(materializedComponents).length || + names.some((name) => materializedComponents[name] !== winnerComponents[name]) + ) { + throw new ConfigError( + 'improve(): profileComponents.apply must round-trip every winning component exactly', + ) + } + return validated + } + if (typeof winner !== 'string') { + throw new ConfigError(`improve(): the '${surface}' candidate cannot form an AgentProfile`) + } + switch (surface) { + case 'prompt': + candidate = { ...profile, prompt: { ...profile.prompt, systemPrompt: winner } } + break + case 'skills': { + const selectedSkill = inlineSkill(profile, skills) + candidate = { + ...profile, + resources: { + ...profile.resources, + skills: profile.resources?.skills?.map((resource) => + resource === selectedSkill ? { ...resource, content: winner } : resource, + ), + }, + } + break + } + case 'tools': + candidate = { ...profile, tools: parseWinnerJson(winner, surface) } + break + case 'mcp': + candidate = { ...profile, mcp: parseWinnerJson(winner, surface) } + break + case 'hooks': + candidate = { ...profile, hooks: parseWinnerJson(winner, surface) } + break + case 'subagents': + candidate = { ...profile, subagents: parseWinnerJson(winner, surface) } + break + case 'agent-profile': + candidate = parseWinnerJson(winner, surface) + break + case 'memory': + candidate = { + ...profile, + resources: { + ...profile.resources, + instructions: replaceProfileInstructions(profile, winner), + }, + } + break + case 'rollout-policy': { + const policy = normalizeRolloutPolicy(parseWinnerJson(winner, surface)) + if (!policy) { + throw new ConfigError( + `improve(): the shipped 'rollout-policy' winner is not a valid StructuralRolloutPolicy ` + + `(integer k >= 1, repairRounds >= 0, testgen >= 0), so it cannot be applied: ${winner}`, + ) + } + candidate = applyRolloutPolicyToProfile(profile, policy) + break + } + } + return validateProfileCandidate(candidate, surface) +} + +function validateProfileCandidate(candidate: unknown, surface: ImproveSurface): AgentProfile { + const parsed = agentProfileSchema.safeParse(candidate) + if (!parsed.success) { + throw new ConfigError( + `improve(): the '${surface}' candidate does not produce a valid AgentProfile: ${parsed.error.message}`, + ) + } + return immutableCandidateValue(parsed.data) +} + +export function createProfileCandidateMaterializer( + profile: AgentProfile, + surface: ImproveProfileSurface, + baselineSurface: MutableSurface, + skills?: ImproveSkillsOptions, + profileComponents?: ImproveProfileComponents, +): (candidateSurface: MutableSurface) => AgentProfile { + const baselineDigest = canonicalCandidateDigest(baselineSurface) + if (profileComponents) { + const reappliedBaseline = materializeImprovementProfileCandidate( + profile, + surface, + baselineSurface, + skills, + profileComponents, + ) + if (canonicalCandidateDigest(reappliedBaseline) !== canonicalCandidateDigest(profile)) { + throw new ConfigError( + 'improve(): profileComponents.apply(profile, profileComponents.read(profile)) must reproduce the complete baseline profile exactly', + ) + } + } + const candidates = new Map([[baselineDigest, profile]]) + return (candidateSurface) => { + assertCandidateSurfaceKind(surface, baselineSurface, candidateSurface) + const digest = canonicalCandidateDigest(candidateSurface) + const existing = candidates.get(digest) + if (existing) return existing + const candidate = materializeImprovementProfileCandidate( + profile, + surface, + immutableCandidateValue(candidateSurface), + skills, + profileComponents, + ) + candidates.set(digest, candidate) + return candidate + } +} + +function inlineSkill( + profile: ReadonlyAgentProfile, + options: ImproveSkillsOptions | undefined, +): Readonly> { + const resourceName = options?.resourceName.trim() + if (!resourceName) { + throw new ConfigError( + "improve(): surface 'skills' requires opts.skills.resourceName for one inline profile skill", + ) + } + assertFailClosedResources(profile, 'skills') + const matches = (profile.resources?.skills ?? []).filter( + (resource) => resource.name === resourceName, + ) + if (matches.length !== 1 || matches[0]?.kind !== 'inline') { + throw new ConfigError( + `improve(): skill '${resourceName}' must identify exactly one inline profile resource`, + ) + } + return matches[0] +} + +function profileInstructions(profile: ReadonlyAgentProfile): string { + assertFailClosedResources(profile, 'memory') + const instructions = profile.resources?.instructions + if (instructions === undefined) return '' + if (typeof instructions === 'string') return instructions + if (instructions.kind === 'inline') return instructions.content + throw new ConfigError( + "improve(): surface 'memory' requires inline profile instructions so candidate bytes are exact", + ) +} + +function replaceProfileInstructions( + profile: AgentProfile, + content: string, +): string | AgentProfileResourceRef { + const instructions = profile.resources?.instructions + if (typeof instructions !== 'object') return content + if (instructions.kind !== 'inline') { + throw new ConfigError( + "improve(): surface 'memory' requires inline profile instructions so candidate bytes are exact", + ) + } + return { ...instructions, content } +} + +function assertFailClosedResources( + profile: ReadonlyAgentProfile, + surface: 'skills' | 'memory', +): void { + if (profile.resources?.failOnError !== true) { + throw new ConfigError( + `improve(): surface '${surface}' requires profile.resources.failOnError: true`, + ) + } +} From aa0f1dd9ee692fa3344b036bd0415eeeb1e1fe59 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 24 Jul 2026 11:31:56 -0600 Subject: [PATCH 09/14] refactor(primeintellect): remove private schema versions --- README.md | 6 +-- package.json | 2 +- scripts/gen-primitive-catalog.mjs | 2 +- scripts/verify-primeintellect-live.mjs | 2 +- ...llect-v1.mjs => verify-primeintellect.mjs} | 44 +++++++++---------- src/primeintellect/package.ts | 12 ++--- src/primeintellect/primeintellect.test.ts | 13 +++--- src/primeintellect/runner.ts | 1 - src/primeintellect/types.ts | 3 +- src/primeintellect/validation.ts | 2 +- 10 files changed, 43 insertions(+), 44 deletions(-) rename scripts/{verify-primeintellect-v1.mjs => verify-primeintellect.mjs} (81%) diff --git a/README.md b/README.md index 1a523e99..8133c0b5 100644 --- a/README.md +++ b/README.md @@ -317,7 +317,7 @@ Measure the returned bundle pair, record the review, then activate through `exec ### Run on PrimeIntellect -`@tangle-network/agent-runtime/primeintellect` packages typed train and eval tasks as a Verifiers v1 environment. +`@tangle-network/agent-runtime/primeintellect` packages typed train and eval tasks as a PrimeIntellect Verifiers environment. Prime launches your actual runtime program against an intercepted model endpoint, so `runPersonified`, `runAgentic`, product agents, tool calls, and multiple rounds stay intact. Reference answers remain in Prime's task process and never enter the agent workspace. The runner file must be one executable bundle containing the app and its runtime dependencies. @@ -331,7 +331,7 @@ import { const bundledRunner = await readFile('./dist/prime-runner.mjs', 'utf8') const bundle = createPrimeIntellectPackage({ - name: 'support-agent-v1', + name: 'support-agent', version: '1.0.0', tasks: [ { @@ -355,7 +355,7 @@ const bundle = createPrimeIntellectPackage({ }, }) -await writePrimeIntellectPackage(bundle, './prime/support-agent-v1') +await writePrimeIntellectPackage(bundle, './prime/support-agent') ``` The runner reads the episode and uses the normal runtime APIs: diff --git a/package.json b/package.json index 4e7fd502..b31169a0 100644 --- a/package.json +++ b/package.json @@ -114,7 +114,7 @@ "verify:bench": "pnpm build && pnpm --filter @tangle-network/agent-bench run typecheck:public && pnpm --filter @tangle-network/agent-bench test && pnpm --filter @tangle-network/agent-bench run verify:package:local-runtime", "verify:bench:published": "pnpm build && pnpm --filter @tangle-network/agent-bench run typecheck:public && pnpm --filter @tangle-network/agent-bench test && pnpm --filter @tangle-network/agent-bench run verify:package", "verify:package": "pnpm run check:testing-fixture && pnpm run check:skills && node scripts/verify-package-exports.mjs", - "verify:primeintellect": "pnpm build && node scripts/verify-primeintellect-v1.mjs", + "verify:primeintellect": "pnpm build && node scripts/verify-primeintellect.mjs", "verify:primeintellect:live": "pnpm build && node scripts/verify-primeintellect-live.mjs", "docs:api": "pnpm run build && typedoc && node scripts/gen-primitive-catalog.mjs", "docs:freshness": "node scripts/check-docs-freshness.mjs", diff --git a/scripts/gen-primitive-catalog.mjs b/scripts/gen-primitive-catalog.mjs index 592f81a9..98a6660b 100644 --- a/scripts/gen-primitive-catalog.mjs +++ b/scripts/gen-primitive-catalog.mjs @@ -69,7 +69,7 @@ const ownSurfaceLabels = { './environment-provider': 'Environment provider adapters — generic sandbox/compute bridge', './analyst-loop': 'Analyst loop — trace findings on a running loop', './knowledge': 'Knowledge orchestration — supervised KB updates', - './primeintellect': 'PrimeIntellect: Verifiers v1 package and trace adapter', + './primeintellect': 'PrimeIntellect: Verifiers package and trace adapter', './profiles': 'Built-in agent profiles', './platform': 'Platform glue', './candidate-execution': 'Candidate execution — immutable prepare, run, grade, and receipt', diff --git a/scripts/verify-primeintellect-live.mjs b/scripts/verify-primeintellect-live.mjs index 3f26a2da..51f9eb01 100644 --- a/scripts/verify-primeintellect-live.mjs +++ b/scripts/verify-primeintellect-live.mjs @@ -59,7 +59,7 @@ const scoringProgram = `import json import sys request = json.load(sys.stdin) -assert request["protocol"] == "tangle.primeintellect.score/v1" +assert request["kind"] == "tangle.primeintellect.score" trace = request["trace"] sampled = [node for node in trace["nodes"] if node.get("sampled") is True] contents = [str((node.get("message") or {}).get("content") or "").strip() for node in sampled] diff --git a/scripts/verify-primeintellect-v1.mjs b/scripts/verify-primeintellect.mjs similarity index 81% rename from scripts/verify-primeintellect-v1.mjs rename to scripts/verify-primeintellect.mjs index 9955d191..fa302882 100644 --- a/scripts/verify-primeintellect-v1.mjs +++ b/scripts/verify-primeintellect.mjs @@ -7,14 +7,14 @@ import { writePrimeIntellectPackage, } from '../dist/primeintellect/index.js' -const root = mkdtempSync(join(tmpdir(), 'agent-runtime-primeintellect-v1-')) -const output = join(root, 'strict-exact-v1') -const referenceOutput = join(root, 'reference-v1') -const commandOutput = join(root, 'command-v1') +const root = mkdtempSync(join(tmpdir(), 'agent-runtime-primeintellect-')) +const output = join(root, 'strict-exact') +const referenceOutput = join(root, 'reference') +const commandOutput = join(root, 'command') try { const bundle = createPrimeIntellectPackage({ - name: 'strict-exact-v1', + name: 'strict-exact', version: '1.0.0', tasks: [ { id: 'train', split: 'train', prompt: 'Is a subscription refundable?', answer: 'No' }, @@ -30,7 +30,7 @@ try { await writePrimeIntellectPackage(bundle, output) await writePrimeIntellectPackage( createPrimeIntellectPackage({ - name: 'reference-v1', + name: 'reference', version: '1.0.0', tasks: [ { id: 'train-ref', split: 'train', prompt: 'Practice reference?', answer: 'No' }, @@ -47,7 +47,7 @@ try { ) await writePrimeIntellectPackage( createPrimeIntellectPackage({ - name: 'command-v1', + name: 'command', version: '1.0.0', tasks: [ { id: 'train-command', split: 'train', prompt: 'Practice command?' }, @@ -58,7 +58,7 @@ try { command: ['python', 'scoring/score.py'], files: { 'score.py': - 'import json, sys\nrequest = json.load(sys.stdin)\nassert request["protocol"] == "tangle.primeintellect.score/v1"\nprint(json.dumps({"reward": 0.25, "metrics": {"custom": 0.75}}))\n', + 'import json, sys\nrequest = json.load(sys.stdin)\nassert request["kind"] == "tangle.primeintellect.score"\nprint(json.dumps({"reward": 0.25, "metrics": {"custom": 0.75}}))\n', }, }, runner: { @@ -78,18 +78,18 @@ from types import SimpleNamespace import verifiers.v1 as vf ROOT = Path(__file__).resolve().parent.parent -sys.path.insert(0, str(ROOT / "reference-v1")) -sys.path.insert(0, str(ROOT / "command-v1")) -from command_v1 import TangleTaskset as CommandTaskset -from command_v1.taskset import TangleTasksetConfig as CommandTasksetConfig -from reference_v1 import TangleTaskset as ReferenceTaskset -from reference_v1.taskset import TangleTasksetConfig as ReferenceTasksetConfig -from strict_exact_v1 import TangleRuntimeHarness, TangleTaskset -from strict_exact_v1.harness import TangleRuntimeHarnessConfig -from strict_exact_v1.taskset import TangleTasksetConfig +sys.path.insert(0, str(ROOT / "reference")) +sys.path.insert(0, str(ROOT / "command")) +from command import TangleTaskset as CommandTaskset +from command.taskset import TangleTasksetConfig as CommandTasksetConfig +from reference import TangleTaskset as ReferenceTaskset +from reference.taskset import TangleTasksetConfig as ReferenceTasksetConfig +from strict_exact import TangleRuntimeHarness, TangleTaskset +from strict_exact.harness import TangleRuntimeHarnessConfig +from strict_exact.taskset import TangleTasksetConfig -train = TangleTaskset(TangleTasksetConfig(id="strict-exact-v1", split="train")).load() -heldout = TangleTaskset(TangleTasksetConfig(id="strict-exact-v1", split="eval")).load() +train = TangleTaskset(TangleTasksetConfig(id="strict-exact", split="train")).load() +heldout = TangleTaskset(TangleTasksetConfig(id="strict-exact", split="eval")).load() assert [task.data.name for task in train] == ["train"] assert [task.data.name for task in heldout] == ["eval"] @@ -103,7 +103,7 @@ assert asyncio.run(task.task_reward(ScoreTrace("No, but the policy allows it"))) assert asyncio.run(task.task_reward(ScoreTrace("Yes, it is allowed"))) == 0.0 reference_task = ReferenceTaskset( - ReferenceTasksetConfig(id="reference-v1", split="eval") + ReferenceTasksetConfig(id="reference", split="eval") ).load()[0] reference_trace = SimpleNamespace(last_reply="", transcript="") assert asyncio.run(reference_task.task_reward(reference_trace)) == 0.0 @@ -117,7 +117,7 @@ class CommandTrace: self.recorded.update(metrics) command_task = CommandTaskset( - CommandTasksetConfig(id="command-v1", split="eval") + CommandTasksetConfig(id="command", split="eval") ).load()[0] command_trace = CommandTrace() assert asyncio.run(command_task.task_reward(command_trace)) == 0.25 @@ -136,7 +136,7 @@ class Runtime: return vf.ProgramResult(exit_code=0, stdout="", stderr="") runtime = Runtime() -harness = TangleRuntimeHarness(TangleRuntimeHarnessConfig(id="strict-exact-v1")) +harness = TangleRuntimeHarness(TangleRuntimeHarnessConfig(id="strict-exact")) asyncio.run(harness.setup(runtime)) asyncio.run(harness.launch( SimpleNamespace(model="openai/gpt-5.4-20260601"), diff --git a/src/primeintellect/package.ts b/src/primeintellect/package.ts index 9e5a6122..1acc6469 100644 --- a/src/primeintellect/package.ts +++ b/src/primeintellect/package.ts @@ -25,7 +25,7 @@ export interface WritePrimeIntellectPackageOptions { replace?: boolean } -/** Build a complete PrimeIntellect Verifiers v1 package without writing to disk. */ +/** Build a complete PrimeIntellect Verifiers package without writing to disk. */ export function createPrimeIntellectPackage( options: PrimeIntellectPackageOptions, ): PrimeIntellectPackageBundle { @@ -64,7 +64,7 @@ export function createPrimeIntellectPackage( .map(([path, contents]) => [path, sha256(contents)]), ) const manifest: PrimeIntellectPackageManifest = { - schema: 'tangle.primeintellect.package/v1', + kind: 'tangle.primeintellect.package', name: validated.name, moduleName, version: validated.version, @@ -328,7 +328,7 @@ function renderInit(moduleName: string): string { function renderTaskset(moduleName: string, scoring: PrimeIntellectScoring): string { const config = scoringConfig(scoring) - return `import asyncio\nimport json\nimport math\nimport os\nfrom importlib.resources import files\nfrom pathlib import Path\nfrom typing import Any, Literal\n\nimport verifiers.v1 as vf\n\nSCORING = json.loads(${pythonString(JSON.stringify(config))})\nPACKAGE_ROOT = Path(__file__).resolve().parent\n\n\nclass TangleTaskData(vf.TaskData):\n split: Literal["train", "eval"]\n answer: str | list[str] | None = None\n metadata: dict[str, Any] = {}\n\n\nclass TangleTaskConfig(vf.TaskConfig):\n scoring: Literal["exact", "reference-judge", "command"] = SCORING["kind"]\n normalization: Literal["none", "trim", "trim-casefold"] = SCORING.get("normalization", "trim")\n judge_model: str = SCORING.get("model", "openai/gpt-5.4-nano")\n judge_prompt: str | None = SCORING.get("prompt")\n judge_view: Literal["last_reply", "full_trace"] = SCORING.get("view", "last_reply")\n score_program: list[str] = SCORING.get("command", [])\n score_forward_env: list[str] = SCORING.get("forwardEnv", [])\n score_timeout_seconds: float = SCORING.get("timeoutSeconds", 300)\n\n\ndef _normalize(value: str, mode: str) -> str:\n if mode == "none":\n return value\n value = value.strip()\n return value.casefold() if mode == "trim-casefold" else value\n\n\nasync def _run_score_command(config: TangleTaskConfig, data: TangleTaskData, trace: vf.Trace) -> float:\n if not config.score_program:\n raise ValueError("command scoring requires score_program")\n safe_env = {\n key: os.environ[key]\n for key in ("PATH", "HOME", "TMPDIR", "LANG")\n if key in os.environ\n }\n safe_env.update(\n {key: os.environ[key] for key in config.score_forward_env if key in os.environ}\n )\n request = {\n "protocol": "tangle.primeintellect.score/v1",\n "task": data.model_dump(mode="json", exclude_none=True),\n "trace": trace.model_dump(mode="json", exclude_none=True),\n }\n process = await asyncio.create_subprocess_exec(\n *config.score_program,\n cwd=PACKAGE_ROOT,\n env=safe_env,\n stdin=asyncio.subprocess.PIPE,\n stdout=asyncio.subprocess.PIPE,\n stderr=asyncio.subprocess.PIPE,\n )\n payload = json.dumps(request, separators=(",", ":")).encode()\n try:\n stdout, stderr = await asyncio.wait_for(\n process.communicate(payload), timeout=config.score_timeout_seconds\n )\n except TimeoutError:\n process.kill()\n await process.communicate()\n raise RuntimeError(\n f"score command timed out after {config.score_timeout_seconds}s"\n )\n if process.returncode != 0:\n detail = (stderr or stdout).decode(errors="replace").strip()[-2000:]\n raise RuntimeError(f"score command exited {process.returncode}: {detail}")\n try:\n result = json.loads(stdout)\n except json.JSONDecodeError as error:\n raise ValueError(f"score command returned invalid JSON: {error}") from error\n if not isinstance(result, dict):\n raise ValueError("score command must return an object")\n reward = result.get("reward")\n if isinstance(reward, bool) or not isinstance(reward, (int, float)) or not math.isfinite(reward):\n raise ValueError("score command reward must be a finite number")\n metrics = result.get("metrics", {})\n if not isinstance(metrics, dict):\n raise ValueError("score command metrics must be an object")\n for name, value in metrics.items():\n if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):\n raise ValueError(f"score command metric {name!r} must be a finite number")\n trace.record_metrics(metrics)\n return float(reward)\n\n\nclass TangleTask(vf.Task[TangleTaskData, vf.State, TangleTaskConfig]):\n @vf.reward(weight=1.0)\n async def task_reward(self, trace: vf.Trace) -> float:\n if self.config.scoring == "command":\n return await _run_score_command(self.config, self.data, trace)\n if self.data.answer is None:\n raise ValueError(f"task {self.data.name!r} has no reference answer")\n if self.config.scoring == "reference-judge":\n judge = vf.ReferenceJudge(\n vf.ReferenceJudgeConfig(\n model=self.config.judge_model,\n prompt=self.config.judge_prompt,\n view=self.config.judge_view,\n )\n )\n return await judge.score(self.data, trace)\n expected = self.data.answer if isinstance(self.data.answer, list) else [self.data.answer]\n actual = _normalize(trace.last_reply, self.config.normalization)\n return float(any(actual == _normalize(answer, self.config.normalization) for answer in expected))\n\n\nclass TangleTasksetConfig(vf.TasksetConfig):\n split: Literal["train", "eval"] = "eval"\n task: TangleTaskConfig = TangleTaskConfig()\n\n\nclass TangleTaskset(vf.Taskset[TangleTask, TangleTasksetConfig]):\n def load(self) -> list[TangleTask]:\n resource = files(${pythonString(moduleName)}).joinpath("tasks.jsonl")\n tasks: list[TangleTask] = []\n with resource.open("r", encoding="utf-8") as handle:\n for line in handle:\n if not line.strip():\n continue\n row = json.loads(line)\n if row["split"] != self.config.split:\n continue\n tasks.append(TangleTask(TangleTaskData.model_validate(row), self.config.task))\n if not tasks:\n raise ValueError(f"task split {self.config.split!r} is empty")\n return tasks\n\n\n__all__ = ["TangleTaskset"]\n` + return `import asyncio\nimport json\nimport math\nimport os\nfrom importlib.resources import files\nfrom pathlib import Path\nfrom typing import Any, Literal\n\nimport verifiers.v1 as vf\n\nSCORING = json.loads(${pythonString(JSON.stringify(config))})\nPACKAGE_ROOT = Path(__file__).resolve().parent\n\n\nclass TangleTaskData(vf.TaskData):\n split: Literal["train", "eval"]\n answer: str | list[str] | None = None\n metadata: dict[str, Any] = {}\n\n\nclass TangleTaskConfig(vf.TaskConfig):\n scoring: Literal["exact", "reference-judge", "command"] = SCORING["kind"]\n normalization: Literal["none", "trim", "trim-casefold"] = SCORING.get("normalization", "trim")\n judge_model: str = SCORING.get("model", "openai/gpt-5.4-nano")\n judge_prompt: str | None = SCORING.get("prompt")\n judge_view: Literal["last_reply", "full_trace"] = SCORING.get("view", "last_reply")\n score_program: list[str] = SCORING.get("command", [])\n score_forward_env: list[str] = SCORING.get("forwardEnv", [])\n score_timeout_seconds: float = SCORING.get("timeoutSeconds", 300)\n\n\ndef _normalize(value: str, mode: str) -> str:\n if mode == "none":\n return value\n value = value.strip()\n return value.casefold() if mode == "trim-casefold" else value\n\n\nasync def _run_score_command(config: TangleTaskConfig, data: TangleTaskData, trace: vf.Trace) -> float:\n if not config.score_program:\n raise ValueError("command scoring requires score_program")\n safe_env = {\n key: os.environ[key]\n for key in ("PATH", "HOME", "TMPDIR", "LANG")\n if key in os.environ\n }\n safe_env.update(\n {key: os.environ[key] for key in config.score_forward_env if key in os.environ}\n )\n request = {\n "kind": "tangle.primeintellect.score",\n "task": data.model_dump(mode="json", exclude_none=True),\n "trace": trace.model_dump(mode="json", exclude_none=True),\n }\n process = await asyncio.create_subprocess_exec(\n *config.score_program,\n cwd=PACKAGE_ROOT,\n env=safe_env,\n stdin=asyncio.subprocess.PIPE,\n stdout=asyncio.subprocess.PIPE,\n stderr=asyncio.subprocess.PIPE,\n )\n payload = json.dumps(request, separators=(",", ":")).encode()\n try:\n stdout, stderr = await asyncio.wait_for(\n process.communicate(payload), timeout=config.score_timeout_seconds\n )\n except TimeoutError:\n process.kill()\n await process.communicate()\n raise RuntimeError(\n f"score command timed out after {config.score_timeout_seconds}s"\n )\n if process.returncode != 0:\n detail = (stderr or stdout).decode(errors="replace").strip()[-2000:]\n raise RuntimeError(f"score command exited {process.returncode}: {detail}")\n try:\n result = json.loads(stdout)\n except json.JSONDecodeError as error:\n raise ValueError(f"score command returned invalid JSON: {error}") from error\n if not isinstance(result, dict):\n raise ValueError("score command must return an object")\n reward = result.get("reward")\n if isinstance(reward, bool) or not isinstance(reward, (int, float)) or not math.isfinite(reward):\n raise ValueError("score command reward must be a finite number")\n metrics = result.get("metrics", {})\n if not isinstance(metrics, dict):\n raise ValueError("score command metrics must be an object")\n for name, value in metrics.items():\n if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):\n raise ValueError(f"score command metric {name!r} must be a finite number")\n trace.record_metrics(metrics)\n return float(reward)\n\n\nclass TangleTask(vf.Task[TangleTaskData, vf.State, TangleTaskConfig]):\n @vf.reward(weight=1.0)\n async def task_reward(self, trace: vf.Trace) -> float:\n if self.config.scoring == "command":\n return await _run_score_command(self.config, self.data, trace)\n if self.data.answer is None:\n raise ValueError(f"task {self.data.name!r} has no reference answer")\n if self.config.scoring == "reference-judge":\n judge = vf.ReferenceJudge(\n vf.ReferenceJudgeConfig(\n model=self.config.judge_model,\n prompt=self.config.judge_prompt,\n view=self.config.judge_view,\n )\n )\n return await judge.score(self.data, trace)\n expected = self.data.answer if isinstance(self.data.answer, list) else [self.data.answer]\n actual = _normalize(trace.last_reply, self.config.normalization)\n return float(any(actual == _normalize(answer, self.config.normalization) for answer in expected))\n\n\nclass TangleTasksetConfig(vf.TasksetConfig):\n split: Literal["train", "eval"] = "eval"\n task: TangleTaskConfig = TangleTaskConfig()\n\n\nclass TangleTaskset(vf.Taskset[TangleTask, TangleTasksetConfig]):\n def load(self) -> list[TangleTask]:\n resource = files(${pythonString(moduleName)}).joinpath("tasks.jsonl")\n tasks: list[TangleTask] = []\n with resource.open("r", encoding="utf-8") as handle:\n for line in handle:\n if not line.strip():\n continue\n row = json.loads(line)\n if row["split"] != self.config.split:\n continue\n tasks.append(TangleTask(TangleTaskData.model_validate(row), self.config.task))\n if not tasks:\n raise ValueError(f"task split {self.config.split!r} is empty")\n return tasks\n\n\n__all__ = ["TangleTaskset"]\n` } function renderHarness(moduleName: string): string { @@ -356,7 +356,7 @@ function scoringConfig(scoring: PrimeIntellectScoring): Record\n\`\`\`\n\n## Train\n\nUse \`prime.train.toml\` as the environment config for Prime RL. The train and eval rows are disjoint and selected by \`taskset.split\`.\n\nThe runner receives only the prompt, metadata, intercepted model endpoint, and MCP URLs. Reference answers remain in the task process and are never written into the runner workspace or environment.\n` + return `# ${options.name}\n\nPrimeIntellect Verifiers tasks that run the caller's Tangle agent program.\n\n## Evaluate\n\n\`\`\`bash\nuv run eval @ prime.eval.toml --model \n\`\`\`\n\n## Train\n\nUse \`prime.train.toml\` as the environment config for Prime RL. The train and eval rows are disjoint and selected by \`taskset.split\`.\n\nThe runner receives only the prompt, metadata, intercepted model endpoint, and MCP URLs. Reference answers remain in the task process and are never written into the runner workspace or environment.\n` } function toml(value: string): string { @@ -402,9 +402,9 @@ async function assertGeneratedPackage(output: string): Promise { if ( manifest === null || typeof manifest !== 'object' || - (manifest as Record).schema !== 'tangle.primeintellect.package/v1' + (manifest as Record).kind !== 'tangle.primeintellect.package' ) { - throw new Error(`refusing to replace ${output}: manifest schema does not match`) + throw new Error(`refusing to replace ${output}: manifest kind does not match`) } } diff --git a/src/primeintellect/primeintellect.test.ts b/src/primeintellect/primeintellect.test.ts index ff893d15..cef0b52c 100644 --- a/src/primeintellect/primeintellect.test.ts +++ b/src/primeintellect/primeintellect.test.ts @@ -18,7 +18,7 @@ import type { PrimeIntellectPackageOptions } from './types' function packageOptions(): PrimeIntellectPackageOptions { return { - name: 'support-agent-v1', + name: 'support-agent', version: '1.0.0', tasks: [ { @@ -44,18 +44,19 @@ function packageOptions(): PrimeIntellectPackageOptions { } } -describe('PrimeIntellect v1 package', () => { +describe('PrimeIntellect package', () => { it('builds a split-safe package around the caller runtime program', () => { const bundle = createPrimeIntellectPackage(packageOptions()) expect(bundle.manifest.splits).toEqual({ train: 1, eval: 1 }) + expect(bundle.manifest.kind).toBe('tangle.primeintellect.package') expect(bundle.manifest.verifiers).toBe('>=0.2.0,<0.3.0') expect(bundle.files['prime.eval.toml']).toContain('max_turns = 16') expect(bundle.files['prime.eval.toml']).toContain('type = "docker"') - expect(bundle.files['support_agent_v1/taskset.py']).toContain('import verifiers.v1 as vf') - expect(bundle.files['support_agent_v1/harness.py']).toContain('runtime.run_program') - expect(bundle.files['support_agent_v1/harness.py']).not.toContain('data.answer') - expect(bundle.files['support_agent_v1/harness.py']).not.toContain('reference') + expect(bundle.files['support_agent/taskset.py']).toContain('import verifiers.v1 as vf') + expect(bundle.files['support_agent/harness.py']).toContain('runtime.run_program') + expect(bundle.files['support_agent/harness.py']).not.toContain('data.answer') + expect(bundle.files['support_agent/harness.py']).not.toContain('reference') }) it('requires both splits and rejects duplicate ids and unsafe runner files', () => { diff --git a/src/primeintellect/runner.ts b/src/primeintellect/runner.ts index 2cc78960..5139d8af 100644 --- a/src/primeintellect/runner.ts +++ b/src/primeintellect/runner.ts @@ -35,7 +35,6 @@ export function readPrimeIntellectEpisodeContext( const mcpServers = validateStringMap(parsedMcp, ENV.mcpServers) return { - protocol: 'tangle.primeintellect.episode/v1', task, model: { name: requiredEnv(env, ENV.model), diff --git a/src/primeintellect/types.ts b/src/primeintellect/types.ts index f3df50c6..52a82a73 100644 --- a/src/primeintellect/types.ts +++ b/src/primeintellect/types.ts @@ -86,7 +86,7 @@ export interface PrimeIntellectPackageOptions { } export interface PrimeIntellectPackageManifest { - schema: 'tangle.primeintellect.package/v1' + kind: 'tangle.primeintellect.package' name: string moduleName: string version: string @@ -113,7 +113,6 @@ export interface PrimeIntellectPublicTask { } export interface PrimeIntellectEpisodeContext { - protocol: 'tangle.primeintellect.episode/v1' task: PrimeIntellectPublicTask model: { name: string diff --git a/src/primeintellect/validation.ts b/src/primeintellect/validation.ts index 53a60b26..b4ae88d8 100644 --- a/src/primeintellect/validation.ts +++ b/src/primeintellect/validation.ts @@ -1,6 +1,6 @@ import type { PrimeIntellectJson, PrimeIntellectMessage } from './types' -/** Validate the Verifiers v1 prompt schema shared by package creation and runner input. */ +/** Validate the PrimeIntellect prompt shape shared by package creation and runner input. */ export function validatePrimeIntellectPrompt( value: unknown, path: string, From 7b6ed908cb6163d34daff59ba7f133509c987dd6 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 24 Jul 2026 14:25:26 -0600 Subject: [PATCH 10/14] feat(optimization): prove official engines end to end --- .github/workflows/ci.yml | 23 + .github/workflows/publish.yml | 7 + README.md | 55 +- bench/package.json | 4 +- bench/src/swe-arena/gepa-seat.mts | 4 +- bench/src/swe-arena/outer-loop.mts | 3 +- bench/src/swe-code-improve.mts | 4 +- docs/agent-managed-compute/current-state.md | 9 +- docs/api/primitive-catalog.md | 19 +- docs/canonical-api.md | 21 +- package.json | 7 +- pnpm-lock.yaml | 71 +-- .../verify-official-optimizers-consumer.mjs | 496 ++++++++++++++++++ scripts/verify-official-optimizers.mjs | 273 ++++++++++ src/improvement/code-execution.ts | 1 + src/improvement/improve-types.ts | 2 + src/improvement/improve.test.ts | 40 +- src/improvement/improve.ts | 31 +- src/improvement/method-cost.test.ts | 111 ++++ src/improvement/method-cost.ts | 101 ++++ src/improvement/method-execution.ts | 63 +-- src/improvement/method-identity.test.ts | 160 ++++++ src/improvement/method-identity.ts | 113 ++++ src/improvement/official-optimizers.test.ts | 258 ++++++++- src/improvement/official-optimizers.ts | 219 ++++++-- src/improvement/official-packages.test.ts | 242 +++++++++ .../official-runtime-integration.test.ts | 261 +++++++++ src/improvement/official-test-support.ts | 69 +++ src/improvement/raw-trace-distiller.ts | 2 +- src/intelligence/improvement-cycle.ts | 27 +- src/intelligence/index.ts | 2 + src/intelligence/intelligence.test.ts | 5 + src/intelligence/optimization-receipt.ts | 288 ++++++++++ src/redact.ts | 16 +- tests/helpers/improvement-method-fixture.ts | 170 ++++++ tests/improvement-cycle.test.ts | 109 +--- tests/optimization-receipt.test.ts | 254 +++++++++ 37 files changed, 3215 insertions(+), 325 deletions(-) create mode 100644 scripts/verify-official-optimizers-consumer.mjs create mode 100644 scripts/verify-official-optimizers.mjs create mode 100644 src/improvement/method-cost.test.ts create mode 100644 src/improvement/method-cost.ts create mode 100644 src/improvement/method-identity.test.ts create mode 100644 src/improvement/method-identity.ts create mode 100644 src/improvement/official-packages.test.ts create mode 100644 src/improvement/official-runtime-integration.test.ts create mode 100644 src/improvement/official-test-support.ts create mode 100644 src/intelligence/optimization-receipt.ts create mode 100644 tests/helpers/improvement-method-fixture.ts create mode 100644 tests/optimization-receipt.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 48e1b5dc..6f6ed2b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,3 +61,26 @@ jobs: - name: Verify agent-bench against current agent-runtime run: pnpm run verify:bench + + official-optimizers: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install package set + run: pnpm install --frozen-lockfile + + - name: Verify packed Runtime with official GEPA and SkillOpt + run: pnpm run verify:official-optimizers diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d05fd6bc..b50feffa 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -22,6 +22,10 @@ jobs: cache: pnpm registry-url: https://registry.npmjs.org + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install deps run: pnpm install --frozen-lockfile @@ -40,6 +44,9 @@ jobs: - name: Verify packed package exports run: pnpm run verify:package + - name: Verify packed Runtime with official GEPA and SkillOpt + run: pnpm run verify:official-optimizers + - name: Verify agent-bench against this release run: pnpm run verify:bench diff --git a/README.md b/README.md index 8133c0b5..2a7cf33e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # @tangle-network/agent-runtime -A TypeScript runtime for running AI agents: as a **chat turn**, a **one-shot task**, or a **team of agents** working toward a goal: that records every run and uses those records to **measure and improve** agents against real pass/fail checks. It is the engine Tangle's own production agents run on. +A TypeScript runtime for chat agents, one-shot tasks, and agent teams. +It records each run so you can measure changes against real pass/fail checks and improve the agent without changing your product integration. Domain behavior (models, tools, knowledge) plugs in as adapters; the scoring statistics and the ship decision come from [`@tangle-network/agent-eval`](https://www.npmjs.com/package/@tangle-network/agent-eval); sandboxed execution from [`@tangle-network/sandbox`](https://www.npmjs.com/package/@tangle-network/sandbox). @@ -24,7 +25,10 @@ pnpm add @tangle-network/agent-runtime @tangle-network/agent-eval @tangle-networ ## Quickstart (offline, no API keys) -The core move everything else builds on: a driver runs a worker, reads the worker's real output, and writes the next prompt from it until a check passes. This is the full working loop from [`examples/quickstart/quickstart.ts`](./examples/quickstart/quickstart.ts) (the worker is a scripted stand-in so it runs with zero credentials; swap it for a real sandbox, CLI-harness, or router backend without changing the driver): +A driver runs a worker, reads its output, and writes the next prompt until a check passes. +This excerpt shows the driver from the runnable [`examples/quickstart/quickstart.ts`](./examples/quickstart/quickstart.ts). +That file defines the scripted `worker`, `output`, and `validator` used below so it runs without credentials. +Replace the scripted worker with a sandbox, CLI bridge, or router backend without changing the driver. ```ts import { inProcessSandboxClient, runLoop } from '@tangle-network/agent-runtime/loops' @@ -61,7 +65,7 @@ shot 1: PASS: "Shipped one-click restore with an instant rollback path." decision: pick-winner: winner: shot 1 ``` -The fully annotated version of this loop, with every seam explained, is [`examples/driver-loop`](./examples/driver-loop). +The annotated version is [`examples/driver-loop`](./examples/driver-loop). ## What you do with it @@ -184,19 +188,25 @@ There is no local fallback. Install its optional Python process before using it: ```bash -python -m pip install agent-eval-rpc +python -m pip install "agent-eval-rpc==0.126.5" +python -m pip install "gepa[full]==0.1.4" +``` + +The published GEPA 0.1.4 wheel supports the direct `gepa` engine. +Sequential, adaptive, best-of, vote, Omni, AutoResearch, Meta Harness, and Best-of-N require the tested official source revision: + +```bash python -m pip install "gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f" ``` Use `officialSkillOpt(...)` for Microsoft's SkillOpt: ```bash -python -m pip install agent-eval-rpc +python -m pip install "agent-eval-rpc==0.126.5" python -m pip install "skillopt @ git+https://github.com/microsoft/SkillOpt.git@61735e3922efc2b90c6d6cab561e62e98452ca90" ``` -The source pins are deliberate. -The published GEPA wheel lacks Optimize Anything, and the published SkillOpt wheel lacks files required by `ReflACTTrainer`. +SkillOpt 0.2.0's published wheel omits prompt files required by `ReflACTTrainer`, so the tested SkillOpt source revision remains necessary. SkillOpt and GEPA's standard reflection engine require `optimizer: { model, baseUrl, apiKey, budget }`. Agent-based GEPA engines may own their model connection instead. Agent Eval proxies those model calls, enforces the nested budget, and records their cost. @@ -218,15 +228,24 @@ To optimize several named profile fields together, also provide `profileComponen Tools, MCP, hooks, subagents, curated instructions, and rollout policy are also exact profile coordinates. Runtime does not choose an optimizer for them. -Official optimizer factories reject a selected profile surface containing common live credentials or private values. -They redact common private values from findings, described scenarios, described artifacts, and judge notes before sending those values to Python. -`describeScenario` still controls the exact train and selection fields sent to the external optimizer, so return only approved fields. +Without `describeScenario`, the external optimizer receives only each development case ID. +Without `describeArtifact`, evaluation feedback contains no artifact body. +When either descriptor is present, its result passes through `redact` together with findings, background text, profile name, and judge notes. +The built-in redactor removes common credentials and email addresses. +Supply a domain redactor for customer names, account IDs, or other private data the built-in rules cannot identify. +Runtime applies that hook first and then still applies its built-in scrubber. +Set `redact: false` only when every outbound value is public and already reviewed. + +The selected profile surface is the optimizer's candidate and cannot be redacted without changing the measured candidate. +Runtime always rejects recognized credentials in those bytes. +It also rejects structurally sensitive fields such as MCP env, headers, URLs, metadata, and extensions unless `approveSensitiveProfileSurface: true`. +That approval asserts the exact candidate contains only public values or safe references. Code is the exception. It uses Runtime's isolated git worktrees and coding-agent candidate execution: ```ts -const result = await improve(baseProfile, { +const result = await improve({ surface: 'code', code: { repoRoot, baseRef, generator }, scenarios, @@ -296,7 +315,11 @@ Runtime owns candidate identity, measurement, review binding, expiry, retry iden ### Improve a knowledge base -`runKnowledgeImprovementJob` is the runtime-owned front door for KB, wiki, memory-backed, and RAG improvement jobs. It creates a candidate copy, runs supervised agents against it, checks readiness through `@tangle-network/agent-knowledge`, and returns frozen baseline and candidate snapshots with spend and timing. It never changes the live knowledge base. Use `improve(..., { surface: 'memory' })` for the agent's curated lesson document; use this job for source, retrieval, and knowledge-store changes. +`runKnowledgeImprovementJob` runs KB, wiki, memory-backed, and RAG improvement jobs. +It creates a candidate copy, runs agents against it, checks it through `@tangle-network/agent-knowledge`, and returns frozen baseline and candidate snapshots with spend and timing. +It never changes the live knowledge base. +Use `improve(profile, { surface: 'memory', ... })` for the agent's curated lesson document. +Use this job for source, retrieval, and knowledge-store changes. ```ts import { runKnowledgeImprovementJob } from '@tangle-network/agent-runtime/knowledge' @@ -378,10 +401,10 @@ Use `importPrimeIntellectTraces(...)` to convert them to agent-eval `RunRecord`s ## How it works (the short version) -- **One agent, run two ways.** The same agent runs at "do the task" speed and at "get better at the task" speed. "Driver", "worker", and "coordinator" are roles one agent plays, not separate types. -- **Everything is measured.** Every run is a trace: tokens, dollars, time, and a pass/fail score from a real check. "Better" is a number with a denominator, not a vibe, and "equally good but cheaper" is a result you can prove. -- **Improvement is gated.** A change ships only after it beats the current agent on fresh tasks no tuning step ever saw, with a statistical test, not a single lucky run. -- **The grader is honest.** Whatever gives feedback never sees the answer key, and scores are recomputed from the attempts actually run. An agent cannot fabricate its own win. +- **Roles are configuration.** Driver, worker, and coordinator describe what an agent does in a run. They are not separate agent types. +- **Runs are recorded.** A run can report tokens, dollars, time, outputs, and scores. +- **Candidates face fresh tasks.** The optimizer uses train and selection tasks. Promotion uses a separate final set. +- **Scores come from executed attempts.** Runtime recomputes results from the recorded cells and rejects incomplete cost or source evidence. ## Primitives diff --git a/bench/package.json b/bench/package.json index f770b282..9aa48380 100644 --- a/bench/package.json +++ b/bench/package.json @@ -40,9 +40,9 @@ "verify:pier": "tsx scripts/verify-pier-pair.mts" }, "dependencies": { - "@tangle-network/agent-eval": "0.126.1", + "@tangle-network/agent-eval": "0.126.5", "@tangle-network/agent-interface": "0.32.0", - "@tangle-network/agent-knowledge": "^4.1.0", + "@tangle-network/agent-knowledge": "5.0.0", "@tangle-network/agent-runtime": "workspace:*", "@tangle-network/sandbox": "^0.12.0" }, diff --git a/bench/src/swe-arena/gepa-seat.mts b/bench/src/swe-arena/gepa-seat.mts index fb78c05c..82479d64 100644 --- a/bench/src/swe-arena/gepa-seat.mts +++ b/bench/src/swe-arena/gepa-seat.mts @@ -228,8 +228,8 @@ export function innerSmokeJudge(): JudgeConfig { // --------------------------------------------------------------------------- export const GEPA_PYTHON_INSTALL_HINT = - 'install the optional Python bridge with `python -m pip install agent-eval-rpc`, ' + - "then install the GEPA source revision pinned in agent-eval's Python client README" + 'install `agent-eval-rpc==0.126.5`, then install ' + + '`gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f`' export type GepaMethodFactory = ( config: GepaOptimizationMethodConfig, diff --git a/bench/src/swe-arena/outer-loop.mts b/bench/src/swe-arena/outer-loop.mts index 8cd4bfd5..d2590c44 100644 --- a/bench/src/swe-arena/outer-loop.mts +++ b/bench/src/swe-arena/outer-loop.mts @@ -2047,10 +2047,9 @@ export async function runRound(config: OuterLoopConfig, signal?: AbortSignal): P // ── the improve() call: the optimizer seat ─────────────────────────── // Typed from improve()'s own parameter: the monorepo hoists two // agent-interface majors, so a nominal import can resolve to the wrong one. - const profile = { name: 'loops-pi-supervisor' } as Parameters[0] signal?.throwIfAborted() log(`round ${config.round} runId=${runId}: improve(surface:'code') over ${config.loopsRepo}@${config.loopsBaseRef}`) - const result = await improve(profile, { + const result = await improve({ surface: 'code', // analyzeGeneration wins over this flag; the composite above embeds // rawTraceDistiller directly so the raw-trace mechanism stays active. diff --git a/bench/src/swe-code-improve.mts b/bench/src/swe-code-improve.mts index 2fb707e5..8d7ae100 100644 --- a/bench/src/swe-code-improve.mts +++ b/bench/src/swe-code-improve.mts @@ -33,7 +33,6 @@ import { spawn, spawnSync } from 'node:child_process' import { existsSync, mkdirSync, symlinkSync } from 'node:fs' import { join } from 'node:path' import { improve, agenticGenerator } from '@tangle-network/agent-runtime' -import type { AgentProfile } from '@tangle-network/agent-interface' import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract' import { createSweBenchAdapter } from './benchmarks/swe-bench' import type { BenchTask } from './benchmarks/types' @@ -295,11 +294,10 @@ async function main(): Promise { runHarness: runHarness as any, }) - const profile: AgentProfile = { name: 'swe-scaffold', prompt: { systemPrompt: '' } } const scenarios: Scenario[] = allIds.map((id) => ({ id, kind: 'swe-bench-verified' })) const holdoutScenarios: Scenario[] = holdoutIds.map((id) => ({ id, kind: 'swe-bench-verified' })) - const out = await improve(profile, { + const out = await improve({ surface: 'code', gate: 'holdout', code: { repoRoot: REPO_ROOT, baseRef, worktreeDir, generator }, diff --git a/docs/agent-managed-compute/current-state.md b/docs/agent-managed-compute/current-state.md index 255cdf94..af2fdbe5 100644 --- a/docs/agent-managed-compute/current-state.md +++ b/docs/agent-managed-compute/current-state.md @@ -1,4 +1,7 @@ -# Current State Audit +# Agent-Managed Compute Audit (2026-07-18) + +> This is a dated audit record, not the current package manifest. +> Check `package.json` and the generated API docs for the release in your checkout. ## Tested Baseline @@ -197,13 +200,13 @@ There is no automated test that kills a coordinator, reconnects to a real remote ### 13. Package layering and direct release lines are aligned -The current `agent-knowledge` main branch has no dependency or source import from `agent-runtime`. +At the time of this audit, the `agent-knowledge` main branch had no dependency or source import from `agent-runtime`. `agent-runtime` now owns the batteries-included composition through `src/knowledge/improvement-job.ts`. That is the intended dependency direction and removes the former package cycle. -The runtime manifest requires `agent-knowledge` `^4.1.0`, develops against `agent-eval` 0.122.8, and exposes matching peer ranges for `agent-eval`, `agent-interface`, and `sandbox`. +The audited runtime manifest required `agent-knowledge` `^4.1.0`, developed against `agent-eval` 0.122.8, and exposed matching peer ranges for `agent-eval`, `agent-interface`, and `sandbox`. Older copies may still appear inside transitive dependencies that own their contracts internally. The direct runtime edges share the current public contract versions. diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 18eac659..c99d2b6e 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.104.0` and `@tangle-network/agent-eval@0.126.1` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.104.0` and `@tangle-network/agent-eval@0.126.5` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -66,7 +66,7 @@ Import from `@tangle-network/agent-runtime` — 386 exports. | `formatSupervisedKnowledgeTask` | function | Format the supervisor task with the KB root, readiness requirements, current findings, and metadata. | | `getModels` | function | Fetch the model catalog from the router's `/v1/models`. Throws on a non-2xx | | `handleChatTurn` | function | Run one chat turn. Returns immediately with a `ReadableStream` body; | -| `improve` | function | Optimize one exact profile surface with a complete method, or optimize code | +| `improve` | function | Optimize one exact profile surface with a complete method. | | `isAnalystFinding` | function | Structural guard for the schema-versioned `AnalystFinding` envelope. | | `isDelegatedLoopMode` | function | Type guard — returns true when `value` is a valid `DelegatedLoopMode` string. | | `isDepthExceeded` | function | Refuse further forwarding when the inbound depth has reached the limit. | @@ -333,7 +333,7 @@ Import from `@tangle-network/agent-runtime/conversation` — 53 exports. ### Intelligence SDK — Observe + provable-OFF billing -Import from `@tangle-network/agent-runtime/intelligence` — 141 exports. +Import from `@tangle-network/agent-runtime/intelligence` — 143 exports. | Symbol | Kind | Summary | |---|---|---| @@ -360,13 +360,14 @@ Import from `@tangle-network/agent-runtime/intelligence` — 141 exports. | `isIntelligenceOff` | function | True when these settings admit NO intelligence spawn — the passthrough | | `manifestFromProfile` | function | Lower the EXISTING plane wire (`CertifiedProfile`) into a `CapabilityManifest`. | | `normalizeCertifiedProfile` | function | Deserialize the composed-endpoint response into a `CertifiedProfile`. The | +| `optimizationActivationReceiptFromMetadata` | function | Read and verify the optimizer evidence carried by a measured proposal. | | `parseCandidateProfileMaterialization` | function | Parse and check every native file hash plus both canonical document digests. | | `prepareAgentImprovementProfileActivation` | function | Compare product-owned profiles with an exact measured transition and prepare | | `proposeAgentImprovement` | function | Analyze, search, then remeasure the resulting exact candidate before proposing it. | | `pullCertified` | function | Pull the certified composed profile for a target. Fail-closed: a network | | `resolveEffort` | function | Compile a named tier (plus optional per-field overrides) into the flat | | `resolveIntelligenceBaseUrl` | function | Resolve the ONE Intelligence base URL — the single knob both the send and | -| `resolveRedactor` | function | Resolve the redactor a client uses. A caller-supplied hook replaces the | +| `resolveRedactor` | function | Resolve the redactor a client uses. A caller-supplied hook handles | | `reviewAgentImprovementProposal` | function | Persist a human or tenant-policy decision bound to one exact proposal. | | `runAgentCandidateExperiment` | function | Execute both arms of one immutable experiment and derive its paired result. | | `submitAgentImprovementProposal` | function | Submit a completed Runtime proposal to Intelligence for product-side review. | @@ -447,7 +448,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 141 exports. | `SubmitAgentImprovementProposalOutcome` | type | Typed result for proposal submission. A successful result contains the | | `UsageClass` | type | Usage class for billing. Base-stream tokens bill `'inference'`; every | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentCandidateExperimentCellPlacement`, `AgentImprovementActivationResultStore`, `AgentImprovementActivationTargetPlan`, `AgentImprovementActivationTransitionInput`, `AgentImprovementProfileReplacement`, `AgentImprovementProposal`, `AgentImprovementTargetProfileDiffOptions`, `CreateAgentImprovementActivationOptions`, `CreateAgentImprovementActivationResultOptions`, `CreateAgentImprovementProposalOptions`, `CreateExactProcessCandidateExperimentExecutorOptions`, `ExactProcessCandidateExperimentExecution`, `ExactProcessCandidateExperimentExecutor`, `ExecuteAgentCandidateExperimentCellOptions`, `ExecuteAgentImprovementActivationInput`, `ExecuteAgentImprovementActivationOptions`, `ProposeAgentImprovementOptions`, `ProposeAgentImprovementResult`, `PullCertifiedOptions`, `ReviewAgentImprovementInput`, `RunAgentCandidateExperimentOptions`, `RunAgentCandidateExperimentResult`, `VerifyCandidateExecutionEvidenceOptions`, `AgentImprovementActivationIntent`, `AgentImprovementActivationOutcome`, `AgentImprovementActivationTargetIdentity`, `AgentImprovementProfileActivationPreparation`, `AgentImprovementProfileActivationTarget`, `AgentImprovementProfileSurface`, `AgentImprovementProfileTargetState`, `AgentImprovementProfileTargetTransition`, `AgentImprovementReviewDecision`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentCandidateExperimentCellPlacement`, `AgentImprovementActivationResultStore`, `AgentImprovementActivationTargetPlan`, `AgentImprovementActivationTransitionInput`, `AgentImprovementProfileReplacement`, `AgentImprovementProposal`, `AgentImprovementTargetProfileDiffOptions`, `CreateAgentImprovementActivationOptions`, `CreateAgentImprovementActivationResultOptions`, `CreateAgentImprovementProposalOptions`, `CreateExactProcessCandidateExperimentExecutorOptions`, `ExactProcessCandidateExperimentExecution`, `ExactProcessCandidateExperimentExecutor`, `ExecuteAgentCandidateExperimentCellOptions`, `ExecuteAgentImprovementActivationInput`, `ExecuteAgentImprovementActivationOptions`, `OptimizationActivationReceipt`, `ProposeAgentImprovementOptions`, `ProposeAgentImprovementResult`, `PullCertifiedOptions`, `ReviewAgentImprovementInput`, `RunAgentCandidateExperimentOptions`, `RunAgentCandidateExperimentResult`, `VerifyCandidateExecutionEvidenceOptions`, `AgentImprovementActivationIntent`, `AgentImprovementActivationOutcome`, `AgentImprovementActivationTargetIdentity`, `AgentImprovementProfileActivationPreparation`, `AgentImprovementProfileActivationTarget`, `AgentImprovementProfileSurface`, `AgentImprovementProfileTargetState`, `AgentImprovementProfileTargetTransition`, `AgentImprovementReviewDecision`. ### Recursive atom + loop kernel (alias of ./runtime) @@ -999,14 +1000,14 @@ Import from `@tangle-network/agent-runtime/platform` — 20 exports. **Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AuthorizeUrlOptions`, `CatalogResult`, `ConnectionHealth`, `ConnectionHealthResult`, `ExchangeCodeResult`, `ExecInput`, `MintTokenInput`, `MintTokenResult`, `PlatformHubStatus`, `StartAuthInput`, `StartAuthResult`. -### PrimeIntellect: Verifiers v1 package and trace adapter +### PrimeIntellect: Verifiers package and trace adapter Import from `@tangle-network/agent-runtime/primeintellect` — 27 exports. | Symbol | Kind | Summary | |---|---|---| | `createPrimeIntellectBackend` | function | Build the existing runtime backend against Prime's intercepted model endpoint. | -| `createPrimeIntellectPackage` | function | Build a complete PrimeIntellect Verifiers v1 package without writing to disk. | +| `createPrimeIntellectPackage` | function | Build a complete PrimeIntellect Verifiers package without writing to disk. | | `importPrimeIntellectTraces` | function | Convert all Prime traces to agent-eval RunRecords while retaining one shared run config. | | `parsePrimeIntellectTraces` | function | Parse Prime's durable `traces.jsonl` and reject malformed rows with a line number. | | `primeIntellectTraceToRunRecord` | function | Project one complete Prime trace into the common agent-eval analysis row. | @@ -1341,7 +1342,7 @@ Import from `@tangle-network/agent-eval` — 51 exports. ### CAMPAIGN — profile matrix, gates, improvement loop -Import from `@tangle-network/agent-eval/campaign` — 320 exports. +Import from `@tangle-network/agent-eval/campaign` — 322 exports. | Symbol | Kind | Summary | |---|---|---| @@ -1364,6 +1365,7 @@ Import from `@tangle-network/agent-eval/campaign` — 320 exports. | `classifyUngroundedLiterals` | function | Scan revised artifact text for single-quoted single-word literals (the | | `codeSurfaceIdentityMaterial` | function | Canonical, location-independent identity of a finalized code candidate. | | `compareOptimizationMethods` | function | Compare complete optimization methods on disjoint train, selection, and final test data. | +| `compareRankKeys` | function | Compare fixed-length lexicographic rank keys where each element is higher-is-better. | | `componentSurfaceIdentityMaterial` | function | _(no summary — add a TSDoc line at the declaration)_ | | `composeGate` | function | Compose gates — all must `ship` for the composite to `ship`. First | | `costFromLedgerSummary` | function | Keep the cost fields a custom optimization method must report. | @@ -1440,6 +1442,7 @@ Import from `@tangle-network/agent-eval/campaign` — 320 exports. | `AnalystArtifact` | interface | The analyst's output for one scenario — the artifact the judge scores. | | `AnalystScenario` | interface | A labeled trace scenario: a FIXED trace corpus plus the failure modes a | | `CampaignArtifactWriter` | interface | Scoped artifact writer — `write(path, content)` lands under | +| `CampaignCellFailureReceipt` | interface | Durable `/failure-receipt.json` written before a failed cell can | | `CampaignCostMeter` | interface | Cell-scoped paid-call entry point. The dispatch places every paid operation | | `CampaignScenarioIdentity` | interface | Redacted identity of a complete scenario payload retained in campaign results. | | `CampaignStorage` | interface | `CampaignStorage` — the filesystem seam `runCampaign` writes through | diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 48c54cd4..6b1638e4 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -1,8 +1,14 @@ # `@tangle-network/agent-runtime`: Canonical API Reference - - -> **Version 0.104.0.** The export inventory + per-symbol signatures live in the generated `docs/api/` reference: **`docs/api/primitive-catalog.md`** is the never-stale, grouped list of every primitive to reuse (own surface + the agent-eval judge / authenticity / verification / statistics / campaign / token-usage surfaces), with each one's import path and one-line summary read live from source; the per-module pages hold the full signatures. The pinned dependency is agent-eval `>=0.126.1 <0.127.0`; `@tangle-network/sandbox` materializes profiles into execution environments (peer `>=0.12.0 <1.0.0`). The neutral contract types (`AgentProfile`, `AgentProfileMcpServer`, `HarnessType`, `ReasoningEffort`, `Part`/`ToolPart`/`ToolState`, plus environment-provider types) are owned by **`@tangle-network/agent-interface`** (peer `>=0.32.0 <0.33.0`): the single source of truth. Shared evaluation primitives are re-exported through `@tangle-network/agent-eval/contract` or `/campaign`: the catalog's §2 shows exactly which subpath each lives under. + + +> **Version 0.104.0.** +> [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. +> Agent Eval must satisfy `>=0.126.5 <0.127.0`. +> Sandbox must satisfy `>=0.12.0 <1.0.0`. +> Portable profile and tool-part types come from `@tangle-network/agent-interface` `>=0.32.0 <0.33.0`. > > **`./loops` is the runtime barrel**: `package.json` maps it to `src/runtime/index.ts`. Everything below labelled `/loops` is the recursive-atom + loop-kernel surface. > @@ -32,7 +38,8 @@ Two substrates implement the same recursive-atom over the one `Executor` port an ## 1.5 The AgentProfile rule: author the profile, the substrate materializes it -**An agent IS its `AgentProfile`, and the profile is the WHOLE agent: not just a prompt.** The surface is `systemPrompt + skills + tools + mcp + subagents + hooks + permissions + memory/rag + model` (the `AgentProfile*` family in `@tangle-network/sandbox`, constructed via `defineAgentProfile`). **System prompt ≠ skills**: skills are separate, invokable how-tos the agent reads *when prompted to invoke them*; never concatenate a skill body into the system prompt. +An `AgentProfile` contains the agent's system prompt, skills, tools, MCP servers, subagents, hooks, permissions, memory, retrieval configuration, and model settings. +Skills remain separate resources that the runtime can invoke; do not concatenate them into the system prompt. **You change an agent's behavior by changing its PROFILE: never by writing orchestration code around it.** The behaviors we keep hand-rolling are profile properties: - **Self-verification** is a profile lever, three ways, all configuration and zero glue code: (1) *steered*: the prompt says "run the tests, read failures, fix, repeat"; (2) *process-defined*: its instructions make verify-after-every-change its standing process; or (3) a **post-finish hook** that auto-runs the check and feeds failures back. The harness runs that loop. **You do not write a per-round judge, a `while(!done)`, or a bash hill-climb.** @@ -66,7 +73,7 @@ A general "loop" primitive is the single most common modelling error in this rep | I want to… | Use (import) | Do NOT build | |---|---|---| -| **Just run a supervisor to a goal (one call, scaffolding defaulted)**: START HERE (running agents) | `supervise(profile, task, { budget, backend? })`: `/loops` | hand-wiring `createSupervisor().run` + `blobs`/`perWorker`/`journal`/`executors`; reaching for the lower-level run-verbs below before you need a specific counterparty | +| Run a supervisor toward a goal with default setup | `supervise(profile, task, { budget, backend? })`: `/loops` | hand-wiring `createSupervisor().run` + `blobs`/`perWorker`/`journal`/`executors`; reaching for lower-level calls before you need a specific counterparty | | **Supervise agents to solve a graded `AgenticSurface` task** (workers `runAgentic` the surface, settle on its own check, driver self-improves from the failing tests) | `superviseSurface(profile, task, { surface, worker })`: `/loops` | a worker-seam + a "self-improving supervisor" wrapper around `supervise()`; passing a custom `makeWorkerAgent` that runs `runAgentic` | | Run a profile through a topology shape over the keystone Supervisor, end-to-end | `runPersonified({ persona, shape, task, budget })`: `/loops` | a hand-rolled `createSupervisor().run` + seam-wiring helper | | Loop a worker over one evolving artifact, K rounds, stop-when-good | `loopUntil(seed, spec)` as the `shape`: `/loops` | a `while(!done){runWorker();decide()}` hand-loop or "multi-attempt refine driver" | @@ -80,7 +87,7 @@ A general "loop" primitive is the single most common modelling error in this rep | Adaptive tree search / progressive widening | `widen(spec)` + `flatWidenGate()`: `/loops` | a best-first/MCTS that reads child *scores* to expand (selector=judge); keep `flatWidenGate()` until your gate is proven | | Define the profile record for a personified run | `definePersona(input)`: `/loops` | a "profile-seam" / agent-config wrapper carrying model+prompt+tools+role | | Make a worker self-verify / iterate / audit | a **hook / process / skill on its authored `AgentProfile`**: §1.5 | a per-round judge, a `while(!done)` loop, or a bash hill-climb (it's a profile lever) | -| Run an authored profile on a real harness | author the `AgentProfile`, hand it to the **sandbox substrate**: `@tangle-network/sandbox` (`defineAgentProfile`) | a `profile → opencode.json` realizer or any harness-specific config writer | +| Run an authored profile with Claude Code, Codex, OpenCode, or another supported harness | author the `AgentProfile`; `@tangle-network/sandbox` (`defineAgentProfile`) materializes it for the selected harness | a harness-specific profile or config writer | | Have the supervisor design its workers | author a **full `AgentProfile`** per sub-task (prompt+skills+tools+mcp+hooks+subagents): `/loops` | author a bare `systemPrompt` string (a worker can't act on levers it has no levers for) | | Write a custom driver Agent and run it directly | `createSupervisor().run(root, task, opts)`: `/loops` | a bespoke orchestrator that spawns sub-agents and tallies cost (equal-compute claim breaks there) | | Run depth-vs-breadth (or a custom strategy) over a stateful tool domain | `runAgentic({ surface, task, mode\|strategy, budget })`: `/loops` | a hand-rolled `Supervisor.run` + journal/registry, or a depth/breadth loop | @@ -98,7 +105,7 @@ A general "loop" primitive is the single most common modelling error in this rep | Optimize one text surface with Microsoft SkillOpt | `officialSkillOpt({ trainer, optimizer, ... })`, passed as `improve(...).method` from root `.` | Runtime-owned SkillOpt search or a silent local fallback | | Improve one profile coordinate | `improve(profile, { surface, executionRef, method, trainScenarios, selectionScenarios, testScenarios, judges, agent, costCeiling })` from root `.`; `executionRef` binds saved work to executable behavior, `agent` receives the exact complete candidate profile, and `costCeiling` limits the whole run | an implicit per-surface optimizer, a method that sees final-test cases, an unmeasured profile mutation, or separate optimizer and final-test spend limits | | Compare complete optimization methods directly | `compareOptimizationMethods(...)` from `agent-eval/campaign` | comparing one method's training score to another method's final score | -| Improve repository code | `improve(profile, { surface: 'code', code, scenarios, judge, agent, budget })` from root `.` | passing code through a text optimizer or managing candidate worktrees in product code | +| Improve repository code | `improve({ surface: 'code', code, scenarios, judge, agent, budget })` from root `.` | passing code through a text optimizer or managing candidate worktrees in product code | | Decide ship/hold on a candidate (campaign context) | `defaultProductionGate({ holdoutScenarios, deltaThreshold })`; compose with `heldOutGate` / `composeGate`: `agent-eval/contract` | a raw `h1>h0` point comparison on the training set | | Decide ship/hold from a **`BenchmarkReport`** (per-task cells) | `promotionGate({ report, incumbent, candidate })`: `/loops` | comparing two strategies' mean scores directly; re-deriving the bootstrap | | Run the full multi-generation strategy flywheel + certify | `runStrategyEvolution(config)`: `/loops` | a bespoke gen0→author→gen1→holdout loop with hand-rolled champion selection | diff --git a/package.json b/package.json index b31169a0..36582c44 100644 --- a/package.json +++ b/package.json @@ -114,6 +114,7 @@ "verify:bench": "pnpm build && pnpm --filter @tangle-network/agent-bench run typecheck:public && pnpm --filter @tangle-network/agent-bench test && pnpm --filter @tangle-network/agent-bench run verify:package:local-runtime", "verify:bench:published": "pnpm build && pnpm --filter @tangle-network/agent-bench run typecheck:public && pnpm --filter @tangle-network/agent-bench test && pnpm --filter @tangle-network/agent-bench run verify:package", "verify:package": "pnpm run check:testing-fixture && pnpm run check:skills && node scripts/verify-package-exports.mjs", + "verify:official-optimizers": "node scripts/verify-official-optimizers.mjs", "verify:primeintellect": "pnpm build && node scripts/verify-primeintellect.mjs", "verify:primeintellect:live": "pnpm build && node scripts/verify-primeintellect-live.mjs", "docs:api": "pnpm run build && typedoc && node scripts/gen-primitive-catalog.mjs", @@ -122,7 +123,7 @@ }, "devDependencies": { "@biomejs/biome": "^2.4.15", - "@tangle-network/agent-eval": "0.126.1", + "@tangle-network/agent-eval": "0.126.5", "@tangle-network/agent-interface": "0.32.0", "@tangle-network/sandbox": "^0.12.0", "@types/node": "^25.9.3", @@ -154,7 +155,7 @@ "license": "MIT", "packageManager": "pnpm@10.28.0", "peerDependencies": { - "@tangle-network/agent-eval": ">=0.126.1 <0.127.0", + "@tangle-network/agent-eval": ">=0.126.5 <0.127.0", "@tangle-network/agent-interface": ">=0.32.0 <0.33.0", "@tangle-network/sandbox": ">=0.12.0 <1.0.0", "playwright": "^1.40.0" @@ -168,7 +169,7 @@ } }, "dependencies": { - "@tangle-network/agent-knowledge": "^4.1.0", + "@tangle-network/agent-knowledge": "5.0.0", "@tangle-network/agent-profile-materialize": "0.5.1", "tar-stream": "3.2.0" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 728e10c3..ef25cf7c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: dependencies: '@tangle-network/agent-knowledge': - specifier: ^4.1.0 - version: 4.1.0(typescript@5.9.3) + specifier: 5.0.0 + version: 5.0.0(typescript@5.9.3) '@tangle-network/agent-profile-materialize': specifier: 0.5.1 version: 0.5.1 @@ -22,8 +22,8 @@ importers: specifier: ^2.4.15 version: 2.4.15 '@tangle-network/agent-eval': - specifier: 0.126.1 - version: 0.126.1(typescript@5.9.3) + specifier: 0.126.5 + version: 0.126.5(typescript@5.9.3) '@tangle-network/agent-interface': specifier: 0.32.0 version: 0.32.0 @@ -61,14 +61,14 @@ importers: bench: dependencies: '@tangle-network/agent-eval': - specifier: 0.126.1 - version: 0.126.1(typescript@5.9.3) + specifier: 0.126.5 + version: 0.126.5(typescript@5.9.3) '@tangle-network/agent-interface': specifier: 0.32.0 version: 0.32.0 '@tangle-network/agent-knowledge': - specifier: ^4.1.0 - version: 4.1.0(typescript@5.9.3) + specifier: 5.0.0 + version: 5.0.0(typescript@5.9.3) '@tangle-network/agent-runtime': specifier: workspace:* version: link:.. @@ -680,22 +680,14 @@ packages: '@tangle-network/agent-core@0.3.8': resolution: {integrity: sha512-bZfVpdiFjXbcQwSxSQABdtXEmUuapWChCcXrHkP6fAnwqEy9hWEfeZLlMZUy+1XaOfpTaMLUOEAwYhXfN018wQ==} - '@tangle-network/agent-core@0.4.11': - resolution: {integrity: sha512-5B1IjrJ8xDR7w8Hv/MSk2ixul6NEJQ5Ftzo+z6l7ipeFpc0yaf1+Kkml4HDUEmGW/rqdrNpBf9/MpT7i/SY0PA==} - '@tangle-network/agent-core@0.4.19': resolution: {integrity: sha512-GzygwJ6mmY4tgkFonc4yq2vU+kJSTo6JuZCUNsPQgjZigZeSpWjX+3r+ZxEi38VqAzt79MEn43/Txu+ZflqazQ==} '@tangle-network/agent-core@0.4.20': resolution: {integrity: sha512-gJzZh5PqPJtWW6kMEfm3IP7CGibvHdVXM4uqCAuCy+Kvg9TlVVkzXqZPRt9gU44mjdw5hDGoVzZotFsPoHAlFw==} - '@tangle-network/agent-eval@0.122.8': - resolution: {integrity: sha512-7v0us+6zpcR+VKqt9olG7C7j4KlTctvwhlHgm2qHU+FBorEJFcJs6JDqWMw+3h7t9ba/07muIb0AfuvSsmJJ8w==} - engines: {node: '>=20'} - hasBin: true - - '@tangle-network/agent-eval@0.126.1': - resolution: {integrity: sha512-Tnrw12ecQF2d3IRU1mWSQFEq5wPCO/Xx11tQnF7bmxgAZjU3XgZGBBPoz9CH4YLndlnPre2ybzsslp6iZgQjyw==} + '@tangle-network/agent-eval@0.126.5': + resolution: {integrity: sha512-nXgG+ULn2LwRs86d6TaWLsjL4Y0SrznD9GuVhSTMr3vkeludzzqBZsBEnHd2Zo454M71TT1vqdEtbX+pzJaaKA==} engines: {node: '>=20'} hasBin: true @@ -705,9 +697,6 @@ packages: '@tangle-network/agent-interface@0.17.1': resolution: {integrity: sha512-B7dRJTo0HSUtgBCB1VMwkTFYkLUaRr/4BcRglrQuGhGUwOzKv1RYyMejOVh5M3a5AagY9N79f7GYbjcA3UmnIA==} - '@tangle-network/agent-interface@0.26.0': - resolution: {integrity: sha512-/z4HavFr/9AbaHxi/13bFP9iSUt8oitZbw4wS9g9KAt2TeN2ec5/A2C106udWkKIETZLnu465TfU+K4KDVNkKw==} - '@tangle-network/agent-interface@0.30.0': resolution: {integrity: sha512-GmwPwamrzOFPn+Qx7W0nt9QRUF9VUiZrMNLmF6QpL23R+7TD21HllPblVOYB5MVqA728FpDLXit3HfqY8mStwg==} @@ -717,8 +706,8 @@ packages: '@tangle-network/agent-interface@0.32.0': resolution: {integrity: sha512-8GUiqdr9MZ+iedkx7KVL6jbiuW6jh7CzKPS3VDXbWherJjFbuf+ULsMKmc+wnfG7LvtjJgZGr5y+cZm+JN1YDA==} - '@tangle-network/agent-knowledge@4.1.0': - resolution: {integrity: sha512-MeCoB9ilojavryzsM0BGAdOJpvIEip77/maoiXuR+fAv2fZy7ZBamUEgbAewZkXDI0goxacM+SAyDMu6zw9oIw==} + '@tangle-network/agent-knowledge@5.0.0': + resolution: {integrity: sha512-VGq9UQdDZdPwwlrIVQEIr1xeSjubS5fIUdfjFDPNqDLCIajd6fsLY4OoPGOaTbr5hvXfkjKTvseANsiSnbA7bw==} engines: {node: '>=20.19.0'} hasBin: true @@ -1774,11 +1763,6 @@ snapshots: '@tangle-network/agent-interface': 0.17.1 zod: 4.4.3 - '@tangle-network/agent-core@0.4.11': - dependencies: - '@tangle-network/agent-interface': 0.26.0 - zod: 4.4.3 - '@tangle-network/agent-core@0.4.19': dependencies: '@tangle-network/agent-interface': 0.31.0 @@ -1787,27 +1771,9 @@ snapshots: '@tangle-network/agent-core@0.4.20': dependencies: '@tangle-network/agent-interface': 0.32.0 - - '@tangle-network/agent-eval@0.122.8(typescript@5.9.3)': - dependencies: - '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) - '@ax-llm/ax': 23.0.1(zod@4.4.3) - '@hono/node-server': 2.0.8(hono@4.12.30) - '@tangle-network/agent-core': 0.4.19 - '@tangle-network/agent-interface': 0.31.0 - '@tangle-network/tcloud': 0.4.14(typescript@5.9.3)(zod@4.4.3) - hono: 4.12.30 zod: 4.4.3 - transitivePeerDependencies: - - '@mastra/core' - - '@modelcontextprotocol/sdk' - - ai - - bufferutil - - openai - - typescript - - utf-8-validate - '@tangle-network/agent-eval@0.126.1(typescript@5.9.3)': + '@tangle-network/agent-eval@0.126.5(typescript@5.9.3)': dependencies: '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) '@ax-llm/ax': 23.0.1(zod@4.4.3) @@ -1834,11 +1800,6 @@ snapshots: dependencies: zod: 4.4.3 - '@tangle-network/agent-interface@0.26.0': - dependencies: - '@noble/hashes': 1.8.0 - zod: 4.4.3 - '@tangle-network/agent-interface@0.30.0': dependencies: '@noble/hashes': 1.8.0 @@ -1854,10 +1815,10 @@ snapshots: '@noble/hashes': 1.8.0 zod: 4.4.3 - '@tangle-network/agent-knowledge@4.1.0(typescript@5.9.3)': + '@tangle-network/agent-knowledge@5.0.0(typescript@5.9.3)': dependencies: - '@tangle-network/agent-eval': 0.122.8(typescript@5.9.3) - '@tangle-network/agent-interface': 0.31.0 + '@tangle-network/agent-eval': 0.126.5(typescript@5.9.3) + '@tangle-network/agent-interface': 0.32.0 proper-lockfile: 4.1.2 zod: 4.4.3 transitivePeerDependencies: diff --git a/scripts/verify-official-optimizers-consumer.mjs b/scripts/verify-official-optimizers-consumer.mjs new file mode 100644 index 00000000..94cc26aa --- /dev/null +++ b/scripts/verify-official-optimizers-consumer.mjs @@ -0,0 +1,496 @@ +import { createHash } from 'node:crypto' +import { mkdtemp, rm } from 'node:fs/promises' +import { createServer } from 'node:http' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + improve, + officialGepa, + officialSkillOpt, +} from '@tangle-network/agent-runtime' + +const python = process.env.AGENT_EVAL_TEST_PYTHON?.trim() +if (!python) throw new Error('AGENT_EVAL_TEST_PYTHON is required') + +async function main() { + const mode = process.argv[2] + if (mode === 'wheel') { + await runWheelVerification() + return + } + if (mode === 'omni') { + await runOmniVerification() + return + } + throw new Error(`expected verification mode "wheel" or "omni", found ${String(mode)}`) +} + +async function runWheelVerification() { + const gepaRunDir = await mkdtemp(join(tmpdir(), 'packed-runtime-gepa-')) + const concurrentGepaRunDir = await mkdtemp(join(tmpdir(), 'packed-runtime-gepa-concurrent-')) + const skillOptRunDir = await mkdtemp(join(tmpdir(), 'packed-runtime-skillopt-')) + const gepaModel = await startModelServer('```\n{"k":2}\n```') + const concurrentGepaModel = await startModelServer('```\n{"k":2}\n```', 750) + const skillOptModel = await startModelServer( + JSON.stringify({ + batch_size: 1, + failure_summary: [ + { + count: 1, + description: 'The required response rule is absent.', + failure_type: 'missing_rule', + }, + ], + patch: { + edits: [ + { + content: '\n\n## Required Rule\nALWAYS_RETURN_READY\n', + op: 'append', + }, + ], + reasoning: 'Add the missing response rule.', + }, + }), + ) + + try { + const firstGepa = await runGepa({ + runDir: gepaRunDir, + modelUrl: gepaModel.baseUrl, + resume: 'if-compatible', + }) + assert(firstGepa.provenance?.resumed === false, 'first GEPA run must be fresh') + assert(firstGepa.provenance?.source.version === '0.1.4', 'GEPA version was not observed') + assert( + firstGepa.provenance?.bridge?.version === '0.126.5', + 'agent-eval-rpc version was not observed', + ) + assert(firstGepa.decision === 'ship', 'GEPA candidate was not promoted') + assert( + JSON.parse(String(firstGepa.candidate.value)).k === 2, + 'GEPA did not produce the expected candidate', + ) + + const resumedGepa = await runGepa({ + runDir: gepaRunDir, + modelUrl: gepaModel.baseUrl, + resume: 'required', + }) + assert(resumedGepa.provenance?.resumed === true, 'second GEPA run did not restore state') + assert( + resumedGepa.provenance?.compatibleRunId === firstGepa.provenance?.compatibleRunId, + 'resumed GEPA run changed compatible identity', + ) + + let incompatibleError + try { + await runGepa({ + runDir: gepaRunDir, + modelUrl: gepaModel.baseUrl, + objective: 'Return a different configuration.', + resume: 'required', + }) + } catch (error) { + incompatibleError = error + } + assert(incompatibleError instanceof Error, 'incompatible GEPA state was accepted') + assert( + /no compatible run|does not exist/i.test(incompatibleError.message), + `unexpected incompatible-state error: ${incompatibleError.message}`, + ) + + const concurrentResults = await Promise.allSettled([ + runGepa({ + runDir: concurrentGepaRunDir, + modelUrl: concurrentGepaModel.baseUrl, + resume: 'if-compatible', + }), + runGepa({ + runDir: concurrentGepaRunDir, + modelUrl: concurrentGepaModel.baseUrl, + resume: 'if-compatible', + }), + ]) + const completedConcurrent = concurrentResults.filter((result) => result.status === 'fulfilled') + const rejectedConcurrent = concurrentResults.filter((result) => result.status === 'rejected') + assert( + completedConcurrent.length === 1 && rejectedConcurrent.length === 1, + `expected one completed and one rejected concurrent GEPA run, found ${completedConcurrent.length} completed and ${rejectedConcurrent.length} rejected`, + ) + const completedGepa = completedConcurrent[0].value + const concurrentError = rejectedConcurrent[0].reason + assert(concurrentError instanceof Error, 'concurrent GEPA rejection was not an Error') + assert( + /GEPA run '[0-9a-f]{64}' is already active/.test(concurrentError.message), + `concurrent GEPA run was not rejected by the upstream run lock: ${concurrentError.message}`, + ) + assert(completedGepa.decision === 'ship', 'completed concurrent GEPA run was not promoted') + assert( + JSON.parse(String(completedGepa.candidate.value)).k === 2, + 'completed concurrent GEPA run produced the wrong candidate', + ) + + const skillOpt = await runSkillOpt(skillOptRunDir, skillOptModel.baseUrl) + assert(skillOpt.provenance?.source.package === 'skillopt', 'SkillOpt source was not observed') + assert( + skillOpt.provenance?.source.revision === + '61735e3922efc2b90c6d6cab561e62e98452ca90', + 'SkillOpt revision was not observed', + ) + assert(skillOpt.decision === 'ship', 'SkillOpt candidate was not promoted') + assert( + String(skillOpt.candidate.value).includes('ALWAYS_RETURN_READY'), + 'SkillOpt did not produce the expected candidate', + ) + + process.stdout.write( + `${JSON.stringify({ + runtimeImport: '@tangle-network/agent-runtime', + gepa: { + bridge: firstGepa.provenance.bridge.version, + evaluations: firstGepa.provenance.evaluationCount, + resumed: resumedGepa.provenance.resumed, + source: firstGepa.provenance.source.version, + concurrent: { + completed: completedConcurrent.length, + rejectedByUpstreamLock: rejectedConcurrent.length, + }, + }, + skillopt: { + evaluations: skillOpt.provenance.evaluationCount, + revision: skillOpt.provenance.source.revision, + }, + })}\n`, + ) + } finally { + await Promise.all([gepaModel.close(), concurrentGepaModel.close(), skillOptModel.close()]) + await Promise.all([ + rm(gepaRunDir, { recursive: true, force: true }), + rm(concurrentGepaRunDir, { recursive: true, force: true }), + rm(skillOptRunDir, { recursive: true, force: true }), + ]) + } +} + +async function runOmniVerification() { + const runDir = await mkdtemp(join(tmpdir(), 'packed-runtime-gepa-omni-')) + const model = await startModelServer('```\n{"k":2}\n```') + const profile = { + name: 'packed-official-gepa-omni', + prompt: { systemPrompt: '{"k":1}' }, + } + + try { + const result = await improve(profile, { + surface: 'prompt', + executionRef: digest({ fixture: 'packed-official-gepa-omni' }), + method: officialGepa({ + objective: 'Return a JSON configuration whose k value is 2.', + recipe: { + kind: 'omni', + explore: [gepaEngineRun(11, 4), gepaEngineRun(29, 4)], + continueWith: gepaEngineRun(47, 4), + maxWorkers: 1, + }, + optimizer: optimizerModel(model.baseUrl, 2_000), + runner: { + command: python, + args: ['-m', 'agent_eval_rpc.gepa_bridge'], + }, + }), + trainScenarios: [{ id: 'train', kind: 'official', prompt: 'Set k to 2.' }], + selectionScenarios: [{ id: 'selection', kind: 'official', prompt: 'Set k to 2.' }], + testScenarios: [ + { id: 'test-1', kind: 'official', prompt: 'Set k to 2.' }, + { id: 'test-2', kind: 'official', prompt: 'Set k to 2 again.' }, + ], + agent: async (candidate) => ({ text: candidate.prompt?.systemPrompt ?? '' }), + judges: [kEqualsTwoJudge], + runDir, + costCeiling: 1, + seed: 7, + resamples: 40, + expectUsage: 'off', + optimizationRunOptions: { expectUsage: 'off' }, + }) + + assert(result.method === 'gepa:omni:gepa', `unexpected Omni method: ${result.method}`) + assert(result.provenance?.source.version === '0.1.4', 'source GEPA version was not observed') + assert( + result.provenance?.source.revision === 'f919db0a622e2e9f9204779b81fe00cc1b2d808f', + 'source GEPA revision was not observed', + ) + assert( + result.provenance?.bridge?.version === '0.126.5', + 'Omni agent-eval-rpc version was not observed', + ) + assert(result.decision === 'ship', 'Omni candidate was not promoted') + assert( + JSON.parse(String(result.candidate.value)).k === 2, + 'Omni did not produce the expected candidate', + ) + assert( + Number(result.provenance?.evaluationCount) > 0 && + Number(result.provenance?.evaluationCount) <= 12, + `Omni evaluation count must be between 1 and 12, found ${String(result.provenance?.evaluationCount)}`, + ) + assert( + model.requests.length > 0 && model.requests.length <= 3, + `Omni model requests must be between 1 and 3, found ${model.requests.length}`, + ) + + process.stdout.write( + `${JSON.stringify({ + runtimeImport: '@tangle-network/agent-runtime', + omni: { + bridge: result.provenance.bridge.version, + continuationEngines: 1, + evaluations: result.provenance.evaluationCount, + exploreEngines: 2, + modelRequests: model.requests.length, + revision: result.provenance.source.revision, + source: result.provenance.source.version, + }, + })}\n`, + ) + } finally { + await model.close() + await rm(runDir, { recursive: true, force: true }) + } +} + +function runGepa({ + runDir, + modelUrl, + objective = 'Return a JSON configuration whose k value is 2.', + resume, +}) { + const profile = { + name: 'packed-official-gepa', + prompt: { systemPrompt: '{"k":1}' }, + } + return improve(profile, { + surface: 'prompt', + executionRef: digest({ fixture: 'packed-official-gepa' }), + method: officialGepa({ + objective, + recipe: { + kind: 'engine', + run: gepaEngineRun(7, 5), + }, + optimizer: optimizerModel(modelUrl, 2_000), + resume, + trustResumeState: true, + runner: { + command: python, + args: ['-m', 'agent_eval_rpc.gepa_bridge'], + }, + }), + trainScenarios: [{ id: 'train', kind: 'official', prompt: 'Set k to 2.' }], + selectionScenarios: [{ id: 'selection', kind: 'official', prompt: 'Set k to 2.' }], + testScenarios: [ + { id: 'test-1', kind: 'official', prompt: 'Set k to 2.' }, + { id: 'test-2', kind: 'official', prompt: 'Set k to 2 again.' }, + ], + agent: async (candidate) => ({ text: candidate.prompt?.systemPrompt ?? '' }), + judges: [kEqualsTwoJudge], + runDir, + costCeiling: 1, + seed: 7, + resamples: 40, + expectUsage: 'off', + optimizationRunOptions: { expectUsage: 'off' }, + }) +} + +function gepaEngineRun(seed, maxEvaluations) { + return { + engine: 'gepa', + maxEvaluations, + maxProposerCostUsd: 1, + maxConcurrency: 1, + stopAtScore: 1, + engineConfig: { + engine: { + capture_stdio: false, + max_workers: 1, + parallel: false, + raise_on_exception: true, + seed, + use_cloudpickle: false, + }, + reflection: { + reflection_minibatch_size: 1, + skip_perfect_score: false, + }, + }, + } +} + +function runSkillOpt(runDir, modelUrl) { + const profile = { + name: 'packed-official-skillopt', + resources: { + failOnError: true, + skills: [{ kind: 'inline', name: 'answering', content: '# Answering\nAnswer normally.\n' }], + }, + } + return improve(profile, { + surface: 'skills', + skills: { resourceName: 'answering' }, + executionRef: digest({ fixture: 'packed-official-skillopt' }), + method: officialSkillOpt({ + objective: 'Add the required response rule.', + trainer: { + epochs: 1, + batchSize: 1, + accumulation: 1, + editBudget: 1, + minEditBudget: 1, + analystWorkers: 1, + minibatchSize: 1, + maxAnalystRounds: 1, + evaluationWorkers: 1, + }, + optimizer: optimizerModel(modelUrl, 256), + maxEvaluations: 3, + runner: { command: python }, + }), + trainScenarios: [{ id: 'train', kind: 'official', prompt: 'Return READY.' }], + selectionScenarios: [{ id: 'selection', kind: 'official', prompt: 'Return READY.' }], + testScenarios: [ + { id: 'test-1', kind: 'official', prompt: 'Return READY.' }, + { id: 'test-2', kind: 'official', prompt: 'Return READY again.' }, + ], + agent: async (candidate) => { + const skill = candidate.resources?.skills?.find( + (entry) => entry.kind === 'inline' && entry.name === 'answering', + ) + return { text: skill?.kind === 'inline' ? skill.content : '' } + }, + judges: [requiredRuleJudge], + runDir, + costCeiling: 1, + seed: 7, + resamples: 40, + expectUsage: 'off', + optimizationRunOptions: { expectUsage: 'off' }, + }) +} + +const kEqualsTwoJudge = { + name: 'k-equals-two', + dimensions: [{ key: 'correctness', description: 'candidate sets k to 2' }], + judgeVersion: 'packed-k-equals-two/1', + score: ({ artifact }) => { + let score = 0 + try { + score = JSON.parse(artifact.text).k === 2 ? 1 : 0 + } catch { + score = 0 + } + return { + dimensions: { correctness: score }, + composite: score, + notes: score ? '' : 'The candidate must be JSON with k set to 2.', + } + }, +} + +const requiredRuleJudge = { + name: 'required-rule', + dimensions: [{ key: 'correctness', description: 'candidate contains required rule' }], + judgeVersion: 'packed-required-rule/1', + score: ({ artifact }) => { + const score = artifact.text.includes('ALWAYS_RETURN_READY') ? 1 : 0 + return { + dimensions: { correctness: score }, + composite: score, + notes: score ? '' : 'The required response rule is absent.', + } + }, +} + +function optimizerModel(baseUrl, maxOutputTokensPerRequest) { + return { + model: 'local-model', + baseUrl, + apiKey: 'provider-secret', + budget: { + maxCostUsd: 1, + maxRequests: 10, + maxRequestBytes: 100_000, + maxResponseBytes: 100_000, + maxOutputTokensPerRequest, + pricing: { + inputUsdPerMillion: 1, + outputUsdPerMillion: 2, + }, + }, + } +} + +function digest(value) { + const hex = createHash('sha256').update(JSON.stringify(value)).digest('hex') + return `sha256:${hex}` +} + +function assert(condition, message) { + if (!condition) throw new Error(message) +} + +async function startModelServer(content, responseDelayMs = 0) { + const requests = [] + const server = createServer(async (request, response) => { + const chunks = [] + for await (const chunk of request) chunks.push(Buffer.from(chunk)) + requests.push(JSON.parse(Buffer.concat(chunks).toString('utf8'))) + if (responseDelayMs > 0) { + await new Promise((resolve) => setTimeout(resolve, responseDelayMs)) + } + response.writeHead(200, { 'content-type': 'application/json' }) + response.end( + JSON.stringify({ + choices: [ + { + finish_reason: 'stop', + index: 0, + message: { content, role: 'assistant' }, + }, + ], + created: 0, + id: 'chatcmpl-local', + model: 'local-model', + object: 'chat.completion', + usage: { + completion_tokens: 13, + prompt_tokens: 11, + total_tokens: 24, + }, + }), + ) + }) + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', () => { + server.off('error', reject) + resolve() + }) + }) + const address = server.address() + if (!address || typeof address === 'string') { + throw new Error('optimizer model server failed to bind') + } + return { + baseUrl: `http://127.0.0.1:${address.port}/v1`, + requests, + close: async () => { + server.closeAllConnections?.() + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())) + }) + }, + } +} + +await main() diff --git a/scripts/verify-official-optimizers.mjs b/scripts/verify-official-optimizers.mjs new file mode 100644 index 00000000..c260400b --- /dev/null +++ b/scripts/verify-official-optimizers.mjs @@ -0,0 +1,273 @@ +import { spawnSync } from 'node:child_process' +import { + copyFileSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +const agentEvalVersion = '0.126.5' +const agentKnowledgeVersion = '5.0.0' +const gepaVersion = '0.1.4' +const gepaSourceRevision = 'f919db0a622e2e9f9204779b81fe00cc1b2d808f' +const skillOptRevision = '61735e3922efc2b90c6d6cab561e62e98452ca90' +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..') +const packageJson = readJson(join(repoRoot, 'package.json')) +const tempRoot = mkdtempSync(join(tmpdir(), 'agent-runtime-official-')) + +assertVersion( + packageJson.devDependencies?.['@tangle-network/agent-eval'], + agentEvalVersion, + '@tangle-network/agent-eval development dependency', +) +assertVersion( + packageJson.dependencies?.['@tangle-network/agent-knowledge'], + agentKnowledgeVersion, + '@tangle-network/agent-knowledge dependency', +) +assertVersion( + packageJson.peerDependencies?.['@tangle-network/agent-eval'], + `>=${agentEvalVersion} <0.127.0`, + '@tangle-network/agent-eval peer dependency', +) + +try { + run('pnpm', ['build'], repoRoot) + + const python = installWheelPythonPackages(tempRoot) + run( + 'pnpm', + ['exec', 'vitest', 'run', 'src/improvement/official-packages.test.ts', '--maxWorkers=1'], + repoRoot, + { AGENT_EVAL_TEST_PYTHON: python }, + ) + + const packDir = join(tempRoot, 'pack') + const appDir = join(tempRoot, 'consumer') + mkdirSync(packDir, { recursive: true }) + mkdirSync(appDir, { recursive: true }) + + run( + 'npm', + ['pack', '--loglevel=error', '--ignore-scripts=false', '--pack-destination', packDir], + repoRoot, + ) + const tarballs = readdirSync(packDir).filter((name) => name.endsWith('.tgz')) + if (tarballs.length !== 1) { + throw new Error(`expected one Runtime tarball, found ${tarballs.length}`) + } + + const runtimeTarball = join(packDir, tarballs[0]) + writeFileSync( + join(appDir, 'package.json'), + `${JSON.stringify( + { + name: 'agent-runtime-official-optimizer-verification', + private: true, + type: 'module', + dependencies: { + '@tangle-network/agent-eval': agentEvalVersion, + '@tangle-network/agent-interface': + packageJson.devDependencies['@tangle-network/agent-interface'], + '@tangle-network/agent-runtime': `file:${runtimeTarball}`, + }, + }, + null, + 2, + )}\n`, + ) + copyFileSync( + join(repoRoot, 'scripts', 'verify-official-optimizers-consumer.mjs'), + join(appDir, 'verify.mjs'), + ) + run( + 'npm', + [ + 'install', + '--strict-peer-deps', + '--loglevel=error', + '--ignore-scripts=false', + '--no-audit', + '--no-fund', + '--cache', + join(tempRoot, 'npm-cache'), + '--prefer-online', + ], + appDir, + ) + + assertInstalledVersion(appDir, '@tangle-network/agent-runtime', packageJson.version) + assertInstalledVersion(appDir, '@tangle-network/agent-eval', agentEvalVersion) + assertInstalledVersion(appDir, '@tangle-network/agent-knowledge', agentKnowledgeVersion) + const installedKnowledge = readJson( + join(appDir, 'node_modules', '@tangle-network', 'agent-knowledge', 'package.json'), + ) + assertVersion( + installedKnowledge.dependencies?.['@tangle-network/agent-eval'], + agentEvalVersion, + 'installed Agent Knowledge dependency on Agent Eval', + ) + run( + process.execPath, + ['verify.mjs', 'wheel'], + appDir, + { AGENT_EVAL_TEST_PYTHON: python }, + 10 * 60_000, + ) + const sourcePython = installSourceGepa(tempRoot) + run( + process.execPath, + ['verify.mjs', 'omni'], + appDir, + { AGENT_EVAL_TEST_PYTHON: sourcePython }, + 10 * 60_000, + ) +} finally { + rmSync(tempRoot, { recursive: true, force: true }) +} + +function installWheelPythonPackages(root) { + const python = createPythonVenv(root, 'wheel-python') + installPythonRequirements(python, [ + '--only-binary=agent-eval-rpc,gepa', + `agent-eval-rpc==${agentEvalVersion}`, + `gepa[full]==${gepaVersion}`, + `skillopt @ git+https://github.com/microsoft/SkillOpt.git@${skillOptRevision}`, + ]) + const installed = inspectPythonPackages(python, true) + assertVersion(installed.bridge, agentEvalVersion, 'agent-eval-rpc Python package') + assertVersion(installed.gepa, gepaVersion, 'GEPA Python package') + assertVersion(installed.gepaRevision, null, 'wheel GEPA source revision') + assertVersion(installed.skilloptRevision, skillOptRevision, 'SkillOpt source revision') + return python +} + +function installSourceGepa(root) { + const python = createPythonVenv(root, 'source-python') + installPythonRequirements(python, [ + '--only-binary=agent-eval-rpc', + `agent-eval-rpc==${agentEvalVersion}`, + `gepa[full] @ git+https://github.com/gepa-ai/gepa.git@${gepaSourceRevision}`, + ]) + const installed = inspectPythonPackages(python, false) + assertVersion(installed.bridge, agentEvalVersion, 'source GEPA agent-eval-rpc package') + assertVersion(installed.gepa, gepaVersion, 'source GEPA package version') + assertVersion(installed.gepaRevision, gepaSourceRevision, 'GEPA source revision') + return python +} + +function createPythonVenv(root, name) { + const basePython = process.env.PYTHON ?? 'python' + const version = run( + basePython, + ['-c', 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")'], + repoRoot, + { PYTHONNOUSERSITE: '1' }, + ).trim() + if (version !== '3.12') { + throw new Error(`official optimizer verification requires Python 3.12, found ${version}`) + } + + const venvDir = join(root, name) + run(basePython, ['-m', 'venv', venvDir], repoRoot, { PYTHONNOUSERSITE: '1' }) + return process.platform === 'win32' + ? join(venvDir, 'Scripts', 'python.exe') + : join(venvDir, 'bin', 'python') +} + +function installPythonRequirements(python, requirements) { + run( + python, + [ + '-m', + 'pip', + 'install', + '--disable-pip-version-check', + '--quiet', + '--index-url=https://pypi.org/simple', + '--no-cache-dir', + ...requirements, + ], + repoRoot, + {}, + 10 * 60_000, + ) +} + +function inspectPythonPackages(python, includeSkillOpt) { + return JSON.parse( + run( + python, + [ + '-c', + ` +import json +from importlib.metadata import distribution, version + +gepa = distribution("gepa") +gepa_direct = json.loads(gepa.read_text("direct_url.json") or "{}") +${ + includeSkillOpt + ? `skillopt = distribution("skillopt") +skillopt_direct = json.loads(skillopt.read_text("direct_url.json") or "{}")` + : 'skillopt_direct = {}' +} +print(json.dumps({ + "bridge": version("agent-eval-rpc"), + "gepa": version("gepa"), + "gepaRevision": gepa_direct.get("vcs_info", {}).get("commit_id"), + "skilloptRevision": skillopt_direct.get("vcs_info", {}).get("commit_id"), +})) +`, + ], + repoRoot, + ).trim(), + ) +} + +function assertInstalledVersion(appDir, packageName, expected) { + const actual = readJson(join(appDir, 'node_modules', ...packageName.split('/'), 'package.json')) + .version + assertVersion(actual, expected, `installed ${packageName}`) +} + +function assertVersion(actual, expected, label) { + if (actual !== expected) { + throw new Error(`${label} must be ${expected}, found ${String(actual)}`) + } +} + +function readJson(path) { + return JSON.parse(readFileSync(path, 'utf8')) +} + +function run(command, args, cwd, env = {}, timeout = 5 * 60_000) { + const result = spawnSync(command, args, { + cwd, + encoding: 'utf8', + env: { ...process.env, ...env }, + stdio: ['ignore', 'pipe', 'pipe'], + timeout, + }) + if (result.error || result.status !== 0) { + throw new Error( + [ + `command failed: ${command} ${args.join(' ')}`, + result.error?.message, + result.stdout?.trim(), + result.stderr?.trim(), + ] + .filter(Boolean) + .join('\n'), + ) + } + process.stdout.write(result.stdout) + process.stderr.write(result.stderr) + return result.stdout +} diff --git a/src/improvement/code-execution.ts b/src/improvement/code-execution.ts index cc3bc0e6..1fdf216a 100644 --- a/src/improvement/code-execution.ts +++ b/src/improvement/code-execution.ts @@ -282,6 +282,7 @@ export async function runCodeImprovement( cost: copyImproveCost(raw.cost), durationMs: raw.durationMs, lineage: Object.freeze({ + invocationId: raw.provenance.runId, runId: raw.provenance.runId, developmentSplitDigest: raw.provenance.evidence.search.splitDigest, }), diff --git a/src/improvement/improve-types.ts b/src/improvement/improve-types.ts index 8d09291e..8175fa17 100644 --- a/src/improvement/improve-types.ts +++ b/src/improvement/improve-types.ts @@ -216,6 +216,8 @@ export interface ImproveCost { /** Optimizer ancestry sealed into downstream candidate experiments. */ export interface ImproveLineage { + /** Unique Runtime invocation used to isolate this run's cost receipts. */ + invocationId: string /** Upstream optimizer run when reported, otherwise this Runtime optimization invocation. */ runId: string /** Exact train-plus-selection scenario payloads exposed to candidate selection. */ diff --git a/src/improvement/improve.test.ts b/src/improvement/improve.test.ts index 78ca3fea..24515330 100644 --- a/src/improvement/improve.test.ts +++ b/src/improvement/improve.test.ts @@ -131,28 +131,22 @@ const promptProfile = (): AgentProfile => ({ describe('improve method execution', () => { it('runs a complete method without exposing final-test cases and materializes its prompt', async () => { let observed: OptimizationMethodInput | undefined + let observedEvaluationRef = '' const profile = promptProfile() - const expectedEvaluationRef = canonicalCandidateDigest({ - executionRef, - baselineProfileDigest: canonicalCandidateDigest(profile), - surface: 'prompt', - }) + const method = fixedMethod('improved prompt', (input) => (observed = input)) const result = await improve(profile, { - ...methodOptions(fixedMethod('improved prompt', (input) => (observed = input))), + ...methodOptions(method), surface: 'prompt', + method: (context) => { + observedEvaluationRef = context.evaluationRef + return method + }, }) expect(observed?.trainScenarios.map((scenario) => scenario.id)).toEqual(['train']) expect(observed?.selectionScenarios.map((scenario) => scenario.id)).toEqual(['selection']) - expect(observed?.runOptions.dispatchRef).toBe(`improve:${expectedEvaluationRef}`) - expect(observed?.judges[0]?.judgeVersion).toBe( - canonicalCandidateDigest({ - evaluationRef: expectedEvaluationRef, - name: improvementJudge.name, - dimensions: improvementJudge.dimensions, - declaredJudgeVersion: null, - }), - ) + expect(observed?.runOptions.dispatchRef).toBe(`improve:${observedEvaluationRef}`) + expect(observed?.judges[0]?.judgeVersion).toMatch(/^sha256:[a-f0-9]{64}$/) expect(JSON.stringify(observed)).not.toContain('test-a') expect(result.mode).toBe('method') expect(result.method).toBe('fixed-method') @@ -567,6 +561,7 @@ describe('improve method execution', () => { }) it('rejects method-reported spend above the configured total limit', async () => { + let agentCalls = 0 const method = fixedMethod('improved prompt') method.optimize = async () => ({ winnerSurface: 'improved prompt', @@ -581,8 +576,13 @@ describe('improve method execution', () => { improve(promptProfile(), { ...methodOptions(method), costCeiling: 1, + agent: async (candidate, scenario, context) => { + agentCalls += 1 + return paidProfile(candidate, scenario, context) + }, }), - ).rejects.toThrow(/reported total cost \$2\.\d+ exceeds costCeiling \$1/) + ).rejects.toThrow(/reported cost \$2 above costCeiling \$1; refusing final scoring/) + expect(agentCalls).toBe(0) }) it('rejects an invalid method and malformed profile output', async () => { @@ -676,7 +676,7 @@ describe('improve code execution', () => { const repo = createRepo('improve-code-') try { let generatorCalls = 0 - const result = await improve(promptProfile(), { + const result = await improve({ surface: 'code', findings: [{ claim: 'module.txt is stale' }], scenarios: allScenarios, @@ -726,7 +726,7 @@ describe('improve code execution', () => { it('retains and disposes the incumbent for a baseline-only code run', async () => { const repo = createRepo('improve-code-baseline-') try { - const result = await improve(promptProfile(), { + const result = await improve({ surface: 'code', gate: 'none', scenarios: allScenarios, @@ -762,7 +762,7 @@ describe('improve code execution', () => { const repo = createRepo('improve-code-reject-') try { await expect( - improve(promptProfile(), { + improve({ surface: 'code', scenarios: allScenarios, judge: improvementJudge, @@ -802,7 +802,7 @@ describe('improve code execution', () => { } await expect( - improve(promptProfile(), { + improve({ surface: 'code', scenarios: allScenarios, judge: improvementJudge, diff --git a/src/improvement/improve.ts b/src/improvement/improve.ts index ea01514d..78f34c0d 100644 --- a/src/improvement/improve.ts +++ b/src/improvement/improve.ts @@ -19,7 +19,6 @@ import type { ImproveCodeRunOptions, ImproveMethodOptions, ImproveMethodResult, - ImproveOptions, ImproveResult, } from './improve-types' import { runMethodImprovement } from './method-execution' @@ -49,33 +48,37 @@ export type { } from './improve-types' /** - * Optimize one exact profile surface with a complete method, or optimize code - * through Runtime's isolated worktree path. The input profile is never changed. + * Optimize one exact profile surface with a complete method. */ export function improve( profile: AgentProfile, opts: ImproveMethodOptions, ): Promise +/** + * Optimize repository code through Runtime's isolated worktree path. + */ export function improve( - profile: AgentProfile, opts: ImproveCodeRunOptions, ): Promise> -export function improve( - profile: AgentProfile, - opts: ImproveOptions, -): Promise> export async function improve( - profile: AgentProfile, - opts: ImproveOptions, + profileOrCode: AgentProfile | ImproveCodeRunOptions, + opts?: ImproveMethodOptions, ): Promise> { - const parsedProfile = agentProfileSchema.safeParse(profile) + if (opts === undefined) { + const code = profileOrCode as ImproveCodeRunOptions + if (!code || code.surface !== 'code') { + throw new ConfigError("improve(): the one-argument form requires { surface: 'code', ... }") + } + return runCodeImprovement(code) + } + if ((opts as { surface?: string }).surface === 'code') { + throw new ConfigError("improve(): code takes one argument: improve({ surface: 'code', ... })") + } + const parsedProfile = agentProfileSchema.safeParse(profileOrCode) if (!parsedProfile.success) { throw new ConfigError( `improve(): input is not a valid AgentProfile: ${parsedProfile.error.message}`, ) } - if (opts.surface === 'code') { - return runCodeImprovement(opts) - } return runMethodImprovement(immutableCandidateValue(parsedProfile.data), opts) } diff --git a/src/improvement/method-cost.test.ts b/src/improvement/method-cost.test.ts new file mode 100644 index 00000000..4ebcc977 --- /dev/null +++ b/src/improvement/method-cost.test.ts @@ -0,0 +1,111 @@ +import { CostLedger } from '@tangle-network/agent-eval' +import type { OptimizationMethodInput, Scenario } from '@tangle-network/agent-eval/campaign' +import { canonicalCandidateDigest } from '@tangle-network/agent-interface' +import { describe, expect, it } from 'vitest' +import { assertMethodCostRecorded, methodInputWithScopedCost } from './method-cost' + +const evaluationRef = canonicalCandidateDigest({ fixture: 'shared-cost-ledger' }) + +function scopedInput(ledger: CostLedger, invocationId: string) { + return methodInputWithScopedCost( + { costLedger: ledger } as unknown as OptimizationMethodInput, + { evaluationRef, invocationId }, + ) +} + +async function recordCost(input: ReturnType, cost: number): Promise { + const paid = await input.costLedger.runPaidCall({ + channel: 'driver', + phase: 'optimizer', + actor: 'optimizer-model', + model: 'fixture', + maximumCharge: { externallyEnforcedMaximumUsd: cost }, + execute: async () => 'candidate', + receipt: () => ({ + model: 'fixture', + inputTokens: 1, + outputTokens: 1, + actualCostUsd: cost, + }), + }) + expect(paid.succeeded).toBe(true) +} + +describe('optimizer cost reconciliation', () => { + it('accepts complete spend recorded through the shared account', async () => { + const ledger = new CostLedger({ costCeilingUsd: 1 }) + const paid = await ledger.runPaidCall({ + channel: 'driver', + phase: 'optimizer', + actor: 'optimizer-model', + model: 'fixture', + maximumCharge: { externallyEnforcedMaximumUsd: 0.25 }, + execute: async () => 'candidate', + receipt: () => ({ + model: 'fixture', + inputTokens: 1, + outputTokens: 1, + actualCostUsd: 0.25, + }), + }) + expect(paid.succeeded).toBe(true) + + expect(() => + assertMethodCostRecorded( + 'fixture', + { totalCostUsd: 0.25, accountingComplete: true, incompleteReasons: [] }, + ledger, + 1, + ), + ).not.toThrow() + }) + + it('rejects complete spend that bypassed the shared account', () => { + const ledger = new CostLedger({ costCeilingUsd: 1 }) + + expect(() => + assertMethodCostRecorded( + 'fixture', + { totalCostUsd: 0.25, accountingComplete: true, incompleteReasons: [] }, + ledger, + 1, + ), + ).toThrow(/reported \$0\.25 but recorded \$0 through input\.costLedger/) + }) + + it('rejects unknown spend before final scoring when a total limit is configured', () => { + const ledger = new CostLedger({ costCeilingUsd: 1 }) + + expect(() => + assertMethodCostRecorded( + 'fixture', + { + totalCostUsd: 0, + accountingComplete: false, + incompleteReasons: ['provider receipt unavailable'], + }, + ledger, + 1, + ), + ).toThrow(/incomplete cost accounting under costCeiling; refusing final scoring/) + }) + + it('isolates parallel invocations that share a resumable evaluation identity', async () => { + const ledger = new CostLedger({ costCeilingUsd: 1 }) + const first = scopedInput(ledger, 'invocation-first') + const second = scopedInput(ledger, 'invocation-second') + + await Promise.all([recordCost(first, 0.1), recordCost(second, 0.2)]) + + expect(first.costLedger.summary()).toMatchObject({ + totalCalls: 1, + totalCostUsd: 0.1, + }) + expect(second.costLedger.summary()).toMatchObject({ + totalCalls: 1, + totalCostUsd: 0.2, + }) + expect(ledger.summary().totalCalls).toBe(2) + expect(ledger.summary().totalCostUsd).toBeCloseTo(0.3) + }) +}) diff --git a/src/improvement/method-cost.ts b/src/improvement/method-cost.ts new file mode 100644 index 00000000..6a3629d2 --- /dev/null +++ b/src/improvement/method-cost.ts @@ -0,0 +1,101 @@ +import type { + ComparisonCost, + CostLedgerHandle, + OptimizationMethodInput, + Scenario, +} from '@tangle-network/agent-eval/campaign' +import type { Sha256Digest } from '@tangle-network/agent-interface' +import { ConfigError } from '../errors' + +type LedgerFilter = Parameters[0] +type LedgerWaitOptions = Parameters>[0] + +const EVALUATION_TAG = 'runtimeEvaluationRef' +const INVOCATION_TAG = 'runtimeInvocationId' + +interface MethodCostScope { + evaluationRef: Sha256Digest + invocationId: string +} + +export function methodInputWithScopedCost( + input: OptimizationMethodInput, + scope: MethodCostScope, +): OptimizationMethodInput { + return Object.freeze({ + ...input, + costLedger: scopedCostLedger(input.costLedger, scope), + }) +} + +export function assertMethodCostRecorded( + methodName: string, + cost: ComparisonCost, + ledger: CostLedgerHandle, + costCeiling: number | undefined, +): void { + const observed = ledger.summary() + if (costCeiling !== undefined && !cost.accountingComplete) { + throw new ConfigError( + `improve(): method '${methodName}' returned incomplete cost accounting under costCeiling; refusing final scoring`, + ) + } + if (costCeiling !== undefined && exceeds(cost.totalCostUsd, costCeiling)) { + throw new ConfigError( + `improve(): method '${methodName}' reported cost $${cost.totalCostUsd} above costCeiling $${costCeiling}; refusing final scoring`, + ) + } + if (cost.accountingComplete && !observed.accountingComplete) { + throw new ConfigError( + `improve(): method '${methodName}' reported complete cost accounting but its shared cost receipts are incomplete`, + ) + } + if (cost.accountingComplete && !approximatelyEqual(cost.totalCostUsd, observed.totalCostUsd)) { + throw new ConfigError( + `improve(): method '${methodName}' reported $${cost.totalCostUsd} but recorded $${observed.totalCostUsd} through input.costLedger`, + ) + } +} + +function scopedCostLedger(parent: CostLedgerHandle, scope: MethodCostScope): CostLedgerHandle { + const tags = { + [EVALUATION_TAG]: scope.evaluationRef, + [INVOCATION_TAG]: scope.invocationId, + } + return { + costCeilingUsd: parent.costCeilingUsd, + runPaidCall: (input) => + parent.runPaidCall({ + ...input, + tags: { ...(input.tags ?? {}), ...tags }, + }), + reconcile: (...args) => parent.reconcile(...args), + list: (filter) => parent.list(withTags(filter, tags)), + listPending: (filter) => parent.listPending?.(withTags(filter, tags)) ?? [], + summary: (filter) => parent.summary(withTags(filter, tags)), + waitForIdle: (options: LedgerWaitOptions = {}) => + parent.waitForIdle?.({ + ...options, + filter: withTags(options.filter, tags), + }) ?? Promise.resolve(parent.summary(withTags(options.filter, tags)).pendingCalls === 0), + markCompleted: (count) => parent.markCompleted(count), + costPerCompletedTask: () => parent.costPerCompletedTask(), + } +} + +function withTags(filter: LedgerFilter, tags: Record): LedgerFilter { + return { + ...(filter ?? {}), + tags: { ...(filter?.tags ?? {}), ...tags }, + } +} + +function approximatelyEqual(left: number, right: number): boolean { + const tolerance = Number.EPSILON * Math.max(1, Math.abs(left), Math.abs(right)) * 8 + return Math.abs(left - right) <= tolerance +} + +function exceeds(value: number, limit: number): boolean { + const tolerance = Number.EPSILON * Math.max(1, Math.abs(value), Math.abs(limit)) * 8 + return value - limit > tolerance +} diff --git a/src/improvement/method-execution.ts b/src/improvement/method-execution.ts index f70c886e..a80c4826 100644 --- a/src/improvement/method-execution.ts +++ b/src/improvement/method-execution.ts @@ -1,6 +1,5 @@ import { randomUUID } from 'node:crypto' import { - campaignSplitDigest, compareOptimizationMethods, type OptimizationMethod, type OptimizationMethodComparison, @@ -21,6 +20,8 @@ import type { ImproveMethodSource, ImprovementProfileCandidate, } from './improve-types' +import { assertMethodCostRecorded, methodInputWithScopedCost } from './method-cost' +import { buildMethodEvaluationIdentity } from './method-identity' import { assertCandidateSurfaceKind, createProfileCandidateMaterializer, @@ -50,11 +51,7 @@ function resolveOptimizationMethod( function copyProvenance( provenance: NonNullable, ): NonNullable { - return { - ...provenance, - source: { ...provenance.source }, - ...(provenance.tokenUsage ? { tokenUsage: { ...provenance.tokenUsage } } : {}), - } + return immutableCandidateValue(provenance) } function validateExecutionRef(value: unknown): Sha256Digest { @@ -65,17 +62,6 @@ function validateExecutionRef(value: unknown): Sha256Digest { return parsed.data } -function developmentSplitDigest( - trainScenarios: readonly TScenario[], - selectionScenarios: readonly TScenario[], - reps: number, -): Sha256Digest { - return canonicalCandidateDigest({ - train: campaignSplitDigest(trainScenarios, reps), - selection: campaignSplitDigest(selectionScenarios, reps), - }) -} - export async function runMethodImprovement( profile: AgentProfile, opts: ImproveMethodOptions, @@ -106,20 +92,29 @@ export async function runMethodImprovement + const identifiedJudges = comparisonOptions.judges.map((judge, index) => Object.freeze({ ...judge, judgeVersion: canonicalCandidateDigest({ evaluationRef, - name: judge.name, - dimensions: judge.dimensions, - declaredJudgeVersion: judge.judgeVersion ?? null, + descriptor: identity.judgeDescriptors[index], }), }), ) @@ -130,12 +125,7 @@ export async function runMethodImprovement = { ...method, async optimize(input) { - const result = await method.optimize(input) + const scopedInput = methodInputWithScopedCost(input, { + evaluationRef, + invocationId: runtimeInvocationId, + }) + const result = await method.optimize(scopedInput) + assertMethodCostRecorded( + method.name, + result.cost, + scopedInput.costLedger, + comparisonOptions.costCeiling, + ) materializeProfile(result.winnerSurface) return result }, @@ -196,8 +196,9 @@ export async function runMethodImprovement = { + name: 'identity-quality', + dimensions: [{ key: 'quality', description: 'whether the selected skill improved' }], + judgeVersion: 'identity-quality/v1', + score: ({ artifact }) => ({ + composite: artifact.improved ? 1 : 0, + dimensions: { quality: artifact.improved ? 1 : 0 }, + notes: '', + }), +} + +const train: IdentityScenario[] = [{ id: 'train', kind: 'identity', privateInput: 'train-a' }] +const selection: IdentityScenario[] = [ + { id: 'selection', kind: 'identity', privateInput: 'selection-a' }, +] +const finalScenarios: IdentityScenario[] = [ + { id: 'final-a', kind: 'identity' }, + { id: 'final-b', kind: 'identity' }, +] +const executionRef = canonicalCandidateDigest({ callback: 'identity-fixture' }) +const profileDigest = canonicalCandidateDigest({ profile: 'identity-fixture' }) + +describe('method evaluation identity', () => { + it('changes for every behavior-bearing development coordinate', () => { + const baseline = buildMethodEvaluationIdentity({ + executionRef, + baselineProfileDigest: profileDigest, + baselineSurface: 'same bytes', + surface: 'skills', + skills: { resourceName: 'skill-a' }, + findings: [{ issue: 'missing citation' }], + trainScenarios: train, + selectionScenarios: selection, + judges: [judge], + reps: 1, + optimizationRunOptions: { reps: 1 }, + }).evaluationRef + const evaluationRef = (overrides: { + skills?: { resourceName: string } + findings?: readonly unknown[] + trainScenarios?: IdentityScenario[] + judges?: JudgeConfig[] + reps?: number + optimizationRunOptions?: { reps?: number } + }) => + buildMethodEvaluationIdentity({ + executionRef, + baselineProfileDigest: profileDigest, + baselineSurface: 'same bytes', + surface: 'skills', + skills: { resourceName: 'skill-a' }, + findings: [{ issue: 'missing citation' }], + trainScenarios: train, + selectionScenarios: selection, + judges: [judge], + reps: 1, + optimizationRunOptions: { reps: 1 }, + ...overrides, + }).evaluationRef + + expect(evaluationRef({ skills: { resourceName: 'skill-b' } })).not.toBe(baseline) + expect( + evaluationRef({ + trainScenarios: [{ ...train[0]!, privateInput: 'changed-hidden-input' }], + }), + ).not.toBe(baseline) + expect( + evaluationRef({ + judges: [{ ...judge, judgeVersion: 'identity-quality/v2' }], + }), + ).not.toBe(baseline) + expect(evaluationRef({ optimizationRunOptions: { reps: 2 } })).not.toBe(baseline) + expect(evaluationRef({ reps: 2 })).not.toBe(baseline) + expect(evaluationRef({ findings: [{ issue: 'different failure' }] })).not.toBe(baseline) + expect(evaluationRef({})).toBe(baseline) + }) + + it('does not reuse measurements for two same-content skills', async () => { + const profile: AgentProfile = { + name: 'identity-agent', + resources: { + failOnError: true, + skills: [ + { kind: 'inline', name: 'skill-a', content: 'same bytes' }, + { kind: 'inline', name: 'skill-b', content: 'same bytes' }, + ], + }, + } + const storage = inMemoryCampaignStorage() + let agentCalls = 0 + const method: OptimizationMethod = { + name: 'identity-fixed', + async optimize() { + return { + winnerSurface: 'improved bytes', + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + } + }, + } + const options = { + executionRef, + method, + trainScenarios: train, + selectionScenarios: selection, + testScenarios: finalScenarios, + judges: [judge], + agent: async (candidate: ReadonlyAgentProfile) => { + agentCalls += 1 + return { + improved: + candidate.resources?.skills?.some( + (skill) => skill.kind === 'inline' && skill.content === 'improved bytes', + ) === true, + } + }, + runDir: 'mem://same-content-skills', + storage, + resamples: 40, + confidence: 0.95, + expectUsage: 'off' as const, + } + + await improve(profile, { + ...options, + surface: 'skills', + skills: { resourceName: 'skill-a' }, + }) + const afterFirst = agentCalls + await improve(profile, { + ...options, + surface: 'skills', + skills: { resourceName: 'skill-b' }, + }) + + expect(afterFirst).toBeGreaterThan(0) + expect(agentCalls).toBeGreaterThan(afterFirst) + }) +}) diff --git a/src/improvement/method-identity.ts b/src/improvement/method-identity.ts new file mode 100644 index 00000000..ae52ed1e --- /dev/null +++ b/src/improvement/method-identity.ts @@ -0,0 +1,113 @@ +import { + campaignSplitDigest, + type JudgeConfig, + type MutableSurface, + type Scenario, +} from '@tangle-network/agent-eval/campaign' +import type { Sha256Digest } from '@tangle-network/agent-interface' +import { canonicalCandidateDigest } from '../candidate-execution/digest' +import type { + ImproveOptimizationRunOptions, + ImproveProfileSurface, + ImproveSkillsOptions, +} from './improve-types' + +export interface MethodEvaluationIdentity { + evaluationRef: Sha256Digest + developmentSplitDigest: Sha256Digest + judgeDescriptors: readonly MethodJudgeDescriptor[] +} + +interface MethodJudgeDescriptor { + name: string + dimensions: readonly { key: string; description: string }[] + declaredVersion: string | null + scoreImplementation: string + appliesToImplementation: string | null +} + +export function buildMethodEvaluationIdentity(input: { + executionRef: Sha256Digest + baselineProfileDigest: Sha256Digest + baselineSurface: MutableSurface + surface: ImproveProfileSurface + skills?: ImproveSkillsOptions + findings: readonly unknown[] + trainScenarios: readonly TScenario[] + selectionScenarios: readonly TScenario[] + judges: readonly JudgeConfig[] + seed?: number + reps?: number + costCeiling?: number + optimizationRunOptions?: ImproveOptimizationRunOptions +}): MethodEvaluationIdentity { + const optimizationReps = input.optimizationRunOptions?.reps ?? 1 + const developmentSplitDigest = canonicalCandidateDigest({ + train: campaignSplitDigest(input.trainScenarios, optimizationReps), + selection: campaignSplitDigest(input.selectionScenarios, optimizationReps), + }) + const judgeDescriptors = input.judges.map(judgeDescriptor) + const evaluationRef = canonicalCandidateDigest({ + executionRef: input.executionRef, + baselineProfileDigest: input.baselineProfileDigest, + coordinate: profileCoordinate(input.surface, input.baselineSurface, input.skills), + developmentSplitDigest, + findings: input.findings, + judges: judgeDescriptors, + run: { + seed: input.seed ?? 42, + finalReps: input.reps ?? 1, + optimizationReps, + costCeiling: input.costCeiling ?? null, + resumable: input.optimizationRunOptions?.resumable ?? true, + maxConcurrency: input.optimizationRunOptions?.maxConcurrency ?? 2, + abortOnCellError: input.optimizationRunOptions?.abortOnCellError ?? false, + dispatchTimeoutMs: input.optimizationRunOptions?.dispatchTimeoutMs ?? null, + dispatchShutdownTimeoutMs: input.optimizationRunOptions?.dispatchShutdownTimeoutMs ?? 5_000, + tracing: input.optimizationRunOptions?.tracing ?? 'on', + expectUsage: input.optimizationRunOptions?.expectUsage ?? 'warn', + captureSource: input.optimizationRunOptions?.captureSource ?? null, + captureSourceVersionHash: input.optimizationRunOptions?.captureSourceVersionHash ?? null, + }, + }) + return { + evaluationRef, + developmentSplitDigest, + judgeDescriptors, + } +} + +function profileCoordinate( + surface: ImproveProfileSurface, + baselineSurface: MutableSurface, + skills: ImproveSkillsOptions | undefined, +): { + surface: ImproveProfileSurface + resourceName: string | null + componentNames: string[] +} { + return { + surface, + resourceName: surface === 'skills' ? (skills?.resourceName.trim() ?? null) : null, + componentNames: + typeof baselineSurface === 'object' && + baselineSurface !== null && + baselineSurface.kind === 'components' + ? Object.keys(baselineSurface.components).sort() + : [], + } +} + +function judgeDescriptor( + judge: JudgeConfig, +): MethodJudgeDescriptor { + return { + name: judge.name, + dimensions: judge.dimensions.map((dimension) => ({ ...dimension })), + declaredVersion: judge.judgeVersion ?? null, + scoreImplementation: Function.prototype.toString.call(judge.score), + appliesToImplementation: judge.appliesTo + ? Function.prototype.toString.call(judge.appliesTo) + : null, + } +} diff --git a/src/improvement/official-optimizers.test.ts b/src/improvement/official-optimizers.test.ts index ba1adb4d..bb91a002 100644 --- a/src/improvement/official-optimizers.test.ts +++ b/src/improvement/official-optimizers.test.ts @@ -140,7 +140,7 @@ function fakeRunner( ' python: { implementation: "CPython", version: "3.12.0" },', ' bridge: { package: "agent-eval-rpc", version: "0.126.1", sourceSha256: "a".repeat(64) },', ' optimizer: optimizerSource,', - ' engineModules: [],', + ' engineModules: [{ module: "fixture-engine", sourceSha256: "d".repeat(64) }],', ' } }))', ' process.exit(0)', '}', @@ -180,6 +180,27 @@ function failingRunner(message: string) { } } +function replacingRedactor( + replacements: ReadonlyArray, +): (value: unknown) => unknown { + const redact = (value: unknown): unknown => { + if (typeof value === 'string') { + return replacements.reduce((result, [from, to]) => result.replaceAll(from, to), value) + } + if (Array.isArray(value)) return value.map(redact) + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value as Record).map(([key, child]) => [ + key, + redact(child), + ]), + ) + } + return value + } + return redact +} + const testOptimizer = { model: 'optimizer-model', baseUrl: 'http://127.0.0.1:1/v1', @@ -242,7 +263,6 @@ describe('official optimizer methods', () => { const observed = JSON.parse(readFileSync(observedInputPath, 'utf8')) as Record expect(observed).toMatchObject({ - evaluationId: evaluationRef, resume: 'if-compatible', trustedResumeState: true, recipe: { @@ -256,6 +276,7 @@ describe('official optimizer methods', () => { trainSet: [{ id: 'train', data: { prompt: 'visible train' } }], selectionSet: [{ id: 'selection', data: { prompt: 'visible selection' } }], }) + expect(observed.evaluationId).toMatch(/^sha256:[a-f0-9]{64}$/) expect(observed).not.toHaveProperty('version') expect(String(observed.background)).toContain('answers omit citations') expect(JSON.stringify(observed)).not.toContain('SECRET') @@ -263,10 +284,23 @@ describe('official optimizer methods', () => { expect(result.method).toBe('gepa:gepa') expect(result.provenance).toMatchObject({ source: { package: 'gepa', version: 'test' }, + bridge: { package: 'agent-eval-rpc', version: '0.126.1' }, + modules: [{ module: 'fixture-engine', sourceSha256: 'd'.repeat(64) }], + python: { implementation: 'CPython', version: '3.12.0' }, runId: observed.runId, resumed: false, evaluationCount: 0, }) + expect(result.provenance).not.toBe(result.raw.best.provenance) + expect(result.provenance?.bridge).not.toBe(result.raw.best.provenance?.bridge) + expect(result.provenance?.modules).not.toBe(result.raw.best.provenance?.modules) + expect(Object.isFrozen(result.provenance)).toBe(true) + expect(Object.isFrozen(result.provenance?.bridge)).toBe(true) + expect(Object.isFrozen(result.provenance?.modules)).toBe(true) + expect(() => { + if (result.provenance?.bridge) result.provenance.bridge.version = 'mutated' + }).toThrow() + expect(result.raw.best.provenance?.bridge?.version).toBe('0.126.1') expect(result.decision).toBe('ship') expect(result.candidate.profile?.prompt?.systemPrompt).toBe('improved prompt') }) @@ -330,7 +364,6 @@ describe('official optimizer methods', () => { const observed = JSON.parse(readFileSync(observedInputPath, 'utf8')) as Record expect(observed).toMatchObject({ - evaluationId: evaluationRef, resume: 'if-compatible', trainer: { epochs: 1, @@ -339,6 +372,7 @@ describe('official optimizer methods', () => { trainSet: [{ id: 'train', data: { prompt: 'visible train' } }], selectionSet: [{ id: 'selection', data: { prompt: 'visible selection' } }], }) + expect(observed.evaluationId).toMatch(/^sha256:[a-f0-9]{64}$/) expect(observed).not.toHaveProperty('version') expect(observed.trainer).toEqual({ epochs: 1, batchSize: 1 }) expect(String(observed.background)).toContain('answers omit citations') @@ -355,10 +389,226 @@ describe('official optimizer methods', () => { expect(result.decision).toBe('ship') }) + it('keeps ID-only scenarios and omits artifacts when descriptors are absent', async () => { + const root = runDir() + const observedInputPath = join(root, 'observed-default-input.json') + const observedResponsePath = join(root, 'observed-default-response.json') + const result = await improve(profile, { + ...commonOptions( + officialGepa({ + objective: 'Improve the agent prompt.', + recipe: { + kind: 'engine', + run: { engine: 'gepa', maxEvaluations: 1, maxProposerCostUsd: 1 }, + }, + optimizer: testOptimizer, + runner: fakeRunner('gepa', observedInputPath, { + exampleId: 'train', + responsePath: observedResponsePath, + }), + }), + ), + agent: async (candidate, scenario, ctx) => ({ + ...(await agent(candidate, scenario, ctx)), + privateDetail: `artifact for ${scenario.privateNote}`, + }), + runDir: join(root, 'run'), + }) + + const observedInput = JSON.parse(readFileSync(observedInputPath, 'utf8')) as { + trainSet: unknown + selectionSet: unknown + } + const observedResponse = JSON.parse(readFileSync(observedResponsePath, 'utf8')) as { + info: Record + } + expect(observedInput.trainSet).toEqual([{ id: 'train', data: { id: 'train' } }]) + expect(observedInput.selectionSet).toEqual([{ id: 'selection', data: { id: 'selection' } }]) + expect(observedResponse.info).not.toHaveProperty('artifact') + expect(JSON.stringify(observedInput)).not.toContain('privateNote') + expect(JSON.stringify(observedInput)).not.toContain('SECRET') + expect(JSON.stringify(observedResponse)).not.toContain('SECRET') + expect(result.provenance?.evaluationCount).toBe(1) + }) + + it('applies a caller redactor to arbitrary PII in supplied descriptors', async () => { + const root = runDir() + const observedInputPath = join(root, 'observed-domain-redaction-input.json') + const observedResponsePath = join(root, 'observed-domain-redaction-response.json') + const customerName = 'Jane Example' + const accountReference = 'ACCT-8472-NORTH' + const redact = replacingRedactor([ + [customerName, '[customer]'], + [accountReference, '[account]'], + ]) + const result = await improve(profile, { + ...commonOptions( + officialGepa({ + objective: `Improve support for ${customerName}.`, + background: `Account ${accountReference}.`, + recipe: { + kind: 'engine', + run: { engine: 'gepa', maxEvaluations: 1, maxProposerCostUsd: 1 }, + }, + optimizer: testOptimizer, + describeScenario: (scenario) => ({ + prompt: scenario.prompt, + customerName, + accountReference, + }), + describeArtifact: (artifact) => ({ + text: artifact.text, + customerName, + accountReference, + }), + redact, + runner: fakeRunner('gepa', observedInputPath, { + exampleId: 'train', + responsePath: observedResponsePath, + }), + }), + ), + runDir: join(root, 'run'), + }) + + const outbound = `${readFileSync(observedInputPath, 'utf8')}\n${readFileSync( + observedResponsePath, + 'utf8', + )}` + expect(outbound).not.toContain(customerName) + expect(outbound).not.toContain(accountReference) + expect(outbound).toContain('[customer]') + expect(outbound).toContain('[account]') + expect(result.decision).toBe('ship') + }) + + it('keeps built-in credential redaction after a caller redactor', async () => { + const root = runDir() + const observedInputPath = join(root, 'observed-composed-redaction-input.json') + const customerName = 'Jane Example' + await improve(profile, { + ...commonOptions( + officialGepa({ + objective: `Improve support for ${customerName}.`, + background: 'Authorization: Bearer abcdefghijklmnop', + recipe: { + kind: 'engine', + run: { engine: 'gepa', maxEvaluations: 1, maxProposerCostUsd: 1 }, + }, + optimizer: testOptimizer, + describeScenario: (scenario) => ({ + prompt: scenario.prompt, + customerName, + apiKey: 'sk-abcdefghijklmnop', + }), + redact: replacingRedactor([[customerName, '[customer]']]), + runner: fakeRunner('gepa', observedInputPath), + }), + ), + runDir: join(root, 'run'), + }) + + const outbound = readFileSync(observedInputPath, 'utf8') + expect(outbound).not.toContain(customerName) + expect(outbound).not.toContain('abcdefghijklmnop') + expect(outbound).toContain('[customer]') + expect(outbound).toContain('[redacted]') + }) + + it.each([ + [ + 'a signed MCP URL', + { + remote: { + transport: 'http' as const, + url: 'https://mcp.example.test/callback?signature=opaque-7f91d8e4', + }, + }, + '$.remote.url', + ], + [ + 'an opaque MCP header', + { + remote: { + transport: 'http' as const, + url: 'https://mcp.example.test', + headers: { 'x-workspace-proof': 'opaque-7f91d8e4-customer-value' }, + }, + }, + '$.remote.headers', + ], + [ + 'opaque MCP metadata', + { + local: { + transport: 'stdio' as const, + command: 'mcp-server', + metadata: { customer: 'north-star-8472' }, + }, + }, + '$.local.metadata', + ], + ])('rejects %s unless the selected surface is explicitly approved', async (_, mcp, path) => { + await expect( + improve( + { + name: 'optimizer-fixture', + mcp, + }, + { + ...commonOptions( + officialGepa({ + objective: 'Improve the MCP profile.', + recipe: { + kind: 'engine', + run: { engine: 'gepa', maxEvaluations: 1, maxProposerCostUsd: 1 }, + }, + optimizer: testOptimizer, + runner: failingRunner('runner must not start'), + }), + ), + surface: 'mcp', + }, + ), + ).rejects.toMatchObject({ + message: expect.stringContaining(path), + }) + }) + + it('allows a reviewed sensitive profile surface only with explicit approval', () => { + const mcp = { + remote: { + transport: 'http' as const, + url: 'https://public.example.test/mcp', + }, + } + const method = officialGepa({ + objective: 'Improve the MCP profile.', + recipe: { + kind: 'engine', + run: { engine: 'gepa', maxEvaluations: 1, maxProposerCostUsd: 1 }, + }, + optimizer: testOptimizer, + approveSensitiveProfileSurface: true, + runner: failingRunner('runner must not start'), + }) + + expect(() => + method({ + profile: { name: 'optimizer-fixture', mcp }, + evaluationRef, + surface: 'mcp', + baselineSurface: JSON.stringify(mcp), + baselineValue: mcp, + findings: [], + }), + ).not.toThrow() + }) + it.each([ [ 'gepa', - 'gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f', + 'gepa[full]==0.1.4', () => officialGepa({ objective: 'Improve the agent prompt.', diff --git a/src/improvement/official-optimizers.ts b/src/improvement/official-optimizers.ts index ca92e9e6..5ee103c0 100644 --- a/src/improvement/official-optimizers.ts +++ b/src/improvement/official-optimizers.ts @@ -10,16 +10,17 @@ import { } from '@tangle-network/agent-eval/campaign' import { canonicalCandidateDigest } from '../candidate-execution/digest' import { ConfigError } from '../errors' -import { defaultRedactor } from '../redact' +import { defaultRedactor, type Redactor, resolveRedactor } from '../redact' import type { ImproveMethodContext, ImproveMethodFactory } from './improve' const defaultMaxFindingsChars = 50_000 const pythonClientDocs = 'https://github.com/tangle-network/agent-eval/tree/main/clients/python' -const gepaInstall = - '`python -m pip install agent-eval-rpc`, then ' + +const bridgeInstall = '`python -m pip install "agent-eval-rpc==0.126.5"`' +const gepaWheelInstall = '`python -m pip install "gepa[full]==0.1.4"`' +const gepaSourceInstall = '`python -m pip install "gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f"`' const skillOptInstall = - '`python -m pip install agent-eval-rpc`, then ' + + `${bridgeInstall}, then ` + '`python -m pip install "skillopt @ git+https://github.com/microsoft/SkillOpt.git@61735e3922efc2b90c6d6cab561e62e98452ca90"`' /** Runtime context appended to an official optimizer's own configuration. */ @@ -30,6 +31,19 @@ export interface OfficialOptimizerContextOptions { includeFindings?: boolean /** Reject oversized serialized findings before starting Python. Default 50,000 characters. */ maxFindingsChars?: number + /** + * Redact caller-supplied context and descriptors before they leave Runtime. + * The built-in redactor is the default. Pass `false` only for public data + * that has already been reviewed. + */ + redact?: Redactor | false + /** + * Confirm that structurally sensitive candidate fields contain only safe + * references or public values. Required for fields such as MCP env, headers, + * URLs, metadata, and extensions because candidate bytes cannot be redacted + * without changing what the optimizer measures. + */ + approveSensitiveProfileSurface?: boolean } /** Official GEPA configuration plus bounded Runtime findings context. */ @@ -54,7 +68,11 @@ export class OfficialOptimizerUnavailableError extends ConfigError { const detail = cause instanceof Error ? cause.message : String(cause) const install = optimizer === 'gepa' - ? `Install the Python bridge and pinned GEPA source: ${gepaInstall}.` + ? [ + `Install the Python bridge: ${bridgeInstall}.`, + `The direct GEPA engine uses the published wheel: ${gepaWheelInstall}.`, + `Composed recipes and source-only engines use the tested source revision: ${gepaSourceInstall}.`, + ].join(' ') : `Install Microsoft SkillOpt: ${skillOptInstall}.` super( [ @@ -85,17 +103,24 @@ export function officialGepa { - assertSafeOptimizerSurface('officialGepa', context) + assertSafeOptimizerSurface('officialGepa', context, approveSensitiveProfileSurface) return withDependencyHelp( 'gepa', context.evaluationRef, + redactor, + redactionPolicyRef, gepaOptimizationMethod({ ...config, + objective, evaluationId: context.evaluationRef, background: methodBackground({ context, @@ -103,13 +128,30 @@ export function officialGepa - redactOptimizerEvidence(describeScenario ? describeScenario(scenario) : scenario), - describeArtifact: (artifact, scenario) => - redactOptimizerEvidence( - describeArtifact ? describeArtifact(artifact, scenario) : artifact, - ), + ...(describeScenario + ? { + describeScenario: (scenario) => + redactOptimizerEvidence( + 'officialGepa', + 'scenario descriptor', + describeScenario(scenario), + redactor, + ), + } + : {}), + ...(describeArtifact + ? { + describeArtifact: (artifact, scenario) => + redactOptimizerEvidence( + 'officialGepa', + 'artifact descriptor', + describeArtifact(artifact, scenario), + redactor, + ), + } + : {}), }), ) } @@ -128,17 +170,24 @@ export function officialSkillOpt< maxFindingsChars, describeScenario, describeArtifact, + redact, + approveSensitiveProfileSurface = false, ...config } = options + const redactor = resolveRedactor(redact) + const redactionPolicyRef = optimizerRedactionPolicyRef(redact) assertMaxFindingsChars('officialSkillOpt', maxFindingsChars) - assertSafeCallerText('officialSkillOpt', 'objective', config.objective) + const objective = redactOptimizerText('officialSkillOpt', 'objective', config.objective, redactor) return (context) => { - assertSafeOptimizerSurface('officialSkillOpt', context) + assertSafeOptimizerSurface('officialSkillOpt', context, approveSensitiveProfileSurface) return withDependencyHelp( 'skillopt', context.evaluationRef, + redactor, + redactionPolicyRef, skillOptOptimizationMethod({ ...config, + objective, evaluationId: context.evaluationRef, background: methodBackground({ context, @@ -146,13 +195,30 @@ export function officialSkillOpt< includeFindings, maxFindingsChars, label: 'officialSkillOpt', + redactor, }), - describeScenario: (scenario) => - redactOptimizerEvidence(describeScenario ? describeScenario(scenario) : scenario), - describeArtifact: (artifact, scenario) => - redactOptimizerEvidence( - describeArtifact ? describeArtifact(artifact, scenario) : artifact, - ), + ...(describeScenario + ? { + describeScenario: (scenario) => + redactOptimizerEvidence( + 'officialSkillOpt', + 'scenario descriptor', + describeScenario(scenario), + redactor, + ), + } + : {}), + ...(describeArtifact + ? { + describeArtifact: (artifact, scenario) => + redactOptimizerEvidence( + 'officialSkillOpt', + 'artifact descriptor', + describeArtifact(artifact, scenario), + redactor, + ), + } + : {}), }), ) } @@ -170,6 +236,7 @@ function methodBackground(options: { includeFindings: boolean maxFindingsChars: number | undefined label: string + redactor: Redactor }): string { const { context, @@ -177,17 +244,28 @@ function methodBackground(options: { includeFindings, maxFindingsChars = defaultMaxFindingsChars, label, + redactor, } = options - assertSafeCallerText(label, 'background', background) - assertSafeCallerText(label, 'profile name', context.profile.name) + const safeBackground = + background === undefined + ? undefined + : redactOptimizerText(label, 'background', background, redactor) + const safeProfileName = + context.profile.name === undefined + ? undefined + : redactOptimizerText(label, 'profile name', context.profile.name, redactor) const sections = [ - background?.trim(), - `Agent profile: ${context.profile.name}. Surface: ${context.surface}.`, + safeBackground?.trim(), + safeProfileName + ? `Agent profile: ${safeProfileName}. Surface: ${context.surface}.` + : `Agent surface: ${context.surface}.`, ].filter((value): value is string => Boolean(value)) if (includeFindings && context.findings.length > 0) { let serialized: string try { - serialized = canonicalJson(defaultRedactor(context.findings)) + serialized = canonicalJson( + redactOptimizerEvidence(label, 'findings', context.findings, redactor), + ) } catch (cause) { throw new ConfigError(`${label}: findings must be JSON-serializable`, { cause }) } @@ -201,7 +279,11 @@ function methodBackground(options: { return sections.join('\n\n') } -function assertSafeOptimizerSurface(label: string, context: ImproveMethodContext): void { +function assertSafeOptimizerSurface( + label: string, + context: ImproveMethodContext, + approveSensitiveProfileSurface: boolean, +): void { const redactedValue = defaultRedactor(context.baselineValue) const redactedSurface = defaultRedactor(context.baselineSurface) if ( @@ -213,31 +295,90 @@ function assertSafeOptimizerSurface(label: string, context: ImproveMethodContext 'Store live credentials as provider references, or remove private data before starting an external optimizer.', ) } + const sensitivePaths = sensitiveProfileSurfacePaths(context.baselineValue) + if (!approveSensitiveProfileSurface && sensitivePaths.length > 0) { + throw new ConfigError( + `${label}: the selected profile surface contains fields that may carry private values: ` + + `${sensitivePaths.slice(0, 8).join(', ')}. Remove them, replace values with safe references, ` + + 'or set approveSensitiveProfileSurface: true after reviewing the exact candidate bytes.', + ) + } } -function redactOptimizerEvidence(value: unknown): unknown { - return defaultRedactor(value) +function sensitiveProfileSurfacePaths(value: unknown): string[] { + const paths: string[] = [] + const seen = new WeakSet() + const visit = (current: unknown, path: string): void => { + if (current === null || typeof current !== 'object') return + if (seen.has(current)) return + seen.add(current) + if (Array.isArray(current)) { + current.forEach((child, index) => { + visit(child, `${path}[${index}]`) + }) + return + } + for (const [key, child] of Object.entries(current as Record)) { + const childPath = `${path}.${key}` + if (['env', 'headers', 'url', 'metadata', 'extensions'].includes(key.toLowerCase())) { + paths.push(childPath) + continue + } + visit(child, childPath) + } + } + visit(value, '$') + return paths } -function redactJudgeScore(score: JudgeScore): JudgeScore { - const notes = defaultRedactor(score.notes) +function redactOptimizerEvidence( + label: string, + field: string, + value: unknown, + redactor: Redactor, +): unknown { + try { + return redactor(value) + } catch (cause) { + throw new ConfigError(`${label}: ${field} redaction failed`, { cause }) + } +} + +function redactOptimizerText( + label: string, + field: string, + value: string, + redactor: Redactor, +): string { + const redacted = redactOptimizerEvidence(label, field, value, redactor) + if (typeof redacted !== 'string' || !redacted.trim()) { + throw new ConfigError(`${label}: ${field} redaction must return a non-empty string`) + } + return redacted +} + +function redactJudgeScore(score: JudgeScore, redactor: Redactor): JudgeScore { + const notes = redactOptimizerEvidence('official optimizer', 'judge notes', score.notes, redactor) return { ...score, notes: typeof notes === 'string' ? notes : '[redacted]', } } -function assertSafeCallerText(label: string, field: string, value: string | undefined): void { - if (value !== undefined && defaultRedactor(value) !== value) { - throw new ConfigError( - `${label}: ${field} contains a common credential or private value. Sanitize it before starting an external optimizer.`, - ) - } +function optimizerRedactionPolicyRef(redact: Redactor | false | undefined): string { + if (redact === undefined) return 'default-redactor' + if (redact === false) return 'caller-approved-raw' + return canonicalCandidateDigest({ + kind: 'caller-redactor', + source: Function.prototype.toString.call(redact), + }) } function withDependencyHelp( optimizer: 'gepa' | 'skillopt', evaluationRef: ImproveMethodContext['evaluationRef'], + redactor: Redactor, + redactionPolicyRef: string, method: OptimizationMethod, ): OptimizationMethod { return { @@ -252,10 +393,10 @@ function withDependencyHelp[0]) { - return redactJudgeScore(await judge.score(scoreInput)) + return redactJudgeScore(await judge.score(scoreInput), redactor) }, }), ) diff --git a/src/improvement/official-packages.test.ts b/src/improvement/official-packages.test.ts new file mode 100644 index 00000000..03023c80 --- /dev/null +++ b/src/improvement/official-packages.test.ts @@ -0,0 +1,242 @@ +import { execFileSync } from 'node:child_process' +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' + +const python = process.env.AGENT_EVAL_TEST_PYTHON +const runDirs: string[] = [] + +afterEach(() => { + for (const runDir of runDirs.splice(0)) { + rmSync(runDir, { recursive: true, force: true }) + } +}) + +function runDir(): string { + const value = mkdtempSync(join(tmpdir(), 'runtime-official-package-')) + runDirs.push(value) + return value +} + +function requiredPython(): string { + if (!python) throw new Error('AGENT_EVAL_TEST_PYTHON is required') + return python +} + +function runPython(script: string, args: readonly string[] = []): unknown { + const stdout = execFileSync(requiredPython(), ['-c', script, ...args], { + encoding: 'utf8', + env: { + ...process.env, + PYTHONDONTWRITEBYTECODE: '1', + }, + timeout: 60_000, + }) + return JSON.parse(stdout.trim()) as unknown +} + +function inspectBridge(module: 'gepa_bridge' | 'skillopt_bridge'): Record { + const root = runDir() + const inputPath = join(root, 'input.json') + const outputPath = join(root, 'output.json') + writeFileSync( + inputPath, + JSON.stringify({ + operation: 'inspect', + ...(module === 'gepa_bridge' ? { engineModules: [] } : {}), + }), + ) + execFileSync( + requiredPython(), + ['-m', `agent_eval_rpc.${module}`, '--input', inputPath, '--output', outputPath], + { + encoding: 'utf8', + env: { + ...process.env, + PYTHONDONTWRITEBYTECODE: '1', + }, + timeout: 60_000, + }, + ) + return JSON.parse(readFileSync(outputPath, 'utf8')) as Record +} + +describe('documented official Python packages', () => { + it.skipIf(!python)('loads the released GEPA 0.1.4 API and performs real work', () => { + const stateRoot = runDir() + const result = runPython( + ` +import contextlib +import io +import json +import sys +from importlib.metadata import version +from pathlib import Path +from gepa.optimize_anything import EngineConfig, GEPAConfig, ReflectionConfig, optimize_anything +from gepa.utils import ScoreThresholdStopper + +class DeterministicModel: + def __init__(self): + self.calls = 0 + + def __call__(self, prompt): + self.calls += 1 + return "ALWAYS_RETURN_READY" + +evaluations = [] + +def evaluate(candidate, example): + evaluations.append([candidate, example["id"]]) + score = 1.0 if "ALWAYS_RETURN_READY" in candidate else 0.0 + return score, {"feedback": "Add ALWAYS_RETURN_READY."} + +root = Path(sys.argv[1]) +model = DeterministicModel() +with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + result = optimize_anything( + "BASELINE", + evaluator=evaluate, + dataset=[{"id": "train"}], + valset=[{"id": "selection"}], + objective="Add the required response rule.", + config=GEPAConfig( + engine=EngineConfig( + capture_stdio=False, + max_metric_calls=4, + max_workers=1, + parallel=False, + raise_on_exception=True, + run_dir=str(root / "state"), + seed=7, + use_cloudpickle=False, + ), + reflection=ReflectionConfig( + reflection_lm=model, + reflection_minibatch_size=1, + skip_perfect_score=False, + ), + stop_callbacks=[ScoreThresholdStopper(1.0)], + ), + ) + +print(json.dumps({ + "version": version("gepa"), + "bestCandidate": result.best_candidate, + "bestScore": result.val_aggregate_scores[result.best_idx], + "totalEvaluations": result.total_metric_calls, + "modelCalls": model.calls, + "evaluations": evaluations, + "stateBytes": (root / "state" / "gepa_state.bin").stat().st_size, + "candidateBytes": (root / "state" / "candidates.json").stat().st_size, +})) +`, + [stateRoot], + ) + + expect(result).toEqual({ + version: '0.1.4', + bestCandidate: 'ALWAYS_RETURN_READY', + bestScore: 1, + totalEvaluations: 4, + modelCalls: 1, + evaluations: [ + ['BASELINE', 'selection'], + ['BASELINE', 'train'], + ['ALWAYS_RETURN_READY', 'train'], + ['ALWAYS_RETURN_READY', 'selection'], + ], + stateBytes: expect.any(Number), + candidateBytes: expect.any(Number), + }) + expect((result as { stateBytes: number }).stateBytes).toBeGreaterThan(0) + expect((result as { candidateBytes: number }).candidateBytes).toBeGreaterThan(0) + + const inspected = inspectBridge('gepa_bridge') as { + runtime: { + bridge: { package: string; version: string; sourceSha256: string } + optimizer: { package: string; version: string; sourceSha256: string } + } + } + expect(inspected.runtime.bridge).toMatchObject({ + package: 'agent-eval-rpc', + version: '0.126.5', + sourceSha256: expect.stringMatching(/^[a-f0-9]{64}$/), + }) + expect(inspected.runtime.optimizer).toMatchObject({ + package: 'gepa', + version: '0.1.4', + sourceSha256: expect.stringMatching(/^[a-f0-9]{64}$/), + }) + }) + + it.skipIf(!python)('loads the pinned SkillOpt trainer and all required prompt files', () => { + const result = runPython(` +import json +from importlib import resources +from importlib.metadata import distribution, version +from skillopt.engine.trainer import ReflACTTrainer + +prompt_root = resources.files("skillopt.prompts") +prompts = sorted( + entry.name + for entry in prompt_root.iterdir() + if entry.is_file() and entry.name.endswith(".md") and entry.read_text().strip() +) +direct_url = json.loads(distribution("skillopt").read_text("direct_url.json") or "{}") +print(json.dumps({ + "version": version("skillopt"), + "trainer": ReflACTTrainer.__name__, + "prompts": prompts, + "revision": direct_url.get("vcs_info", {}).get("commit_id"), +})) +`) + + expect(result).toEqual({ + version: '0.2.0', + trainer: 'ReflACTTrainer', + prompts: [ + 'analyst_error.md', + 'analyst_error_full_rewrite.md', + 'analyst_error_rewrite.md', + 'analyst_success.md', + 'analyst_success_full_rewrite.md', + 'analyst_success_rewrite.md', + 'lr_autonomous.md', + 'merge_failure.md', + 'merge_failure_full_rewrite.md', + 'merge_failure_rewrite.md', + 'merge_final.md', + 'merge_final_full_rewrite.md', + 'merge_final_rewrite.md', + 'merge_success.md', + 'merge_success_full_rewrite.md', + 'merge_success_rewrite.md', + 'meta_skill.md', + 'ranking.md', + 'ranking_rewrite.md', + 'rewrite_skill.md', + 'slow_update.md', + ], + revision: '61735e3922efc2b90c6d6cab561e62e98452ca90', + }) + + const inspected = inspectBridge('skillopt_bridge') as { + runtime: { + bridge: { package: string; version: string; sourceSha256: string } + optimizer: { package: string; version: string; revision: string; sourceSha256: string } + } + } + expect(inspected.runtime.bridge).toMatchObject({ + package: 'agent-eval-rpc', + version: '0.126.5', + sourceSha256: expect.stringMatching(/^[a-f0-9]{64}$/), + }) + expect(inspected.runtime.optimizer).toMatchObject({ + package: 'skillopt', + version: '0.2.0', + revision: '61735e3922efc2b90c6d6cab561e62e98452ca90', + sourceSha256: expect.stringMatching(/^[a-f0-9]{64}$/), + }) + }) +}) diff --git a/src/improvement/official-runtime-integration.test.ts b/src/improvement/official-runtime-integration.test.ts new file mode 100644 index 00000000..3156a53c --- /dev/null +++ b/src/improvement/official-runtime-integration.test.ts @@ -0,0 +1,261 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import type { JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign' +import { type AgentProfile, canonicalCandidateDigest } from '@tangle-network/agent-interface' +import { expect, it } from 'vitest' +import { improve } from './improve' +import { officialGepa, officialSkillOpt } from './official-optimizers' +import { startOptimizerModelServer } from './official-test-support' + +interface OptimizerScenario extends Scenario { + kind: 'official' + prompt: string +} + +interface OptimizerArtifact { + text: string +} + +const officialPython = process.env.AGENT_EVAL_TEST_PYTHON?.trim() +const officialIt = officialPython ? it : it.skip + +officialIt( + 'runs Runtime through the installed GEPA package and promotes its candidate', + async () => { + const runDir = await mkdtemp(join(tmpdir(), 'runtime-official-gepa-')) + const modelServer = await startOptimizerModelServer('```\n{"k":2}\n```') + const profile: AgentProfile = { + name: 'official-gepa-runtime', + prompt: { systemPrompt: '{"k":1}' }, + } + + try { + const result = await improve(profile, { + surface: 'prompt', + executionRef: canonicalCandidateDigest({ fixture: 'official-gepa-runtime' }), + method: officialGepa({ + objective: 'Return a JSON configuration whose k value is 2.', + recipe: { + kind: 'engine', + run: { + engine: 'gepa', + maxEvaluations: 4, + maxProposerCostUsd: 1, + maxConcurrency: 1, + stopAtScore: 1, + engineConfig: { + engine: { + capture_stdio: false, + max_workers: 1, + parallel: false, + raise_on_exception: true, + seed: 7, + }, + reflection: { + reflection_minibatch_size: 1, + skip_perfect_score: false, + }, + }, + }, + }, + optimizer: optimizerModel(modelServer.baseUrl, 2_000), + runner: { + command: officialPython!, + args: ['-m', 'agent_eval_rpc.gepa_bridge'], + }, + }), + trainScenarios: [{ id: 'train', kind: 'official', prompt: 'Set k to 2.' }], + selectionScenarios: [{ id: 'selection', kind: 'official', prompt: 'Set k to 2.' }], + testScenarios: [ + { id: 'test-1', kind: 'official', prompt: 'Set k to 2.' }, + { id: 'test-2', kind: 'official', prompt: 'Set k to 2 again.' }, + ], + agent: async (candidate) => ({ text: candidate.prompt?.systemPrompt ?? '' }), + judges: [kEqualsTwoJudge], + runDir, + costCeiling: 1, + seed: 7, + resamples: 40, + expectUsage: 'off', + optimizationRunOptions: { expectUsage: 'off' }, + }) + + expect(modelServer.requests).toHaveLength(1) + expect(modelServer.requests[0]).toMatchObject({ + authorization: 'Bearer provider-secret', + path: '/v1/chat/completions', + body: { model: 'local-model', max_tokens: 2_000 }, + }) + expect(JSON.parse(String(result.candidate.value))).toEqual({ k: 2 }) + expect(result.decision).toBe('ship') + expect(result.provenance).toMatchObject({ + source: { package: 'gepa', evidence: 'observed' }, + bridge: { package: 'agent-eval-rpc', evidence: 'observed' }, + tokenUsage: { + inputTokens: 11, + outputTokens: 13, + totalTokens: 24, + calls: 1, + }, + }) + expect(result.cost.accountingComplete).toBe(true) + } finally { + await modelServer.close() + await rm(runDir, { recursive: true, force: true }) + } + }, + 300_000, +) + +officialIt( + 'runs Runtime through the installed SkillOpt package and promotes its candidate', + async () => { + const runDir = await mkdtemp(join(tmpdir(), 'runtime-official-skillopt-')) + const modelServer = await startOptimizerModelServer( + JSON.stringify({ + batch_size: 1, + failure_summary: [ + { + count: 1, + description: 'The required response rule is absent.', + failure_type: 'missing_rule', + }, + ], + patch: { + edits: [ + { + content: '\n\n## Required Rule\nALWAYS_RETURN_READY\n', + op: 'append', + }, + ], + reasoning: 'Add the missing response rule.', + }, + }), + ) + const profile: AgentProfile = { + name: 'official-skillopt-runtime', + resources: { + failOnError: true, + skills: [{ kind: 'inline', name: 'answering', content: '# Answering\nAnswer normally.\n' }], + }, + } + + try { + const result = await improve(profile, { + surface: 'skills', + skills: { resourceName: 'answering' }, + executionRef: canonicalCandidateDigest({ fixture: 'official-skillopt-runtime' }), + method: officialSkillOpt({ + objective: 'Add the required response rule.', + trainer: { + epochs: 1, + batchSize: 1, + accumulation: 1, + editBudget: 1, + minEditBudget: 1, + analystWorkers: 1, + minibatchSize: 1, + maxAnalystRounds: 1, + evaluationWorkers: 1, + }, + optimizer: optimizerModel(modelServer.baseUrl, 256), + maxEvaluations: 3, + runner: { command: officialPython! }, + }), + trainScenarios: [{ id: 'train', kind: 'official', prompt: 'Return READY.' }], + selectionScenarios: [{ id: 'selection', kind: 'official', prompt: 'Return READY.' }], + testScenarios: [ + { id: 'test-1', kind: 'official', prompt: 'Return READY.' }, + { id: 'test-2', kind: 'official', prompt: 'Return READY again.' }, + ], + agent: async (candidate) => { + const skill = candidate.resources?.skills?.find( + (entry) => entry.kind === 'inline' && entry.name === 'answering', + ) + return { text: skill?.kind === 'inline' ? skill.content : '' } + }, + judges: [requiredRuleJudge], + runDir, + costCeiling: 1, + seed: 7, + resamples: 40, + expectUsage: 'off', + optimizationRunOptions: { expectUsage: 'off' }, + }) + + expect(modelServer.requests).toHaveLength(1) + expect(modelServer.requests[0]).toMatchObject({ + authorization: 'Bearer provider-secret', + path: '/v1/chat/completions', + body: { model: 'local-model', max_tokens: 256 }, + }) + expect(String(result.candidate.value)).toContain('ALWAYS_RETURN_READY') + expect(result.decision).toBe('ship') + expect(result.provenance?.tokenUsage).toEqual({ + inputTokens: 11, + outputTokens: 13, + totalTokens: 24, + calls: 1, + }) + expect(result.cost.accountingComplete).toBe(true) + } finally { + await modelServer.close() + await rm(runDir, { recursive: true, force: true }) + } + }, + 60_000, +) + +const kEqualsTwoJudge: JudgeConfig = { + name: 'k-equals-two', + dimensions: [{ key: 'correctness', description: 'candidate sets k to 2' }], + judgeVersion: 'k-equals-two/1', + score: ({ artifact }) => { + let score = 0 + try { + const candidate = JSON.parse(artifact.text) as { k?: unknown } + score = candidate.k === 2 ? 1 : 0 + } catch { + score = 0 + } + return { + dimensions: { correctness: score }, + composite: score, + notes: score ? '' : 'The candidate must be JSON with k set to 2.', + } + }, +} + +const requiredRuleJudge: JudgeConfig = { + name: 'required-rule', + dimensions: [{ key: 'correctness', description: 'candidate contains required rule' }], + judgeVersion: 'required-rule/1', + score: ({ artifact }) => { + const score = artifact.text.includes('ALWAYS_RETURN_READY') ? 1 : 0 + return { + dimensions: { correctness: score }, + composite: score, + notes: score ? '' : 'The required response rule is absent.', + } + }, +} + +function optimizerModel(baseUrl: string, maxOutputTokensPerRequest: number) { + return { + model: 'local-model', + baseUrl, + apiKey: 'provider-secret', + budget: { + maxCostUsd: 1, + maxRequests: 10, + maxRequestBytes: 100_000, + maxResponseBytes: 100_000, + maxOutputTokensPerRequest, + pricing: { + inputUsdPerMillion: 1, + outputUsdPerMillion: 2, + }, + }, + } +} diff --git a/src/improvement/official-test-support.ts b/src/improvement/official-test-support.ts new file mode 100644 index 00000000..2302376c --- /dev/null +++ b/src/improvement/official-test-support.ts @@ -0,0 +1,69 @@ +import { createServer } from 'node:http' + +export interface CapturedModelRequest { + authorization: string | undefined + path: string | undefined + body: Record +} + +export async function startOptimizerModelServer(content: string): Promise<{ + baseUrl: string + requests: CapturedModelRequest[] + close: () => Promise +}> { + const requests: CapturedModelRequest[] = [] + const server = createServer(async (request, response) => { + const chunks: Buffer[] = [] + for await (const chunk of request) chunks.push(Buffer.from(chunk)) + const body = JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record + requests.push({ + authorization: request.headers.authorization, + path: request.url, + body, + }) + response.writeHead(200, { 'content-type': 'application/json' }) + response.end( + JSON.stringify({ + choices: [ + { + finish_reason: 'stop', + index: 0, + message: { content, role: 'assistant' }, + }, + ], + created: 0, + id: 'chatcmpl-local', + model: 'local-model', + object: 'chat.completion', + usage: { + completion_tokens: 13, + prompt_tokens: 11, + total_tokens: 24, + }, + }), + ) + }) + + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', () => { + server.off('error', reject) + resolve() + }) + }) + const address = server.address() + if (!address || typeof address === 'string') { + throw new Error('optimizer model server failed to bind') + } + + return { + baseUrl: `http://127.0.0.1:${address.port}/v1`, + requests, + close: async () => { + server.closeAllConnections?.() + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())) + }) + }, + } +} diff --git a/src/improvement/raw-trace-distiller.ts b/src/improvement/raw-trace-distiller.ts index 493359ee..8c154b86 100644 --- a/src/improvement/raw-trace-distiller.ts +++ b/src/improvement/raw-trace-distiller.ts @@ -77,7 +77,7 @@ interface CellTrace { * * Drop-in for `analyzeGeneration` on `improve({ surface: 'code' })`: * - * await improve(profile, { + * await improve({ * surface: 'code', * findings: seedFindings, * code: { repoRoot }, diff --git a/src/intelligence/improvement-cycle.ts b/src/intelligence/improvement-cycle.ts index 2d4df1fb..6f3cd2c6 100644 --- a/src/intelligence/improvement-cycle.ts +++ b/src/intelligence/improvement-cycle.ts @@ -78,6 +78,12 @@ import { buildAgentImprovementActivationTargets, deriveChangedSurfaces, } from './improvement-surfaces' +import { + assertNoCallerOptimizationReceipt, + attachOptimizationActivationReceipt, + createOptimizationActivationReceipt, + optimizationActivationReceiptFromMetadata, +} from './optimization-receipt' export type { AgentImprovementActivation, @@ -327,19 +333,26 @@ export function createAgentImprovementMeasuredComparison( export async function proposeAgentImprovement( options: ProposeAgentImprovementOptions, ): Promise> { + assertNoCallerOptimizationReceipt(options.metadata) const analysis = await runAnalystLoop({ ...options.analysis, runId: options.runId }) const findings = assertNoJudgeVerdict( analysis.analystResult.findings, 'proposeAgentImprovement findings', ) - const improvement = await improve(options.profile, { + const improvementInput = { ...options.improvement, findings: [...(options.improvement.findings ?? []), ...findings], - }) + } + const improvement = + improvementInput.surface === 'code' + ? await improve(improvementInput) + : await improve(options.profile, improvementInput) try { if (improvement.decision !== 'ship') { throw new Error('agent improvement search did not produce a promotable candidate') } + const optimizationReceipt = + improvement.mode === 'method' ? createOptimizationActivationReceipt(improvement) : undefined const experiment = sealAgentImprovementExperiment( await options.buildExperiment({ analysis, improvement }), improvement, @@ -353,7 +366,13 @@ export async function proposeAgentImprovement { const redacted = defaultRedactor('judge note: {"password":"hunter2"} token=plain-secret') expect(redacted).toBe('judge note: {"password":[redacted]} token=[redacted]') }) + + it('redacts a complete bearer token after an authorization label', () => { + const redacted = defaultRedactor('Authorization: Bearer abcdefghijklmnop') + expect(redacted).toBe('Authorization: [redacted]') + }) }) describe('createIntelligenceClient / traceRun — Observe', () => { diff --git a/src/intelligence/optimization-receipt.ts b/src/intelligence/optimization-receipt.ts new file mode 100644 index 00000000..c5f0dd6e --- /dev/null +++ b/src/intelligence/optimization-receipt.ts @@ -0,0 +1,288 @@ +import type { + OptimizationMethodComparison, + OptimizationPackageSource, + OptimizationTokenUsage, +} from '@tangle-network/agent-eval/campaign' +import { + type AgentCandidateJsonValue, + type AgentImprovementMeasuredComparison, + type AgentProfile, + agentProfileModelHintsSchema, + type Sha256Digest, +} from '@tangle-network/agent-interface' + +import { + canonicalCandidateDigest, + canonicalCandidateDocument, + immutableCandidateValue, + omitTopLevelDigest, +} from '../candidate-execution/digest' +import type { ImproveMethodResult } from '../improvement/improve' + +type OptimizationProvenance = NonNullable + +export interface OptimizationActivationReceipt { + kind: 'optimization-activation-receipt' + method: string + source: OptimizationPackageSource + bridge?: OptimizationPackageSource + modules?: OptimizationProvenance['modules'] + python?: OptimizationProvenance['python'] + model?: { + role: 'candidate' + identity: NonNullable + } + usage: { + evaluations: number + tokens?: OptimizationTokenUsage + } + cost: { + totalUsd: number + accountingComplete: boolean + incompleteReasons: string[] + } + invocation: { + runtimeInvocationId: string + optimizerRunId: string + compatibleOptimizerRunId?: string + resumed: boolean + artifactDir: string + } + developmentDataDigest: Sha256Digest + digest: Sha256Digest +} + +export const optimizationReceiptMetadataKey = 'optimizationReceipt' + +/** Build a detached receipt only for methods backed by an identified external optimizer. */ +export function createOptimizationActivationReceipt( + improvement: ImproveMethodResult, +): OptimizationActivationReceipt | undefined { + const provenance = improvement.provenance + if (!provenance) return undefined + + return canonicalCandidateDocument({ + kind: 'optimization-activation-receipt', + method: improvement.method, + source: provenance.source, + ...(provenance.bridge ? { bridge: provenance.bridge } : {}), + ...(provenance.modules ? { modules: provenance.modules } : {}), + ...(provenance.python ? { python: provenance.python } : {}), + ...(improvement.candidate.profile.model + ? { + model: { + role: 'candidate', + identity: improvement.candidate.profile.model, + }, + } + : {}), + usage: { + evaluations: provenance.evaluationCount, + ...(provenance.tokenUsage ? { tokens: provenance.tokenUsage } : {}), + }, + cost: { + totalUsd: improvement.cost.totalCostUsd, + accountingComplete: improvement.cost.accountingComplete, + incompleteReasons: improvement.cost.incompleteReasons, + }, + invocation: { + runtimeInvocationId: improvement.lineage.invocationId, + optimizerRunId: provenance.runId, + ...(provenance.compatibleRunId + ? { compatibleOptimizerRunId: provenance.compatibleRunId } + : {}), + resumed: provenance.resumed, + artifactDir: provenance.artifactDir, + }, + developmentDataDigest: improvement.lineage.developmentSplitDigest, + }).value +} + +/** Add Runtime-owned optimizer evidence without aliasing caller metadata. */ +export function attachOptimizationActivationReceipt( + metadata: AgentImprovementMeasuredComparison['metadata'], + receipt: OptimizationActivationReceipt, +): NonNullable { + assertNoCallerOptimizationReceipt(metadata) + return immutableCandidateValue({ + ...(metadata ?? {}), + [optimizationReceiptMetadataKey]: receipt as unknown as AgentCandidateJsonValue, + }) +} + +export function assertNoCallerOptimizationReceipt( + metadata: AgentImprovementMeasuredComparison['metadata'], +): void { + if (metadata && Object.hasOwn(metadata, optimizationReceiptMetadataKey)) { + throw new Error(`candidate metadata reserves '${optimizationReceiptMetadataKey}' for Runtime`) + } +} + +/** Read and verify the optimizer evidence carried by a measured proposal. */ +export function optimizationActivationReceiptFromMetadata( + metadata: AgentImprovementMeasuredComparison['metadata'], +): OptimizationActivationReceipt | undefined { + const value = metadata?.[optimizationReceiptMetadataKey] + if (value === undefined) return undefined + return immutableCandidateValue(parseOptimizationActivationReceipt(value)) +} + +function parseOptimizationActivationReceipt( + value: AgentCandidateJsonValue, +): OptimizationActivationReceipt { + if (!isRecord(value) || value.kind !== 'optimization-activation-receipt') { + throw new Error('optimization receipt must be an optimization-activation-receipt') + } + if ( + !isNonEmptyString(value.method) || + !isPackageSource(value.source) || + (value.bridge !== undefined && !isPackageSource(value.bridge)) || + !isModules(value.modules) || + !isPythonRuntime(value.python) || + !isCandidateModel(value.model) || + !isUsage(value.usage) || + !isCost(value.cost) || + !isInvocation(value.invocation) || + !isSha256Digest(value.developmentDataDigest) || + !isSha256Digest(value.digest) + ) { + throw new Error('optimization receipt contains invalid evidence') + } + const receipt = value as unknown as OptimizationActivationReceipt + if (canonicalCandidateDigest(omitTopLevelDigest(receipt)) !== receipt.digest) { + throw new Error('optimization receipt digest does not match its evidence') + } + return receipt +} + +function isPackageSource(value: unknown): value is OptimizationPackageSource { + if ( + !isRecord(value) || + value.kind !== 'package' || + (value.evidence !== 'observed' && value.evidence !== 'declared') || + !isNonEmptyString(value.package) || + !isNonEmptyString(value.version) + ) { + return false + } + return ( + isOptionalNonEmptyString(value.sourceUrl) && + isOptionalNonEmptyString(value.revision) && + (value.sourceSha256 === undefined || + (typeof value.sourceSha256 === 'string' && /^[0-9a-f]{64}$/.test(value.sourceSha256))) + ) +} + +function isModules(value: unknown): boolean { + return ( + value === undefined || + (Array.isArray(value) && + value.every( + (module) => + isRecord(module) && + isNonEmptyString(module.module) && + typeof module.sourceSha256 === 'string' && + /^[0-9a-f]{64}$/.test(module.sourceSha256), + )) + ) +} + +function isPythonRuntime(value: unknown): boolean { + return ( + value === undefined || + (isRecord(value) && isNonEmptyString(value.implementation) && isNonEmptyString(value.version)) + ) +} + +function isCandidateModel(value: unknown): boolean { + return ( + value === undefined || + (isRecord(value) && + value.role === 'candidate' && + agentProfileModelHintsSchema.safeParse(value.identity).success) + ) +} + +function isUsage(value: unknown): boolean { + if (!isRecord(value) || !isNonNegativeInteger(value.evaluations)) return false + if (value.tokens === undefined) return true + const tokens = value.tokens + if (!isRecord(tokens)) return false + const inputTokens = tokens.inputTokens + const outputTokens = tokens.outputTokens + const totalTokens = tokens.totalTokens + const calls = tokens.calls + if ( + !isNonNegativeInteger(inputTokens) || + !isNonNegativeInteger(outputTokens) || + !isNonNegativeInteger(totalTokens) || + !isNonNegativeInteger(calls) + ) { + return false + } + if ( + !isOptionalNonNegativeInteger(tokens.cachedInputTokens) || + !isOptionalNonNegativeInteger(tokens.cacheWriteInputTokens) || + !isOptionalNonNegativeInteger(tokens.reasoningTokens) + ) { + return false + } + const cachedInputTokens = + typeof tokens.cachedInputTokens === 'number' ? tokens.cachedInputTokens : 0 + const cacheWriteInputTokens = + typeof tokens.cacheWriteInputTokens === 'number' ? tokens.cacheWriteInputTokens : 0 + const reasoningTokens = typeof tokens.reasoningTokens === 'number' ? tokens.reasoningTokens : 0 + return ( + totalTokens === inputTokens + outputTokens && + cachedInputTokens + cacheWriteInputTokens <= inputTokens && + reasoningTokens <= outputTokens && + (totalTokens === 0 || calls > 0) + ) +} + +function isCost(value: unknown): boolean { + return ( + isRecord(value) && + typeof value.totalUsd === 'number' && + Number.isFinite(value.totalUsd) && + value.totalUsd >= 0 && + typeof value.accountingComplete === 'boolean' && + Array.isArray(value.incompleteReasons) && + value.incompleteReasons.every((reason) => typeof reason === 'string') + ) +} + +function isInvocation(value: unknown): boolean { + return ( + isRecord(value) && + isNonEmptyString(value.runtimeInvocationId) && + isNonEmptyString(value.optimizerRunId) && + isOptionalNonEmptyString(value.compatibleOptimizerRunId) && + typeof value.resumed === 'boolean' && + isNonEmptyString(value.artifactDir) + ) +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value) +} + +function isNonEmptyString(value: unknown): value is string { + return typeof value === 'string' && value.trim().length > 0 +} + +function isOptionalNonEmptyString(value: unknown): boolean { + return value === undefined || isNonEmptyString(value) +} + +function isNonNegativeInteger(value: unknown): value is number { + return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 +} + +function isOptionalNonNegativeInteger(value: unknown): boolean { + return value === undefined || isNonNegativeInteger(value) +} + +function isSha256Digest(value: unknown): value is Sha256Digest { + return typeof value === 'string' && /^sha256:[0-9a-f]{64}$/.test(value) +} diff --git a/src/redact.ts b/src/redact.ts index 7cfb31bc..6942ab53 100644 --- a/src/redact.ts +++ b/src/redact.ts @@ -45,11 +45,11 @@ const secretAssignmentPattern = /** Scrub a single string of in-value secret/PII patterns. */ function scrubString(input: string): string { - let out = input.replace(secretAssignmentPattern, `$1${redactedMarker}`) + let out = input for (const pattern of valuePatterns) { out = out.replace(pattern, redactedMarker) } - return out + return out.replace(secretAssignmentPattern, `$1${redactedMarker}`) } /** @@ -87,13 +87,13 @@ function walk(value: unknown, seen: WeakSet, depth: number): unknown { } /** - * Resolve the redactor a client uses. A caller-supplied hook replaces the - * default entirely (the customer owns their PII rules); absent one, the - * built-in `defaultRedactor` runs. Returning `false` is the explicit opt-out — - * NO redaction, for a caller who has already sanitized upstream and wants raw - * fidelity. Opt-out is loud (an explicit `false`), never a silent default. + * Resolve the redactor a client uses. A caller-supplied hook handles + * domain-specific values first, then the built-in scrubber still removes + * common credentials and email addresses. Returning `false` is the explicit + * opt-out for already-reviewed public values. */ export function resolveRedactor(redact: Redactor | false | undefined): Redactor { if (redact === false) return (value) => value - return redact ?? defaultRedactor + if (!redact) return defaultRedactor + return (value) => defaultRedactor(redact(value)) } diff --git a/tests/helpers/improvement-method-fixture.ts b/tests/helpers/improvement-method-fixture.ts new file mode 100644 index 00000000..999f8835 --- /dev/null +++ b/tests/helpers/improvement-method-fixture.ts @@ -0,0 +1,170 @@ +import type { AnalystFinding } from '@tangle-network/agent-eval' +import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' +import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract' + +import { canonicalCandidateDigest } from '../../src/candidate-execution/digest' +import { + agentCandidateProfileAsAgentProfile, + freezeGenericAgentCandidateProfile, +} from '../../src/candidate-execution/profile' +import type { ImproveMethodFactory, ReadonlyAgentProfile } from '../../src/improvement' +import { + type AgentImprovementExperimentMaterial, + proposeAgentImprovement, +} from '../../src/intelligence/improvement-cycle' +import { redigestCandidateBundle } from './candidate-execution-fixture' +import { + type CandidateExperimentFixture, + createCandidateExperimentFixture, +} from './candidate-experiment-fixture' + +export const improvementFinding: AnalystFinding = { + schema_version: '1.0.0', + finding_id: 'finding-1', + analyst_id: 'improvement', + produced_at: '2026-07-10T00:00:00.000Z', + severity: 'high', + area: 'prompt', + claim: 'The agent omits the required answer.', + evidence_refs: [{ kind: 'span', id: 'span-1' }], + recommended_action: 'Return the measured answer.', + confidence: 0.9, + subject: 'agent-profile:prompt.systemPrompt', +} + +export interface ImprovementScenario extends Scenario { + kind: 'improvement-test' +} + +const improvementScenarios: ImprovementScenario[] = Array.from({ length: 12 }, (_, index) => ({ + id: `improvement-${index}`, + kind: 'improvement-test', +})) +const improvementTrain = improvementScenarios.slice(0, 4) +const improvementSelection = improvementScenarios.slice(4, 8) +const improvementTest = improvementScenarios.slice(8) + +const improvementAgent = async ( + profile: ReadonlyAgentProfile, + _scenario: ImprovementScenario, + context: DispatchContext, +): Promise => { + const paid = await context.cost.runPaidCall({ + channel: 'agent', + actor: 'improvement-cycle-test', + model: 'deterministic-test', + maximumCharge: { externallyEnforcedMaximumUsd: 0.0001 }, + execute: async () => profile.prompt?.systemPrompt ?? '', + receipt: () => ({ + model: 'deterministic-test', + inputTokens: 1, + outputTokens: 1, + actualCostUsd: 0.0001, + }), + }) + if (!paid.succeeded) throw paid.error + return paid.value +} + +const improvementJudge: JudgeConfig = { + name: 'exact-prompt', + dimensions: [{ key: 'quality', description: 'Candidate prompt selected.' }], + score: ({ artifact }) => { + const composite = artifact === 'PROMOTED' ? 1 : 0 + return { dimensions: { quality: composite }, composite, notes: '' } + }, +} + +export const improvementMethod: ImproveMethodFactory = (context) => ({ + name: 'deterministic-complete-method', + async optimize() { + if ( + !context.findings.some( + (item) => (item as { finding_id?: string }).finding_id === improvementFinding.finding_id, + ) + ) { + throw new Error('analysis findings did not reach the optimization method') + } + return { + winnerSurface: 'PROMOTED', + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + } + }, +}) + +export function improvementOptions() { + return { + surface: 'prompt' as const, + executionRef: canonicalCandidateDigest({ + agent: Function.prototype.toString.call(improvementAgent), + judge: Function.prototype.toString.call(improvementJudge.score), + method: Function.prototype.toString.call(improvementMethod), + }), + method: improvementMethod, + trainScenarios: improvementTrain, + selectionScenarios: improvementSelection, + testScenarios: improvementTest, + judges: [improvementJudge], + agent: improvementAgent, + runDir: `mem://improvement-cycle-${Math.random()}`, + storage: inMemoryCampaignStorage(), + resamples: 40, + confidence: 0.95, + } +} + +export function candidateExperimentMaterial( + experiment: CandidateExperimentFixture['experiment'], +): AgentImprovementExperimentMaterial { + const { candidateLineage: _candidateLineage, digest: _digest, ...material } = experiment + return material +} + +export async function proposeTestMethodImprovement( + runId: string, + method: ImproveMethodFactory, + configureProfile?: (profile: ReturnType) => void, +) { + const seed = createCandidateExperimentFixture() + const profile = agentCandidateProfileAsAgentProfile(seed.experiment.baseline.profile) + configureProfile?.(profile) + const baseline = redigestCandidateBundle(seed.experiment.baseline, { + profile: freezeGenericAgentCandidateProfile(profile), + }) + let measured: CandidateExperimentFixture | undefined + return proposeAgentImprovement({ + runId, + profile, + analysis: { + registry: { + list: () => [{ id: 'improvement' }], + run: async () => ({ + run_id: runId, + correlation_id: runId, + started_at: '2026-07-10T00:00:00.000Z', + ended_at: '2026-07-10T00:00:01.000Z', + findings: [improvementFinding], + per_analyst: [], + total_cost_usd: 0, + }), + }, + inputs: {}, + findingsStore: null, + log: () => {}, + }, + improvement: { ...improvementOptions(), method }, + buildExperiment: ({ improvement }) => { + if (!improvement.candidate.profile) throw new Error('expected a profile candidate') + const candidate = redigestCandidateBundle(baseline, { + profile: freezeGenericAgentCandidateProfile(improvement.candidate.profile), + }) + measured = createCandidateExperimentFixture({ baseline, candidate }) + return candidateExperimentMaterial(measured.experiment) + }, + placeCell: (input) => { + if (!measured) throw new Error('experiment was not built') + return measured.placeCell(input) + }, + now: () => new Date('2026-07-10T01:00:00.000Z'), + }) +} diff --git a/tests/improvement-cycle.test.ts b/tests/improvement-cycle.test.ts index 8365496a..90576c91 100644 --- a/tests/improvement-cycle.test.ts +++ b/tests/improvement-cycle.test.ts @@ -1,11 +1,3 @@ -import type { AnalystFinding } from '@tangle-network/agent-eval' -import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' -import type { - DispatchContext, - JudgeConfig, - MutableSurface, - Scenario, -} from '@tangle-network/agent-eval/contract' import { sealCandidateBenchmarkSuite, sealCandidateBenchmarkTask, @@ -20,7 +12,6 @@ import { agentCandidateProfileAsAgentProfile, freezeGenericAgentCandidateProfile, } from '../src/candidate-execution/profile' -import type { ImproveMethodFactory } from '../src/improvement' import { createAgentImprovementActivationResult, executeAgentImprovementActivation, @@ -55,107 +46,17 @@ import { cleanupCandidateExperimentFixtures, createCandidateExperimentFixture, } from './helpers/candidate-experiment-fixture' - -const finding: AnalystFinding = { - schema_version: '1.0.0', - finding_id: 'finding-1', - analyst_id: 'improvement', - produced_at: '2026-07-10T00:00:00.000Z', - severity: 'high', - area: 'prompt', - claim: 'The agent omits the required answer.', - evidence_refs: [{ kind: 'span', id: 'span-1' }], - recommended_action: 'Return the measured answer.', - confidence: 0.9, - subject: 'agent-profile:prompt.systemPrompt', -} - -interface ImprovementScenario extends Scenario { - kind: 'improvement-test' -} - -const improvementScenarios: ImprovementScenario[] = Array.from({ length: 12 }, (_, index) => ({ - id: `improvement-${index}`, - kind: 'improvement-test', -})) -const improvementTrain = improvementScenarios.slice(0, 4) -const improvementSelection = improvementScenarios.slice(4, 8) -const improvementTest = improvementScenarios.slice(8) - -const improvementAgent = async ( - surface: MutableSurface, - _scenario: ImprovementScenario, - context: DispatchContext, -): Promise => { - const paid = await context.cost.runPaidCall({ - channel: 'agent', - actor: 'improvement-cycle-test', - model: 'deterministic-test', - maximumCharge: { externallyEnforcedMaximumUsd: 0.0001 }, - execute: async () => String(surface), - receipt: () => ({ - model: 'deterministic-test', - inputTokens: 1, - outputTokens: 1, - actualCostUsd: 0.0001, - }), - }) - if (!paid.succeeded) throw paid.error - return paid.value -} - -const improvementJudge: JudgeConfig = { - name: 'exact-prompt', - dimensions: [{ key: 'quality', description: 'Candidate prompt selected.' }], - score: ({ artifact }) => { - const composite = artifact === 'PROMOTED' ? 1 : 0 - return { dimensions: { quality: composite }, composite, notes: '' } - }, -} - -const improvementMethod: ImproveMethodFactory = (context) => ({ - name: 'deterministic-complete-method', - async optimize() { - if ( - !context.findings.some((item) => (item as { finding_id?: string }).finding_id === 'finding-1') - ) { - throw new Error('analysis findings did not reach the optimization method') - } - return { - winnerSurface: 'PROMOTED', - cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, - } - }, -}) - -function improvementOptions() { - return { - surface: 'prompt' as const, - method: improvementMethod, - trainScenarios: improvementTrain, - selectionScenarios: improvementSelection, - testScenarios: improvementTest, - judges: [improvementJudge], - agent: improvementAgent, - runDir: `mem://improvement-cycle-${Math.random()}`, - storage: inMemoryCampaignStorage(), - resamples: 40, - confidence: 0.95, - } -} +import { + candidateExperimentMaterial, + improvementFinding as finding, + improvementOptions, +} from './helpers/improvement-method-fixture' afterEach(() => { cleanupCandidateExperimentFixtures() cleanupCandidateFixtures() }) -function candidateExperimentMaterial( - experiment: CandidateExperimentFixture['experiment'], -): AgentImprovementExperimentMaterial { - const { candidateLineage: _candidateLineage, digest: _digest, ...material } = experiment - return material -} - // These exercise the whole improvement lifecycle — real content-addressing, real file I/O, a // paired matrix — and on CI the slowest already burn ~4.3s of the 5s default. That leaves the // file one scheduling hiccup from red regardless of what changed. Size the timeout to the work diff --git a/tests/optimization-receipt.test.ts b/tests/optimization-receipt.test.ts new file mode 100644 index 00000000..abba21e3 --- /dev/null +++ b/tests/optimization-receipt.test.ts @@ -0,0 +1,254 @@ +import { afterEach, describe, expect, it } from 'vitest' + +import { canonicalCandidateDigest, omitTopLevelDigest } from '../src/candidate-execution/digest' +import { agentCandidateProfileAsAgentProfile } from '../src/candidate-execution/profile' +import type { ImproveMethodFactory } from '../src/improvement' +import { + createAgentImprovementActivation, + proposeAgentImprovement, + reviewAgentImprovementProposal, +} from '../src/intelligence/improvement-cycle' +import { + optimizationActivationReceiptFromMetadata, + optimizationReceiptMetadataKey, +} from '../src/intelligence/optimization-receipt' +import { cleanupCandidateFixtures } from './helpers/candidate-execution-fixture' +import { + cleanupCandidateExperimentFixtures, + createCandidateExperimentFixture, +} from './helpers/candidate-experiment-fixture' +import { + candidateExperimentMaterial, + type ImprovementScenario, + improvementMethod, + improvementOptions, + proposeTestMethodImprovement, +} from './helpers/improvement-method-fixture' + +const officialImprovementMethod: ImproveMethodFactory = (context) => ({ + name: 'official-test-method', + async optimize() { + return { + winnerSurface: 'PROMOTED', + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + provenance: { + source: { + kind: 'package', + evidence: 'observed', + package: 'official-optimizer', + version: '1.2.3', + sourceUrl: 'https://example.test/official-optimizer', + revision: 'abc123', + sourceSha256: 'a'.repeat(64), + }, + bridge: { + kind: 'package', + evidence: 'observed', + package: 'agent-eval-rpc', + version: '0.126.5', + sourceSha256: 'b'.repeat(64), + }, + modules: [{ module: 'official_optimizer.engine', sourceSha256: 'c'.repeat(64) }], + python: { implementation: 'CPython', version: '3.13.5' }, + runId: `official:${context.evaluationRef}`, + compatibleRunId: `compatible:${context.evaluationRef}`, + resumed: false, + evaluationCount: 7, + artifactDir: '/tmp/official-optimizer', + tokenUsage: { + inputTokens: 100, + cachedInputTokens: 20, + cacheWriteInputTokens: 10, + outputTokens: 30, + reasoningTokens: 5, + totalTokens: 130, + calls: 2, + }, + }, + } + }, +}) + +afterEach(() => { + cleanupCandidateExperimentFixtures() + cleanupCandidateFixtures() +}) + +describe('optimization activation receipt', { timeout: 30_000 }, () => { + it('retains immutable official optimizer evidence through sealing and promotion', async () => { + const result = await proposeTestMethodImprovement( + 'official-receipt-run', + officialImprovementMethod, + (profile) => { + profile.model = { + provider: 'test-provider', + default: 'provider/model', + reasoningEffort: 'high', + } + }, + ) + + const receipt = optimizationActivationReceiptFromMetadata(result.proposal.evaluation.metadata) + expect(receipt).toMatchObject({ + kind: 'optimization-activation-receipt', + method: 'official-test-method', + source: { + package: 'official-optimizer', + version: '1.2.3', + sourceSha256: 'a'.repeat(64), + }, + bridge: { + package: 'agent-eval-rpc', + version: '0.126.5', + sourceSha256: 'b'.repeat(64), + }, + python: { implementation: 'CPython', version: '3.13.5' }, + model: { + role: 'candidate', + identity: { + provider: 'test-provider', + default: 'provider/model', + reasoningEffort: 'high', + }, + }, + usage: { + evaluations: 7, + tokens: { inputTokens: 100, outputTokens: 30, totalTokens: 130, calls: 2 }, + }, + cost: { + totalUsd: result.improvement.cost.totalCostUsd, + accountingComplete: true, + incompleteReasons: [], + }, + invocation: { + runtimeInvocationId: result.improvement.lineage.invocationId, + optimizerRunId: expect.stringMatching(/^official:sha256:/), + compatibleOptimizerRunId: expect.stringMatching(/^compatible:sha256:/), + resumed: false, + artifactDir: '/tmp/official-optimizer', + }, + developmentDataDigest: result.improvement.lineage.developmentSplitDigest, + }) + expect(Object.isFrozen(receipt)).toBe(true) + expect(Object.isFrozen(receipt?.source)).toBe(true) + + if (!result.improvement.provenance) throw new Error('expected optimizer provenance') + expect(receipt?.source).not.toBe(result.improvement.provenance.source) + expect(() => { + if (receipt) receipt.source.version = 'mutated-receipt' + }).toThrow(TypeError) + if (!receipt) throw new Error('expected optimization receipt') + const tamperedReceipt = structuredClone(receipt) + tamperedReceipt.source.version = 'tampered' + expect(() => + optimizationActivationReceiptFromMetadata({ + [optimizationReceiptMetadataKey]: tamperedReceipt, + } as unknown as Parameters[0]), + ).toThrow(/digest does not match/) + const invalidReceipt = structuredClone(receipt) + invalidReceipt.modules = [{ module: '', sourceSha256: 'c'.repeat(64) }] + invalidReceipt.digest = canonicalCandidateDigest(omitTopLevelDigest(invalidReceipt)) + expect(() => + optimizationActivationReceiptFromMetadata({ + [optimizationReceiptMetadataKey]: invalidReceipt, + } as unknown as Parameters[0]), + ).toThrow(/invalid evidence/) + const invalidUsageReceipt = structuredClone(receipt) + if (!invalidUsageReceipt.usage.tokens) throw new Error('expected optimizer token usage') + invalidUsageReceipt.usage.tokens.totalTokens += 1 + invalidUsageReceipt.digest = canonicalCandidateDigest(omitTopLevelDigest(invalidUsageReceipt)) + expect(() => + optimizationActivationReceiptFromMetadata({ + [optimizationReceiptMetadataKey]: invalidUsageReceipt, + } as unknown as Parameters[0]), + ).toThrow(/invalid evidence/) + + const review = reviewAgentImprovementProposal(result.proposal, { + decision: 'approve', + reviewedBy: 'operator@example.com', + reason: 'Official optimizer evidence retained.', + now: () => new Date('2026-07-10T01:01:00.000Z'), + }) + const activation = createAgentImprovementActivation(result.proposal, review, { + intent: 'activate-candidate', + targets: [{ surface: 'prompt', identity: 'tenant/default/profile' }], + fundingOwner: 'tenant/default', + authorizedBy: 'operator@example.com', + expiresAt: '2026-07-10T01:07:00.000Z', + now: () => new Date('2026-07-10T01:02:00.000Z'), + }) + expect(activation.proposalDigest).toBe(result.proposal.digest) + expect(optimizationActivationReceiptFromMetadata(result.proposal.evaluation.metadata)).toEqual( + receipt, + ) + await result.improvement.dispose() + }) + + it('changes its digest when official package evidence changes', async () => { + const run = async (version: string) => { + const method: ImproveMethodFactory = (context) => { + const base = officialImprovementMethod(context) + return { + ...base, + async optimize(input) { + const output = await base.optimize(input) + if (!output.provenance) throw new Error('expected optimizer provenance') + return { + ...output, + provenance: { + ...output.provenance, + source: { ...output.provenance.source, version }, + }, + } + }, + } + } + const result = await proposeTestMethodImprovement(`receipt-digest-${version}`, method) + const receipt = optimizationActivationReceiptFromMetadata(result.proposal.evaluation.metadata) + await result.improvement.dispose() + return receipt + } + + const first = await run('1.2.3') + const second = await run('1.2.4') + expect(first?.digest).not.toBe(second?.digest) + }) + + it('leaves local optimizer proposals without receipt metadata', async () => { + const result = await proposeTestMethodImprovement('local-method-no-receipt', improvementMethod) + + expect(result.proposal.evaluation.metadata).toBeUndefined() + await result.improvement.dispose() + }) + + it('rejects caller-authored receipt metadata before analysis', async () => { + const seed = createCandidateExperimentFixture() + let analysisCalls = 0 + + await expect( + proposeAgentImprovement({ + runId: 'forged-receipt', + profile: agentCandidateProfileAsAgentProfile(seed.experiment.baseline.profile), + analysis: { + registry: { + list: () => [{ id: 'improvement' }], + run: async () => { + analysisCalls += 1 + throw new Error('analysis must not run') + }, + }, + inputs: {}, + findingsStore: null, + log: () => {}, + }, + improvement: improvementOptions(), + buildExperiment: () => candidateExperimentMaterial(seed.experiment), + placeCell: seed.placeCell, + metadata: { + [optimizationReceiptMetadataKey]: { kind: 'caller-authored' }, + }, + }), + ).rejects.toThrow(/reserves 'optimizationReceipt' for Runtime/) + expect(analysisCalls).toBe(0) + }) +}) From e92ee67c68528499c4665e82b9e85128f1c11752 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 24 Jul 2026 15:46:12 -0600 Subject: [PATCH 11/14] fix(optimization): harden official optimizer execution --- .github/workflows/ci.yml | 2 +- CHANGELOG.md | 3 + README.md | 17 +- bench/package.json | 4 +- .../backfill-swe-arena.test.mts | 52 +-- bench/src/swe-arena/gepa-seat.mts | 2 +- bench/src/swe-arena/gepa-seat.test.mts | 5 +- docs/api/index.md | 316 +++++++++++++----- docs/api/intelligence.md | 304 ++++++++++------- docs/api/primitive-catalog.md | 12 +- docs/canonical-api.md | 7 +- package.json | 6 +- pnpm-lock.yaml | 30 +- .../verify-official-optimizers-consumer.mjs | 82 ++++- scripts/verify-official-optimizers.mjs | 4 +- src/improvement/improve-types.ts | 13 + src/improvement/improve.ts | 2 + src/improvement/index.ts | 3 + src/improvement/method-controls.ts | 32 ++ src/improvement/method-cost.test.ts | 134 +++++++- src/improvement/method-cost.ts | 60 +++- src/improvement/method-execution.ts | 50 ++- src/improvement/method-identity.test.ts | 2 + src/improvement/method-identity.ts | 5 + src/improvement/official-optimizers.test.ts | 191 +++++++++-- src/improvement/official-optimizers.ts | 162 ++++++--- src/improvement/official-packages.test.ts | 4 +- src/improvement/profile-surface.ts | 8 +- src/intelligence/index.ts | 10 +- src/intelligence/intelligence.test.ts | 23 ++ src/intelligence/optimization-receipt.ts | 93 ++++-- src/redact.ts | 18 + tests/knowledge-improvement-job.test.ts | 12 +- tests/optimization-receipt.test.ts | 46 ++- 34 files changed, 1293 insertions(+), 421 deletions(-) create mode 100644 src/improvement/method-controls.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f6ed2b4..4c8c5f25 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,7 +64,7 @@ jobs: official-optimizers: runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 60 steps: - uses: actions/checkout@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index a6c6f4bb..b343eb33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ - Add `officialGepa(...)` and `officialSkillOpt(...)` as Runtime adapters over the upstream GEPA and Microsoft SkillOpt implementations in `@tangle-network/agent-eval`. - Require one complete `OptimizationMethod` for profile improvement and keep final-test scenarios outside optimizer input. +- Authorize every exact execution-capable profile candidate before it reaches an agent. +- Preserve resumed optimizer spend, model identity, package provenance, and separate optimization versus final-test costs in activation receipts. +- Verify released Python packages, pinned source revisions, resume behavior, concurrency, and packed external installs in CI. - Keep code improvement on Runtime-owned isolated Git worktrees. - Remove the retired local prompt, profile-diff, campaign OTLP, and record-only optimizer paths. - Require `@tangle-network/agent-eval` 0.126.x. diff --git a/README.md b/README.md index 2a7cf33e..e1016729 100644 --- a/README.md +++ b/README.md @@ -188,7 +188,7 @@ There is no local fallback. Install its optional Python process before using it: ```bash -python -m pip install "agent-eval-rpc==0.126.5" +python -m pip install "agent-eval-rpc==0.126.6" python -m pip install "gepa[full]==0.1.4" ``` @@ -202,7 +202,7 @@ python -m pip install "gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919 Use `officialSkillOpt(...)` for Microsoft's SkillOpt: ```bash -python -m pip install "agent-eval-rpc==0.126.5" +python -m pip install "agent-eval-rpc==0.126.6" python -m pip install "skillopt @ git+https://github.com/microsoft/SkillOpt.git@61735e3922efc2b90c6d6cab561e62e98452ca90" ``` @@ -238,8 +238,11 @@ Set `redact: false` only when every outbound value is public and already reviewe The selected profile surface is the optimizer's candidate and cannot be redacted without changing the measured candidate. Runtime always rejects recognized credentials in those bytes. -It also rejects structurally sensitive fields such as MCP env, headers, URLs, metadata, and extensions unless `approveSensitiveProfileSurface: true`. -That approval asserts the exact candidate contains only public values or safe references. +It also rejects structurally sensitive fields such as MCP env, headers, URLs, metadata, and extensions. +For `tools`, `mcp`, `hooks`, `subagents`, and `agent-profile`, Runtime treats the entire selected coordinate as execution-capable. +Use `authorizeSensitiveCandidate` to inspect and accept each exact immutable profile containing public values or safe references. +The callback runs for the baseline and every distinct candidate before either reaches your agent. +Its `sensitivePaths` includes `$` when the whole coordinate requires review. Code is the exception. It uses Runtime's isolated git worktrees and coding-agent candidate execution: @@ -274,6 +277,7 @@ const result = await proposeAgentImprovement({ analysis, improvement: { surface: 'prompt', + executionRef, method, trainScenarios, selectionScenarios, @@ -312,6 +316,8 @@ const outcome = await executeAgentImprovementActivation( `buildExperimentMaterial`, `placeCell`, and the transaction functions are application ports because storage and compute differ by product. The builder returns only baseline, candidate, tasks, and policy; Runtime adds the search ancestry and seals the final experiment. Runtime owns candidate identity, measurement, review binding, expiry, retry identity, and result validation; the application owns its atomic write. +Official optimizer proposals carry the observed package versions, optimizer model, evaluation and token usage, separate optimization and final-test costs, and resumed-run identity. +`createOptimizationActivationReceipt(result)` exposes the same detached record for callers that need to inspect an `improve()` result before building a proposal. ### Improve a knowledge base @@ -327,6 +333,7 @@ import { runKnowledgeImprovementJob } from '@tangle-network/agent-runtime/knowle const result = await runKnowledgeImprovementJob({ root: './kb', goal: 'Improve support refund-policy knowledge', + implementationRef: 'git:0123456789abcdef0123456789abcdef01234567', readinessSpecs, budget: { maxIterations: 8, maxTokens: 120_000, maxUsd: 10 }, backend, @@ -336,6 +343,8 @@ console.log(result.knowledge?.reference.candidateHash, result.measurement.superv ``` Use it when the product needs one knob for "make this knowledge base better" instead of wiring `improveKnowledgeBase`, a runtime supervisor, candidate workspaces, and readiness checks by hand. +Set `implementationRef` to the deployed `git:<40 hex>` revision or a `sha256:<64 hex>` digest covering every callback, model, index, and external setting that can change the result. +The same run ID resumes only when this identity still matches. Measure the returned bundle pair, record the review, then activate through `executeAgentImprovementActivation`; activation is the only write path. ### Run on PrimeIntellect diff --git a/bench/package.json b/bench/package.json index 9aa48380..0bb82726 100644 --- a/bench/package.json +++ b/bench/package.json @@ -40,9 +40,9 @@ "verify:pier": "tsx scripts/verify-pier-pair.mts" }, "dependencies": { - "@tangle-network/agent-eval": "0.126.5", + "@tangle-network/agent-eval": "0.126.6", "@tangle-network/agent-interface": "0.32.0", - "@tangle-network/agent-knowledge": "5.0.0", + "@tangle-network/agent-knowledge": "5.0.1", "@tangle-network/agent-runtime": "workspace:*", "@tangle-network/sandbox": "^0.12.0" }, diff --git a/bench/src/rollout-ledger/backfill-swe-arena.test.mts b/bench/src/rollout-ledger/backfill-swe-arena.test.mts index e9930bba..2e78339f 100644 --- a/bench/src/rollout-ledger/backfill-swe-arena.test.mts +++ b/bench/src/rollout-ledger/backfill-swe-arena.test.mts @@ -285,31 +285,35 @@ describe('backfillSweArena', () => { expect(stats.workerSessionsJoined).toBe(2) }) - it('marks a session with no readable parts as an incomplete gap line', async () => { - await buildFixtureTree() - await buildFixtureDb() - const { DatabaseSync } = await import('node:sqlite') - const db = new DatabaseSync(dbPath) - // Session row survives; its message parts do not. - db.exec("DELETE FROM part; DELETE FROM message") - db.close() + it( + 'marks a session with no readable parts as an incomplete gap line', + async () => { + await buildFixtureTree() + await buildFixtureDb() + const { DatabaseSync } = await import('node:sqlite') + const db = new DatabaseSync(dbPath) + // Session row survives; its message parts do not. + db.exec("DELETE FROM part; DELETE FROM message") + db.close() - const { lines, stats } = await backfillSweArena(outDir, { - opencodeDb: dbPath, - claudeProjectsDir: projectsDir, - }) - const joinedCwd = lines.filter((l) => l.role === 'worker' && l.outcome.metrics.has_session === true) - expect(joinedCwd.length).toBeGreaterThan(0) - for (const worker of joinedCwd) { - expect(worker.messages).toHaveLength(0) - expect(worker.outcome.is_completed).toBe(false) - expect(worker.provenance.gap).toMatch(/no readable message parts/) - } - // The store-integrity failure is counted apart from the cwd-join failure. - expect(stats.workerSessionsEmpty).toBe(2) - expect(stats.workerCwdsMissed).toBe(1) - expect(stats.workerSessionsJoined).toBe(0) - }) + const { lines, stats } = await backfillSweArena(outDir, { + opencodeDb: dbPath, + claudeProjectsDir: projectsDir, + }) + const joinedCwd = lines.filter((l) => l.role === 'worker' && l.outcome.metrics.has_session === true) + expect(joinedCwd.length).toBeGreaterThan(0) + for (const worker of joinedCwd) { + expect(worker.messages).toHaveLength(0) + expect(worker.outcome.is_completed).toBe(false) + expect(worker.provenance.gap).toMatch(/no readable message parts/) + } + // The store-integrity failure is counted apart from the cwd-join failure. + expect(stats.workerSessionsEmpty).toBe(2) + expect(stats.workerCwdsMissed).toBe(1) + expect(stats.workerSessionsJoined).toBe(0) + }, + 15_000, + ) it('fails loud on a cell cache that carries no scenarioId/rep identity', async () => { await buildFixtureTree() diff --git a/bench/src/swe-arena/gepa-seat.mts b/bench/src/swe-arena/gepa-seat.mts index 82479d64..ed99645b 100644 --- a/bench/src/swe-arena/gepa-seat.mts +++ b/bench/src/swe-arena/gepa-seat.mts @@ -228,7 +228,7 @@ export function innerSmokeJudge(): JudgeConfig { // --------------------------------------------------------------------------- export const GEPA_PYTHON_INSTALL_HINT = - 'install `agent-eval-rpc==0.126.5`, then install ' + + 'install `agent-eval-rpc==0.126.6`, then install ' + '`gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f`' export type GepaMethodFactory = ( diff --git a/bench/src/swe-arena/gepa-seat.test.mts b/bench/src/swe-arena/gepa-seat.test.mts index e1ffb9f8..246e77c9 100644 --- a/bench/src/swe-arena/gepa-seat.test.mts +++ b/bench/src/swe-arena/gepa-seat.test.mts @@ -14,6 +14,7 @@ import { DEFAULT_GEPA_PYTHON, DEFAULT_MAX_METRIC_CALLS, GEPA_INNER_RUNS_DIRNAME, + GEPA_PYTHON_INSTALL_HINT, gepaBridgeScenarios, innerSmokeComposite, innerSmokeJudge, @@ -188,7 +189,7 @@ describe('probeGepaRuntime', () => { it('fails loud with pip install instructions when the bridge module is missing', async () => { const exec = execFailingOn('agent_eval_rpc.gepa_bridge', "ModuleNotFoundError: No module named 'agent_eval_rpc'") await expect(probeGepaRuntime('python3', exec, 'gepa-author')).rejects.toThrow( - /pip install agent-eval-rpc/, + GEPA_PYTHON_INSTALL_HINT, ) await expect(probeGepaRuntime('python3', exec, 'gepa-author')).rejects.toThrow(/not installed/) }) @@ -256,7 +257,7 @@ describe('captureProposerProvenance with a gepa seat', () => { ? { code: 1, stdout: '', stderr: 'ModuleNotFoundError' } : { code: 0, stdout: 'Python 3.12.3', stderr: '' } await expect(captureProposerProvenance([seat()], { exec })).rejects.toThrow( - /pip install agent-eval-rpc/, + GEPA_PYTHON_INSTALL_HINT, ) }) }) diff --git a/docs/api/index.md b/docs/api/index.md index 8466d40b..4d4702d4 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -1135,7 +1135,7 @@ Defined in: [src/errors.ts:117](https://github.com/tangle-network/agent-runtime/ ### OfficialOptimizerUnavailableError -Defined in: [src/improvement/official-optimizers.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L64) +Defined in: [src/improvement/official-optimizers.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L74) Missing optional Python dependencies for an official optimizer. @@ -1149,7 +1149,7 @@ Missing optional Python dependencies for an official optimizer. > **new OfficialOptimizerUnavailableError**(`optimizer`, `cause`): [`OfficialOptimizerUnavailableError`](#officialoptimizerunavailableerror) -Defined in: [src/improvement/official-optimizers.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L67) +Defined in: [src/improvement/official-optimizers.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L77) ###### Parameters @@ -1175,7 +1175,7 @@ Defined in: [src/improvement/official-optimizers.ts:67](https://github.com/tangl > `readonly` **optimizer**: `"gepa"` \| `"skillopt"` -Defined in: [src/improvement/official-optimizers.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L65) +Defined in: [src/improvement/official-optimizers.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L75) *** @@ -6276,9 +6276,53 @@ Findings produced before this search, if any. *** +### ImproveCandidateValidationInput + +Defined in: [src/improvement/improve-types.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L73) + +Exact materialized profile presented for validation before any candidate run. + +#### Extended by + +- [`OfficialSensitiveCandidateInput`](#officialsensitivecandidateinput) + +#### Properties + +##### profile + +> **profile**: `object` + +Defined in: [src/improvement/improve-types.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L74) + +##### surface + +> **surface**: [`ImproveProfileSurface`](#improveprofilesurface) + +Defined in: [src/improvement/improve-types.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L75) + +##### candidateSurface + +> **candidateSurface**: `MutableSurface` + +Defined in: [src/improvement/improve-types.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L76) + +##### value + +> **value**: `unknown` + +Defined in: [src/improvement/improve-types.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L77) + +##### isBaseline + +> **isBaseline**: `boolean` + +Defined in: [src/improvement/improve-types.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L78) + +*** + ### ImproveSkillsOptions -Defined in: [src/improvement/improve-types.ts:155](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L155) +Defined in: [src/improvement/improve-types.ts:168](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L168) #### Properties @@ -6286,7 +6330,7 @@ Defined in: [src/improvement/improve-types.ts:155](https://github.com/tangle-net > **resourceName**: `string` -Defined in: [src/improvement/improve-types.ts:157](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L157) +Defined in: [src/improvement/improve-types.ts:170](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L170) `name` of one inline entry in `profile.resources.skills`. @@ -6294,7 +6338,7 @@ Defined in: [src/improvement/improve-types.ts:157](https://github.com/tangle-net ### ImproveProfileComponents -Defined in: [src/improvement/improve-types.ts:161](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L161) +Defined in: [src/improvement/improve-types.ts:174](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L174) Caller-owned mapping for optimizing several profile fields as one candidate. @@ -6304,7 +6348,7 @@ Caller-owned mapping for optimizing several profile fields as one candidate. > **read**(`profile`): `Readonly`\<`Record`\<`string`, `string`\>\> -Defined in: [src/improvement/improve-types.ts:163](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L163) +Defined in: [src/improvement/improve-types.ts:176](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L176) Extract the exact named text components optimized together. @@ -6320,7 +6364,7 @@ Extract the exact named text components optimized together. > **apply**(`profile`, `components`): `object` -Defined in: [src/improvement/improve-types.ts:165](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L165) +Defined in: [src/improvement/improve-types.ts:178](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L178) Apply a complete winning component map to a detached profile. @@ -6340,7 +6384,7 @@ Apply a complete winning component map to a detached profile. ### ImproveCodeOptions -Defined in: [src/improvement/improve-types.ts:171](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L171) +Defined in: [src/improvement/improve-types.ts:184](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L184) #### Properties @@ -6348,7 +6392,7 @@ Defined in: [src/improvement/improve-types.ts:171](https://github.com/tangle-net > **repoRoot**: `string` -Defined in: [src/improvement/improve-types.ts:173](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L173) +Defined in: [src/improvement/improve-types.ts:186](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L186) Repo root candidate worktrees fork from. @@ -6356,7 +6400,7 @@ Repo root candidate worktrees fork from. > `optional` **baseRef?**: `string` -Defined in: [src/improvement/improve-types.ts:175](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L175) +Defined in: [src/improvement/improve-types.ts:188](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L188) Base ref candidates fork from. Default `main`. @@ -6364,7 +6408,7 @@ Base ref candidates fork from. Default `main`. > `optional` **worktreeDir?**: `string` -Defined in: [src/improvement/improve-types.ts:177](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L177) +Defined in: [src/improvement/improve-types.ts:190](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L190) Directory worktrees are created under. Default `/.worktrees`. @@ -6372,7 +6416,7 @@ Directory worktrees are created under. Default `/.worktrees`. > `optional` **worktree?**: `WorktreeAdapter` -Defined in: [src/improvement/improve-types.ts:180](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L180) +Defined in: [src/improvement/improve-types.ts:193](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L193) Git-compatible adapter override, primarily for tests. Candidate advancement still requires normal Git worktree and commit semantics. @@ -6381,7 +6425,7 @@ still requires normal Git worktree and commit semantics. > `optional` **harness?**: [`LocalHarness`](mcp.md#localharness) -Defined in: [src/improvement/improve-types.ts:182](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L182) +Defined in: [src/improvement/improve-types.ts:195](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L195) Coding harness the agentic generator runs in each worktree. Default `claude`. @@ -6389,7 +6433,7 @@ Coding harness the agentic generator runs in each worktree. Default `claude`. > `optional` **verify?**: [`Verifier`](#verifier) -Defined in: [src/improvement/improve-types.ts:185](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L185) +Defined in: [src/improvement/improve-types.ts:198](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L198) Verify a candidate worktree before it becomes a measurable surface; failures feed the next shot (see `agenticGenerator.verify` / `commandVerifier`). @@ -6398,7 +6442,7 @@ feed the next shot (see `agenticGenerator.verify` / `commandVerifier`). > `optional` **timeoutMs?**: `number` -Defined in: [src/improvement/improve-types.ts:187](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L187) +Defined in: [src/improvement/improve-types.ts:200](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L200) Per-shot wall-clock timeout for the harness (ms). @@ -6406,7 +6450,7 @@ Per-shot wall-clock timeout for the harness (ms). > `optional` **generator?**: [`CandidateGenerator`](#candidategenerator) -Defined in: [src/improvement/improve-types.ts:190](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L190) +Defined in: [src/improvement/improve-types.ts:203](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L203) Byte-producer override, used for tests and custom candidate production. When set, `harness`, `verify`, and `timeoutMs` are unused. @@ -6415,7 +6459,7 @@ When set, `harness`, `verify`, and `timeoutMs` are unused. ### ImprovementProfileCandidate -Defined in: [src/improvement/improve-types.ts:193](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L193) +Defined in: [src/improvement/improve-types.ts:206](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L206) #### Properties @@ -6423,7 +6467,7 @@ Defined in: [src/improvement/improve-types.ts:193](https://github.com/tangle-net > **surface**: [`ImproveProfileSurface`](#improveprofilesurface) -Defined in: [src/improvement/improve-types.ts:195](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L195) +Defined in: [src/improvement/improve-types.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L208) Surface searched by this run. @@ -6431,7 +6475,7 @@ Surface searched by this run. > **value**: `MutableSurface` -Defined in: [src/improvement/improve-types.ts:197](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L197) +Defined in: [src/improvement/improve-types.ts:210](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L210) Exact winning value returned by agent-eval. @@ -6439,7 +6483,7 @@ Exact winning value returned by agent-eval. > **profile**: `object` -Defined in: [src/improvement/improve-types.ts:199](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L199) +Defined in: [src/improvement/improve-types.ts:212](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L212) Exact complete profile instance measured on the final cases. @@ -6447,7 +6491,7 @@ Exact complete profile instance measured on the final cases. ### ImprovementCodeCandidate -Defined in: [src/improvement/improve-types.ts:202](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L202) +Defined in: [src/improvement/improve-types.ts:215](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L215) #### Properties @@ -6455,25 +6499,25 @@ Defined in: [src/improvement/improve-types.ts:202](https://github.com/tangle-net > **surface**: `"code"` -Defined in: [src/improvement/improve-types.ts:203](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L203) +Defined in: [src/improvement/improve-types.ts:216](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L216) ##### value > **value**: `MutableSurface` -Defined in: [src/improvement/improve-types.ts:204](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L204) +Defined in: [src/improvement/improve-types.ts:217](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L217) ##### profile? > `optional` **profile?**: `undefined` -Defined in: [src/improvement/improve-types.ts:205](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L205) +Defined in: [src/improvement/improve-types.ts:218](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L218) *** ### ImproveCost -Defined in: [src/improvement/improve-types.ts:211](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L211) +Defined in: [src/improvement/improve-types.ts:224](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L224) Normalized spend reported for one Runtime improvement run. @@ -6483,25 +6527,25 @@ Normalized spend reported for one Runtime improvement run. > **totalCostUsd**: `number` -Defined in: [src/improvement/improve-types.ts:212](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L212) +Defined in: [src/improvement/improve-types.ts:225](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L225) ##### accountingComplete > **accountingComplete**: `boolean` -Defined in: [src/improvement/improve-types.ts:213](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L213) +Defined in: [src/improvement/improve-types.ts:226](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L226) ##### incompleteReasons > **incompleteReasons**: `string`[] -Defined in: [src/improvement/improve-types.ts:214](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L214) +Defined in: [src/improvement/improve-types.ts:227](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L227) *** ### ImproveLineage -Defined in: [src/improvement/improve-types.ts:218](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L218) +Defined in: [src/improvement/improve-types.ts:231](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L231) Optimizer ancestry sealed into downstream candidate experiments. @@ -6511,7 +6555,7 @@ Optimizer ancestry sealed into downstream candidate experiments. > **invocationId**: `string` -Defined in: [src/improvement/improve-types.ts:220](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L220) +Defined in: [src/improvement/improve-types.ts:233](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L233) Unique Runtime invocation used to isolate this run's cost receipts. @@ -6519,7 +6563,7 @@ Unique Runtime invocation used to isolate this run's cost receipts. > **runId**: `string` -Defined in: [src/improvement/improve-types.ts:222](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L222) +Defined in: [src/improvement/improve-types.ts:235](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L235) Upstream optimizer run when reported, otherwise this Runtime optimization invocation. @@ -6527,7 +6571,7 @@ Upstream optimizer run when reported, otherwise this Runtime optimization invoca > **developmentSplitDigest**: `` `sha256:${string}` `` -Defined in: [src/improvement/improve-types.ts:224](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L224) +Defined in: [src/improvement/improve-types.ts:237](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L237) Exact train-plus-selection scenario payloads exposed to candidate selection. @@ -6535,7 +6579,7 @@ Exact train-plus-selection scenario payloads exposed to candidate selection. > `optional` **executionRef?**: `` `sha256:${string}` `` -Defined in: [src/improvement/improve-types.ts:226](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L226) +Defined in: [src/improvement/improve-types.ts:239](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L239) Complete callback, materializer, model, tool, and closure identity for a profile run. @@ -6543,7 +6587,7 @@ Complete callback, materializer, model, tool, and closure identity for a profile > `optional` **baselineProfileDigest?**: `` `sha256:${string}` `` -Defined in: [src/improvement/improve-types.ts:228](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L228) +Defined in: [src/improvement/improve-types.ts:241](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L241) Complete baseline profile identity for a profile run. @@ -6551,7 +6595,7 @@ Complete baseline profile identity for a profile run. ### ImproveMethodResult -Defined in: [src/improvement/improve-types.ts:253](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L253) +Defined in: [src/improvement/improve-types.ts:266](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L266) #### Extends @@ -6563,7 +6607,7 @@ Defined in: [src/improvement/improve-types.ts:253](https://github.com/tangle-net > **candidate**: [`ImprovementProfileCandidate`](#improvementprofilecandidate) -Defined in: [src/improvement/improve-types.ts:233](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L233) +Defined in: [src/improvement/improve-types.ts:246](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L246) Frozen candidate only. Live state is changed through an approved activation. @@ -6575,7 +6619,7 @@ Frozen candidate only. Live state is changed through an approved activation. > **cost**: [`ImproveCost`](#improvecost) -Defined in: [src/improvement/improve-types.ts:241](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L241) +Defined in: [src/improvement/improve-types.ts:254](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L254) Full search and final-test spend. @@ -6587,7 +6631,7 @@ Full search and final-test spend. > **durationMs**: `number` -Defined in: [src/improvement/improve-types.ts:243](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L243) +Defined in: [src/improvement/improve-types.ts:256](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L256) Full wall-clock duration. @@ -6599,7 +6643,7 @@ Full wall-clock duration. > **lineage**: [`ImproveLineage`](#improvelineage) -Defined in: [src/improvement/improve-types.ts:245](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L245) +Defined in: [src/improvement/improve-types.ts:258](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L258) Optimizer ancestry used when sealing a candidate experiment. @@ -6611,7 +6655,7 @@ Optimizer ancestry used when sealing a candidate experiment. > `optional` **generationsExplored?**: `number` -Defined in: [src/improvement/improve-types.ts:247](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L247) +Defined in: [src/improvement/improve-types.ts:260](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L260) Number of generations explored by Runtime's code path. @@ -6623,19 +6667,19 @@ Number of generations explored by Runtime's code path. > **mode**: `"method"` -Defined in: [src/improvement/improve-types.ts:254](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L254) +Defined in: [src/improvement/improve-types.ts:267](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L267) ##### method > **method**: `string` -Defined in: [src/improvement/improve-types.ts:255](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L255) +Defined in: [src/improvement/improve-types.ts:268](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L268) ##### provenance? > `optional` **provenance?**: `OptimizationMethodProvenance` -Defined in: [src/improvement/improve-types.ts:257](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L257) +Defined in: [src/improvement/improve-types.ts:270](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L270) External optimizer package and resumable run identity, when reported. @@ -6643,7 +6687,7 @@ External optimizer package and resumable run identity, when reported. > **decision**: `"ship"` \| `"hold"` -Defined in: [src/improvement/improve-types.ts:258](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L258) +Defined in: [src/improvement/improve-types.ts:271](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L271) Final-test decision for this search result. @@ -6655,7 +6699,7 @@ Final-test decision for this search result. > **lift**: `number` -Defined in: [src/improvement/improve-types.ts:259](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L259) +Defined in: [src/improvement/improve-types.ts:272](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L272) Final-test lift when one was measured. @@ -6667,7 +6711,7 @@ Final-test lift when one was measured. > **liftInterval**: `object` -Defined in: [src/improvement/improve-types.ts:260](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L260) +Defined in: [src/improvement/improve-types.ts:273](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L273) Paired final-test confidence interval for method-based profile runs. @@ -6687,7 +6731,7 @@ Paired final-test confidence interval for method-based profile runs. > **raw**: `OptimizationMethodComparison` -Defined in: [src/improvement/improve-types.ts:261](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L261) +Defined in: [src/improvement/improve-types.ts:274](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L274) #### Methods @@ -6695,7 +6739,7 @@ Defined in: [src/improvement/improve-types.ts:261](https://github.com/tangle-net > **dispose**(): `Promise`\<`void`\> -Defined in: [src/improvement/improve-types.ts:250](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L250) +Defined in: [src/improvement/improve-types.ts:263](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L263) Release resources owned by this result. Idempotent; currently disposes the returned code worktree and is a no-op for profile-only surfaces. @@ -6712,7 +6756,7 @@ the returned code worktree and is a no-op for profile-only surfaces. ### ImproveCodeResult -Defined in: [src/improvement/improve-types.ts:264](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L264) +Defined in: [src/improvement/improve-types.ts:277](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L277) #### Extends @@ -6734,7 +6778,7 @@ Defined in: [src/improvement/improve-types.ts:264](https://github.com/tangle-net > **candidate**: [`ImprovementCodeCandidate`](#improvementcodecandidate) -Defined in: [src/improvement/improve-types.ts:233](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L233) +Defined in: [src/improvement/improve-types.ts:246](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L246) Frozen candidate only. Live state is changed through an approved activation. @@ -6746,7 +6790,7 @@ Frozen candidate only. Live state is changed through an approved activation. > **decision**: `"ship"` \| `"hold"` \| `"need_more_work"` \| `"model_ceiling"` \| `"arch_ceiling"` -Defined in: [src/improvement/improve-types.ts:235](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L235) +Defined in: [src/improvement/improve-types.ts:248](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L248) Final-test decision for this search result. @@ -6758,7 +6802,7 @@ Final-test decision for this search result. > `optional` **lift?**: `number` -Defined in: [src/improvement/improve-types.ts:237](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L237) +Defined in: [src/improvement/improve-types.ts:250](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L250) Final-test lift when one was measured. @@ -6770,7 +6814,7 @@ Final-test lift when one was measured. > `optional` **liftInterval?**: `object` -Defined in: [src/improvement/improve-types.ts:239](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L239) +Defined in: [src/improvement/improve-types.ts:252](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L252) Paired final-test confidence interval for method-based profile runs. @@ -6790,7 +6834,7 @@ Paired final-test confidence interval for method-based profile runs. > **cost**: [`ImproveCost`](#improvecost) -Defined in: [src/improvement/improve-types.ts:241](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L241) +Defined in: [src/improvement/improve-types.ts:254](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L254) Full search and final-test spend. @@ -6802,7 +6846,7 @@ Full search and final-test spend. > **durationMs**: `number` -Defined in: [src/improvement/improve-types.ts:243](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L243) +Defined in: [src/improvement/improve-types.ts:256](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L256) Full wall-clock duration. @@ -6814,7 +6858,7 @@ Full wall-clock duration. > **lineage**: [`ImproveLineage`](#improvelineage) -Defined in: [src/improvement/improve-types.ts:245](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L245) +Defined in: [src/improvement/improve-types.ts:258](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L258) Optimizer ancestry used when sealing a candidate experiment. @@ -6826,7 +6870,7 @@ Optimizer ancestry used when sealing a candidate experiment. > `optional` **generationsExplored?**: `number` -Defined in: [src/improvement/improve-types.ts:247](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L247) +Defined in: [src/improvement/improve-types.ts:260](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L260) Number of generations explored by Runtime's code path. @@ -6838,13 +6882,13 @@ Number of generations explored by Runtime's code path. > **mode**: `"code"` -Defined in: [src/improvement/improve-types.ts:266](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L266) +Defined in: [src/improvement/improve-types.ts:279](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L279) ##### raw > **raw**: `SelfImproveResult`\<`TScenario`, `TArtifact`\> -Defined in: [src/improvement/improve-types.ts:267](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L267) +Defined in: [src/improvement/improve-types.ts:280](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L280) #### Methods @@ -6852,7 +6896,7 @@ Defined in: [src/improvement/improve-types.ts:267](https://github.com/tangle-net > **dispose**(): `Promise`\<`void`\> -Defined in: [src/improvement/improve-types.ts:250](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L250) +Defined in: [src/improvement/improve-types.ts:263](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L263) Release resources owned by this result. Idempotent; currently disposes the returned code worktree and is a no-op for profile-only surfaces. @@ -7024,7 +7068,7 @@ Minimum tools the server must expose to pass. Default 1. ### OfficialOptimizerContextOptions -Defined in: [src/improvement/official-optimizers.ts:27](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L27) +Defined in: [src/improvement/official-optimizers.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L37) Runtime context appended to an official optimizer's own configuration. @@ -7034,7 +7078,7 @@ Runtime context appended to an official optimizer's own configuration. > `optional` **background?**: `string` -Defined in: [src/improvement/official-optimizers.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L29) +Defined in: [src/improvement/official-optimizers.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L39) Context supplied to the optimizer before Runtime appends the profile surface and findings. @@ -7042,7 +7086,7 @@ Context supplied to the optimizer before Runtime appends the profile surface and > `optional` **includeFindings?**: `boolean` -Defined in: [src/improvement/official-optimizers.ts:31](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L31) +Defined in: [src/improvement/official-optimizers.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L41) Include current trace or analyst findings in the optimizer background. Default true. @@ -7050,7 +7094,7 @@ Include current trace or analyst findings in the optimizer background. Default t > `optional` **maxFindingsChars?**: `number` -Defined in: [src/improvement/official-optimizers.ts:33](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L33) +Defined in: [src/improvement/official-optimizers.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L43) Reject oversized serialized findings before starting Python. Default 50,000 characters. @@ -7058,22 +7102,100 @@ Reject oversized serialized findings before starting Python. Default 50,000 char > `optional` **redact?**: `false` \| [`Redactor`](intelligence.md#redactor) -Defined in: [src/improvement/official-optimizers.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L39) +Defined in: [src/improvement/official-optimizers.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L49) Redact caller-supplied context and descriptors before they leave Runtime. The built-in redactor is the default. Pass `false` only for public data that has already been reviewed. -##### approveSensitiveProfileSurface? +##### authorizeSensitiveCandidate? -> `optional` **approveSensitiveProfileSurface?**: `boolean` +> `optional` **authorizeSensitiveCandidate?**: (`input`) => `boolean` -Defined in: [src/improvement/official-optimizers.ts:46](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L46) +Defined in: [src/improvement/official-optimizers.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L52) -Confirm that structurally sensitive candidate fields contain only safe -references or public values. Required for fields such as MCP env, headers, -URLs, metadata, and extensions because candidate bytes cannot be redacted -without changing what the optimizer measures. +Authorize one exact candidate containing structurally sensitive fields. +The callback must return true for every accepted baseline and candidate. + +###### Parameters + +###### input + +[`OfficialSensitiveCandidateInput`](#officialsensitivecandidateinput) + +###### Returns + +`boolean` + +*** + +### OfficialSensitiveCandidateInput + +Defined in: [src/improvement/official-optimizers.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L55) + +Exact materialized profile presented for validation before any candidate run. + +#### Extends + +- [`ImproveCandidateValidationInput`](#improvecandidatevalidationinput) + +#### Properties + +##### profile + +> **profile**: `object` + +Defined in: [src/improvement/improve-types.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L74) + +###### Inherited from + +[`ImproveCandidateValidationInput`](#improvecandidatevalidationinput).[`profile`](#profile-3) + +##### surface + +> **surface**: [`ImproveProfileSurface`](#improveprofilesurface) + +Defined in: [src/improvement/improve-types.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L75) + +###### Inherited from + +[`ImproveCandidateValidationInput`](#improvecandidatevalidationinput).[`surface`](#surface-2) + +##### candidateSurface + +> **candidateSurface**: `MutableSurface` + +Defined in: [src/improvement/improve-types.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L76) + +###### Inherited from + +[`ImproveCandidateValidationInput`](#improvecandidatevalidationinput).[`candidateSurface`](#candidatesurface) + +##### value + +> **value**: `unknown` + +Defined in: [src/improvement/improve-types.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L77) + +###### Inherited from + +[`ImproveCandidateValidationInput`](#improvecandidatevalidationinput).[`value`](#value-1) + +##### isBaseline + +> **isBaseline**: `boolean` + +Defined in: [src/improvement/improve-types.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L78) + +###### Inherited from + +[`ImproveCandidateValidationInput`](#improvecandidatevalidationinput).[`isBaseline`](#isbaseline) + +##### sensitivePaths + +> **sensitivePaths**: readonly `string`[] + +Defined in: [src/improvement/official-optimizers.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L56) *** @@ -11997,11 +12119,29 @@ Runs one exact materialized profile on one scenario. *** +### ImproveCandidateValidator + +> **ImproveCandidateValidator** = (`input`) => `void` + +Defined in: [src/improvement/improve-types.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L81) + +#### Parameters + +##### input + +[`ImproveCandidateValidationInput`](#improvecandidatevalidationinput) + +#### Returns + +`void` + +*** + ### ImproveOptimizationRunOptions > **ImproveOptimizationRunOptions**\<`TScenario`, `TArtifact`\> = `Omit`\<`NonNullable`\<`CompareOptimizationMethodsOptions`\<`TScenario`, `TArtifact`\>\[`"optimizationRunOptions"`\]\>, `"dispatchRef"`\> -Defined in: [src/improvement/improve-types.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L72) +Defined in: [src/improvement/improve-types.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L83) #### Type Parameters @@ -12019,7 +12159,7 @@ Defined in: [src/improvement/improve-types.ts:72](https://github.com/tangle-netw > **ImproveMethodOptions**\<`TScenario`, `TArtifact`\> = `Omit`\<`CompareOptimizationMethodsOptions`\<`TScenario`, `TArtifact`\>, `"baselineSurface"` \| `"dispatchRef"` \| `"dispatchWithSurface"` \| `"methods"` \| `"optimizationConcurrency"` \| `"optimizationRunOptions"`\> & `object` -Defined in: [src/improvement/improve-types.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L78) +Defined in: [src/improvement/improve-types.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L89) Complete-method configuration for every non-code profile surface. @@ -12050,6 +12190,12 @@ A complete optimizer or a factory that can incorporate current findings. Runs the exact complete profile materialized from one candidate surface. +##### validateCandidate? + +> `optional` **validateCandidate?**: [`ImproveCandidateValidator`](#improvecandidatevalidator) + +Reject a materialized profile before it reaches the agent callback. + ##### findings? > `optional` **findings?**: readonly `unknown`[] @@ -12097,7 +12243,7 @@ Ship only when the paired final-test interval is entirely above this lift. Defau > **ImproveCodeRunOptions**\<`TScenario`, `TArtifact`\> = `Omit`\<`SelfImproveOptions`\<`TScenario`, `TArtifact`\>, `"analyzeGeneration"` \| `"baselineSurface"` \| `"budget"` \| `"findings"` \| `"gate"` \| `"llm"` \| `"method"` \| `"mutationPrimitives"` \| `"proposer"` \| `"proposerTarget"` \| `"selectionScenarios"`\> & `object` -Defined in: [src/improvement/improve-types.ts:114](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L114) +Defined in: [src/improvement/improve-types.ts:127](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L127) Runtime-owned code search in isolated git worktrees. @@ -12170,7 +12316,7 @@ the exam runs; this callback controls how its evidence decides promotion. > **ImproveOptions**\<`TScenario`, `TArtifact`\> = [`ImproveMethodOptions`](#improvemethodoptions)\<`TScenario`, `TArtifact`\> \| [`ImproveCodeRunOptions`](#improvecoderunoptions)\<`TScenario`, `TArtifact`\> -Defined in: [src/improvement/improve-types.ts:151](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L151) +Defined in: [src/improvement/improve-types.ts:164](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L164) The canonical improvement API: complete methods for profiles, worktrees for code. @@ -12190,7 +12336,7 @@ The canonical improvement API: complete methods for profiles, worktrees for code > **ImprovementCandidate** = [`ImprovementProfileCandidate`](#improvementprofilecandidate) \| [`ImprovementCodeCandidate`](#improvementcodecandidate) -Defined in: [src/improvement/improve-types.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L208) +Defined in: [src/improvement/improve-types.ts:221](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L221) *** @@ -12198,7 +12344,7 @@ Defined in: [src/improvement/improve-types.ts:208](https://github.com/tangle-net > **ImproveResult**\<`TScenario`, `TArtifact`\> = [`ImproveMethodResult`](#improvemethodresult) \| [`ImproveCodeResult`](#improvecoderesult)\<`TScenario`, `TArtifact`\> -Defined in: [src/improvement/improve-types.ts:270](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L270) +Defined in: [src/improvement/improve-types.ts:283](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve-types.ts#L283) #### Type Parameters @@ -12216,7 +12362,7 @@ Defined in: [src/improvement/improve-types.ts:270](https://github.com/tangle-net > **OfficialGepaOptions**\<`TScenario`, `TArtifact`\> = `Omit`\<`GepaOptimizationMethodConfig`\<`TScenario`, `TArtifact`\>, `"background"` \| `"evaluationId"`\> & [`OfficialOptimizerContextOptions`](#officialoptimizercontextoptions) -Defined in: [src/improvement/official-optimizers.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L50) +Defined in: [src/improvement/official-optimizers.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L60) Official GEPA configuration plus bounded Runtime findings context. @@ -12236,7 +12382,7 @@ Official GEPA configuration plus bounded Runtime findings context. > **OfficialSkillOptOptions**\<`TScenario`, `TArtifact`\> = `Omit`\<`SkillOptOptimizationMethodConfig`\<`TScenario`, `TArtifact`\>, `"background"` \| `"evaluationId"`\> & [`OfficialOptimizerContextOptions`](#officialoptimizercontextoptions) -Defined in: [src/improvement/official-optimizers.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L57) +Defined in: [src/improvement/official-optimizers.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L67) Official SkillOpt configuration plus bounded Runtime findings context. @@ -14504,7 +14650,7 @@ readonly `unknown`[] > **improve**\<`TScenario`, `TArtifact`\>(`profile`, `opts`): `Promise`\<[`ImproveMethodResult`](#improvemethodresult)\> -Defined in: [src/improvement/improve.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L53) +Defined in: [src/improvement/improve.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L55) Optimize one exact profile surface with a complete method. @@ -14536,7 +14682,7 @@ Optimize one exact profile surface with a complete method. > **improve**\<`TScenario`, `TArtifact`\>(`opts`): `Promise`\<[`ImproveCodeResult`](#improvecoderesult)\<`TScenario`, `TArtifact`\>\> -Defined in: [src/improvement/improve.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L60) +Defined in: [src/improvement/improve.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L62) Optimize repository code through Runtime's isolated worktree path. @@ -14586,7 +14732,7 @@ Build a `Verifier` that boots a generated MCP server over stdio and checks it ex > **officialGepa**\<`TScenario`, `TArtifact`\>(`options`): [`ImproveMethodFactory`](#improvemethodfactory)\<`TScenario`, `TArtifact`\> -Defined in: [src/improvement/official-optimizers.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L97) +Defined in: [src/improvement/official-optimizers.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L107) Build a complete method backed by GEPA's official Optimize Anything API. @@ -14619,7 +14765,7 @@ The recipe is passed through unchanged. Use `engine`, `sequential`, > **officialSkillOpt**\<`TScenario`, `TArtifact`\>(`options`): [`ImproveMethodFactory`](#improvemethodfactory)\<`TScenario`, `TArtifact`\> -Defined in: [src/improvement/official-optimizers.ts:161](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L161) +Defined in: [src/improvement/official-optimizers.ts:182](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/official-optimizers.ts#L182) Build a complete method backed by Microsoft's official SkillOpt trainer. diff --git a/docs/api/intelligence.md b/docs/api/intelligence.md index 4e3d78f3..957068c3 100644 --- a/docs/api/intelligence.md +++ b/docs/api/intelligence.md @@ -2440,7 +2440,7 @@ Defined in: [src/intelligence/improvement-surfaces.ts:51](https://github.com/tan ### UsageSplit -Defined in: [src/intelligence/index.ts:216](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L216) +Defined in: [src/intelligence/index.ts:222](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L222) The per-class cost split carried by every trace and outcome. `off` ⇒ `intelligenceUsd: 0` by construction — there is no intelligence spawn to @@ -2452,7 +2452,7 @@ bill. This is a classification on the trace, NOT a budget-pool split. > **inferenceUsd**: `number` -Defined in: [src/intelligence/index.ts:218](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L218) +Defined in: [src/intelligence/index.ts:224](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L224) Base-stream (model) spend in USD. @@ -2460,7 +2460,7 @@ Base-stream (model) spend in USD. > **intelligenceUsd**: `number` -Defined in: [src/intelligence/index.ts:220](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L220) +Defined in: [src/intelligence/index.ts:226](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L226) Intelligence-spawn spend in USD. Provably `0` at the OFF tier. @@ -2468,7 +2468,7 @@ Intelligence-spawn spend in USD. Provably `0` at the OFF tier. ### RunRecord -Defined in: [src/intelligence/index.ts:230](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L230) +Defined in: [src/intelligence/index.ts:236](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L236) The typed record `withIntelligence` sends per call — serialized through the shipped OTLP builders to the plane's `/v1/otlp` ingest. `input`/`output` are @@ -2482,43 +2482,43 @@ tree under the same `traceId`. > **runId**: `string` -Defined in: [src/intelligence/index.ts:231](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L231) +Defined in: [src/intelligence/index.ts:237](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L237) ##### traceId > **traceId**: `string` -Defined in: [src/intelligence/index.ts:232](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L232) +Defined in: [src/intelligence/index.ts:238](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L238) ##### project > **project**: `string` -Defined in: [src/intelligence/index.ts:233](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L233) +Defined in: [src/intelligence/index.ts:239](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L239) ##### target > **target**: `string` -Defined in: [src/intelligence/index.ts:234](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L234) +Defined in: [src/intelligence/index.ts:240](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L240) ##### input > **input**: `unknown` -Defined in: [src/intelligence/index.ts:235](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L235) +Defined in: [src/intelligence/index.ts:241](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L241) ##### output > **output**: `unknown` -Defined in: [src/intelligence/index.ts:236](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L236) +Defined in: [src/intelligence/index.ts:242](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L242) ##### outcome > **outcome**: `object` -Defined in: [src/intelligence/index.ts:237](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L237) +Defined in: [src/intelligence/index.ts:243](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L243) ###### success? @@ -2536,61 +2536,61 @@ Defined in: [src/intelligence/index.ts:237](https://github.com/tangle-network/ag > `optional` **model?**: `string` -Defined in: [src/intelligence/index.ts:242](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L242) +Defined in: [src/intelligence/index.ts:248](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L248) ##### provider? > `optional` **provider?**: `string` -Defined in: [src/intelligence/index.ts:243](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L243) +Defined in: [src/intelligence/index.ts:249](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L249) ##### loopEvents? > `optional` **loopEvents?**: [`LoopTraceEvent`](runtime.md#looptraceevent)[] -Defined in: [src/intelligence/index.ts:244](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L244) +Defined in: [src/intelligence/index.ts:250](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L250) ##### runtimeEvents? > `optional` **runtimeEvents?**: [`RuntimeStreamEvent`](index.md#runtimestreamevent)[] -Defined in: [src/intelligence/index.ts:245](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L245) +Defined in: [src/intelligence/index.ts:251](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L251) ##### profile? > `optional` **profile?**: `AgentProfile` -Defined in: [src/intelligence/index.ts:246](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L246) +Defined in: [src/intelligence/index.ts:252](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L252) ##### sessionId? > `optional` **sessionId?**: `string` -Defined in: [src/intelligence/index.ts:247](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L247) +Defined in: [src/intelligence/index.ts:253](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L253) ##### harness? > `optional` **harness?**: `string` -Defined in: [src/intelligence/index.ts:248](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L248) +Defined in: [src/intelligence/index.ts:254](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L254) ##### repository? > `optional` **repository?**: `string` -Defined in: [src/intelligence/index.ts:249](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L249) +Defined in: [src/intelligence/index.ts:255](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L255) ##### commitSha? > `optional` **commitSha?**: `string` -Defined in: [src/intelligence/index.ts:250](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L250) +Defined in: [src/intelligence/index.ts:256](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L256) ##### timing? > `optional` **timing?**: `object` -Defined in: [src/intelligence/index.ts:251](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L251) +Defined in: [src/intelligence/index.ts:257](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L257) ###### startedAt @@ -2608,7 +2608,7 @@ Defined in: [src/intelligence/index.ts:251](https://github.com/tangle-network/ag > `optional` **tokens?**: `object` -Defined in: [src/intelligence/index.ts:252](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L252) +Defined in: [src/intelligence/index.ts:258](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L258) ###### input @@ -2630,7 +2630,7 @@ Defined in: [src/intelligence/index.ts:252](https://github.com/tangle-network/ag > `optional` **error?**: `object` -Defined in: [src/intelligence/index.ts:258](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L258) +Defined in: [src/intelligence/index.ts:264](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L264) ###### name @@ -2648,7 +2648,7 @@ Defined in: [src/intelligence/index.ts:258](https://github.com/tangle-network/ag > `optional` **candidateExecution?**: `CandidateExecutionEvidence` -Defined in: [src/intelligence/index.ts:260](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L260) +Defined in: [src/intelligence/index.ts:266](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L266) Exact proposal → review → execution → receipt linkage for candidate runs. @@ -2656,7 +2656,7 @@ Exact proposal → review → execution → receipt linkage for candidate runs. ### RunReport -Defined in: [src/intelligence/index.ts:269](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L269) +Defined in: [src/intelligence/index.ts:275](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L275) What an agent reports (via `applied.record`) to enrich the [RunRecord](#runrecord) sent for its call. All optional — an un-recorded run still sends input/output @@ -2669,79 +2669,79 @@ as pure inference (the base stream). > `optional` **success?**: `boolean` -Defined in: [src/intelligence/index.ts:270](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L270) +Defined in: [src/intelligence/index.ts:276](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L276) ##### score? > `optional` **score?**: `number` -Defined in: [src/intelligence/index.ts:271](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L271) +Defined in: [src/intelligence/index.ts:277](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L277) ##### usage? > `optional` **usage?**: `Partial`\<[`UsageSplit`](#usagesplit)\> -Defined in: [src/intelligence/index.ts:272](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L272) +Defined in: [src/intelligence/index.ts:278](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L278) ##### costUsd? > `optional` **costUsd?**: `number` -Defined in: [src/intelligence/index.ts:273](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L273) +Defined in: [src/intelligence/index.ts:279](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L279) ##### model? > `optional` **model?**: `string` -Defined in: [src/intelligence/index.ts:274](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L274) +Defined in: [src/intelligence/index.ts:280](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L280) ##### provider? > `optional` **provider?**: `string` -Defined in: [src/intelligence/index.ts:275](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L275) +Defined in: [src/intelligence/index.ts:281](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L281) ##### loopEvents? > `optional` **loopEvents?**: [`LoopTraceEvent`](runtime.md#looptraceevent)[] -Defined in: [src/intelligence/index.ts:276](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L276) +Defined in: [src/intelligence/index.ts:282](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L282) ##### runtimeEvents? > `optional` **runtimeEvents?**: [`RuntimeStreamEvent`](index.md#runtimestreamevent)[] -Defined in: [src/intelligence/index.ts:277](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L277) +Defined in: [src/intelligence/index.ts:283](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L283) ##### profile? > `optional` **profile?**: `AgentProfile` -Defined in: [src/intelligence/index.ts:278](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L278) +Defined in: [src/intelligence/index.ts:284](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L284) ##### sessionId? > `optional` **sessionId?**: `string` -Defined in: [src/intelligence/index.ts:279](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L279) +Defined in: [src/intelligence/index.ts:285](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L285) ##### harness? > `optional` **harness?**: `string` -Defined in: [src/intelligence/index.ts:280](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L280) +Defined in: [src/intelligence/index.ts:286](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L286) ##### commitSha? > `optional` **commitSha?**: `string` -Defined in: [src/intelligence/index.ts:281](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L281) +Defined in: [src/intelligence/index.ts:287](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L287) ##### tokens? > `optional` **tokens?**: `object` -Defined in: [src/intelligence/index.ts:282](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L282) +Defined in: [src/intelligence/index.ts:288](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L288) ###### input @@ -2763,7 +2763,7 @@ Defined in: [src/intelligence/index.ts:282](https://github.com/tangle-network/ag > `optional` **error?**: `object` -Defined in: [src/intelligence/index.ts:283](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L283) +Defined in: [src/intelligence/index.ts:289](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L289) ###### name @@ -2781,13 +2781,13 @@ Defined in: [src/intelligence/index.ts:283](https://github.com/tangle-network/ag > `optional` **candidateExecution?**: `CandidateExecutionEvidence` -Defined in: [src/intelligence/index.ts:284](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L284) +Defined in: [src/intelligence/index.ts:290](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L290) *** ### RepoConfig -Defined in: [src/intelligence/index.ts:290](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L290) +Defined in: [src/intelligence/index.ts:296](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L296) Repo coordinates a product may declare for the (later) Gated-PR mode. The Observe slice only records their PRESENCE for `doctor()`; it never touches @@ -2799,25 +2799,25 @@ Repo coordinates a product may declare for the (later) Gated-PR mode. The > **owner**: `string` -Defined in: [src/intelligence/index.ts:291](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L291) +Defined in: [src/intelligence/index.ts:297](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L297) ##### name > **name**: `string` -Defined in: [src/intelligence/index.ts:292](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L292) +Defined in: [src/intelligence/index.ts:298](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L298) ##### baseBranch > **baseBranch**: `string` -Defined in: [src/intelligence/index.ts:293](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L293) +Defined in: [src/intelligence/index.ts:299](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L299) *** ### IntelligenceConfig -Defined in: [src/intelligence/index.ts:299](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L299) +Defined in: [src/intelligence/index.ts:305](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L305) Client configuration. `project` + `apiKey` are the Observe minimum; the rest tune effort, endpoint, redaction, and (for `doctor()` readiness) @@ -2833,7 +2833,7 @@ Client configuration. `project` + `apiKey` are the Observe minimum; the > **project**: `string` -Defined in: [src/intelligence/index.ts:301](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L301) +Defined in: [src/intelligence/index.ts:307](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L307) Stable project id — the tenant dimension every trace is tagged with. @@ -2841,7 +2841,7 @@ Stable project id — the tenant dimension every trace is tagged with. > `optional` **apiKey?**: `string` -Defined in: [src/intelligence/index.ts:303](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L303) +Defined in: [src/intelligence/index.ts:309](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L309) Bearer key for the Intelligence ingest. Reads `TANGLE_API_KEY` when omitted. @@ -2849,7 +2849,7 @@ Bearer key for the Intelligence ingest. Reads `TANGLE_API_KEY` when omitted. > `optional` **effort?**: [`EffortTier`](#efforttier) \| \{ `tier`: [`EffortTier`](#efforttier); `overrides?`: `Partial`\<[`EffortSettings`](#effortsettings)\>; \} -Defined in: [src/intelligence/index.ts:305](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L305) +Defined in: [src/intelligence/index.ts:311](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L311) Effort tier (default `'standard'`) plus optional per-field overrides. @@ -2857,7 +2857,7 @@ Effort tier (default `'standard'`) plus optional per-field overrides. > `optional` **baseUrl?**: `string` -Defined in: [src/intelligence/index.ts:313](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L313) +Defined in: [src/intelligence/index.ts:319](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L319) The ONE Tangle Intelligence base URL — both the send (OTLP `/v1/otlp`) and receive (`/v1/profiles/:target/composed`) paths derive from it. Reads @@ -2869,7 +2869,7 @@ key the ingest requires); absent a key, export is a no-op. > `optional` **redact?**: `false` \| [`Redactor`](#redactor) -Defined in: [src/intelligence/index.ts:319](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L319) +Defined in: [src/intelligence/index.ts:325](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L325) Redaction hook run over every exported input/output. A function replaces the default scrubber; `false` opts out entirely (raw fidelity, caller has @@ -2879,7 +2879,7 @@ sanitized upstream); omitted ⇒ the built-in `defaultRedactor`. > `optional` **surfaces?**: `string`[] -Defined in: [src/intelligence/index.ts:321](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L321) +Defined in: [src/intelligence/index.ts:327](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L327) Mutable surfaces a later PR mode would edit. Recorded for `doctor()` only. @@ -2887,7 +2887,7 @@ Mutable surfaces a later PR mode would edit. Recorded for `doctor()` only. > `optional` **checks?**: `string`[] -Defined in: [src/intelligence/index.ts:323](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L323) +Defined in: [src/intelligence/index.ts:329](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L329) Verification checks a later PR mode would gate on. Recorded for `doctor()` only. @@ -2895,7 +2895,7 @@ Verification checks a later PR mode would gate on. Recorded for `doctor()` only. > `optional` **repo?**: [`RepoConfig`](#repoconfig) -Defined in: [src/intelligence/index.ts:325](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L325) +Defined in: [src/intelligence/index.ts:331](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L331) Repo access a later PR mode would need. Recorded for `doctor()` only. @@ -2903,7 +2903,7 @@ Repo access a later PR mode would need. Recorded for `doctor()` only. > `optional` **profile?**: `AgentProfile` -Defined in: [src/intelligence/index.ts:327](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L327) +Defined in: [src/intelligence/index.ts:333](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L333) Full canonical profile used for this agent. Exported redacted with a stable hash. @@ -2911,7 +2911,7 @@ Full canonical profile used for this agent. Exported redacted with a stable hash > `optional` **commitSha?**: `string` -Defined in: [src/intelligence/index.ts:329](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L329) +Defined in: [src/intelligence/index.ts:335](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L335) Commit that produced the running agent, when known. @@ -2919,7 +2919,7 @@ Commit that produced the running agent, when known. > `optional` **runtimeTelemetry?**: [`RuntimeTelemetryOptions`](index.md#runtimetelemetryoptions) -Defined in: [src/intelligence/index.ts:331](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L331) +Defined in: [src/intelligence/index.ts:337](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L337) Runtime-event payload policy. Tool inputs/results remain off unless explicitly enabled. @@ -2927,7 +2927,7 @@ Runtime-event payload policy. Tool inputs/results remain off unless explicitly e > `optional` **payloadAttributes?**: `"metadata"` \| `"full"` -Defined in: [src/intelligence/index.ts:338](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L338) +Defined in: [src/intelligence/index.ts:344](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L344) Payloads are metadata-only by default: the run span carries a stable hash and UTF-8 byte count, but not the redacted content. Set `full` only when @@ -2938,7 +2938,7 @@ inputs, outputs, and profiles. ### TraceMeta -Defined in: [src/intelligence/index.ts:342](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L342) +Defined in: [src/intelligence/index.ts:348](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L348) Metadata describing one traced run. `runId`/`traceId` default to fresh ids. @@ -2948,7 +2948,7 @@ Metadata describing one traced run. `runId`/`traceId` default to fresh ids. > `optional` **input?**: `unknown` -Defined in: [src/intelligence/index.ts:344](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L344) +Defined in: [src/intelligence/index.ts:350](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L350) The run's input — exported through the redactor. @@ -2956,7 +2956,7 @@ The run's input — exported through the redactor. > `optional` **runId?**: `string` -Defined in: [src/intelligence/index.ts:346](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L346) +Defined in: [src/intelligence/index.ts:352](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L352) Stable run id. Defaults to a fresh id. @@ -2964,7 +2964,7 @@ Stable run id. Defaults to a fresh id. > `optional` **traceId?**: `string` -Defined in: [src/intelligence/index.ts:348](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L348) +Defined in: [src/intelligence/index.ts:354](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L354) 32-hex trace id. Defaults to a fresh id. @@ -2972,7 +2972,7 @@ Defined in: [src/intelligence/index.ts:348](https://github.com/tangle-network/ag > `optional` **model?**: `string` -Defined in: [src/intelligence/index.ts:350](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L350) +Defined in: [src/intelligence/index.ts:356](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L356) Model id, when known — stamped on the span. @@ -2980,7 +2980,7 @@ Model id, when known — stamped on the span. > `optional` **provider?**: `string` -Defined in: [src/intelligence/index.ts:352](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L352) +Defined in: [src/intelligence/index.ts:358](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L358) Provider name, when known — stamped on the span. @@ -2988,7 +2988,7 @@ Provider name, when known — stamped on the span. > `optional` **labels?**: `Record`\<`string`, `string` \| `number` \| `boolean`\> -Defined in: [src/intelligence/index.ts:354](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L354) +Defined in: [src/intelligence/index.ts:360](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L360) Arbitrary extra labels (string/number/boolean) stamped on the span. @@ -2996,7 +2996,7 @@ Arbitrary extra labels (string/number/boolean) stamped on the span. ### TraceHandle -Defined in: [src/intelligence/index.ts:363](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L363) +Defined in: [src/intelligence/index.ts:369](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L369) The trace handle a `traceRun` body records into. `recordOutput` captures the agent's result (redacted on export); `recordOutcome` captures the scored @@ -3009,7 +3009,7 @@ an un-recorded run still exports a span with whatever was set. > **recordOutput**(`output`): `void` -Defined in: [src/intelligence/index.ts:365](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L365) +Defined in: [src/intelligence/index.ts:371](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L371) Capture the run's output. Exported through the redactor. @@ -3027,7 +3027,7 @@ Capture the run's output. Exported through the redactor. > **recordOutcome**(`outcome`): `void` -Defined in: [src/intelligence/index.ts:372](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L372) +Defined in: [src/intelligence/index.ts:378](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L378) Capture the run's outcome. `usage` defaults to inference-only (`intelligenceUsd: 0`) — the OFF baseline; an intelligence-enabled run @@ -3062,7 +3062,7 @@ treated as pure inference. ### RecordTraceMeta -Defined in: [src/intelligence/index.ts:381](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L381) +Defined in: [src/intelligence/index.ts:387](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L387) Metadata for [IntelligenceClient.recordTrace](#recordtrace). @@ -3072,7 +3072,7 @@ Metadata for [IntelligenceClient.recordTrace](#recordtrace). > `optional` **traceId?**: `string` -Defined in: [src/intelligence/index.ts:383](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L383) +Defined in: [src/intelligence/index.ts:389](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L389) 32-hex trace id to anchor every span to. Defaults to a fresh id. @@ -3080,7 +3080,7 @@ Defined in: [src/intelligence/index.ts:383](https://github.com/tangle-network/ag > `optional` **rootParentSpanId?**: `string` -Defined in: [src/intelligence/index.ts:386](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L386) +Defined in: [src/intelligence/index.ts:392](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L392) Span id of an enclosing span the loop root should parent under (e.g. a `traceRun` span). Omitted ⇒ the loop root is the trace root. @@ -3089,7 +3089,7 @@ Span id of an enclosing span the loop root should parent under (e.g. a ### TraceOutcome -Defined in: [src/intelligence/index.ts:391](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L391) +Defined in: [src/intelligence/index.ts:397](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L397) The resolved outcome of one traced run, surfaced on the export span and available to the caller for downstream billing assertions. @@ -3100,25 +3100,25 @@ The resolved outcome of one traced run, surfaced on the export span and > **runId**: `string` -Defined in: [src/intelligence/index.ts:392](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L392) +Defined in: [src/intelligence/index.ts:398](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L398) ##### traceId > **traceId**: `string` -Defined in: [src/intelligence/index.ts:393](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L393) +Defined in: [src/intelligence/index.ts:399](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L399) ##### project > **project**: `string` -Defined in: [src/intelligence/index.ts:394](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L394) +Defined in: [src/intelligence/index.ts:400](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L400) ##### effort > **effort**: [`EffortSettings`](#effortsettings) -Defined in: [src/intelligence/index.ts:396](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L396) +Defined in: [src/intelligence/index.ts:402](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L402) The resolved effort settings this run executed under. @@ -3126,7 +3126,7 @@ The resolved effort settings this run executed under. > **intelligenceOff**: `boolean` -Defined in: [src/intelligence/index.ts:398](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L398) +Defined in: [src/intelligence/index.ts:404](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L404) True when this run ran as pure passthrough (the OFF floor). @@ -3134,19 +3134,19 @@ True when this run ran as pure passthrough (the OFF floor). > `optional` **success?**: `boolean` -Defined in: [src/intelligence/index.ts:399](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L399) +Defined in: [src/intelligence/index.ts:405](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L405) ##### score? > `optional` **score?**: `number` -Defined in: [src/intelligence/index.ts:400](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L400) +Defined in: [src/intelligence/index.ts:406](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L406) ##### usage > **usage**: [`UsageSplit`](#usagesplit) -Defined in: [src/intelligence/index.ts:402](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L402) +Defined in: [src/intelligence/index.ts:408](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L408) Per-class billing split. `intelligenceUsd` is `0` at the OFF tier. @@ -3154,7 +3154,7 @@ Per-class billing split. `intelligenceUsd` is `0` at the OFF tier. ### IntelligenceClient -Defined in: [src/intelligence/index.ts:406](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L406) +Defined in: [src/intelligence/index.ts:412](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L412) The Observe-mode Intelligence client. @@ -3164,7 +3164,7 @@ The Observe-mode Intelligence client. > `readonly` **project**: `string` -Defined in: [src/intelligence/index.ts:408](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L408) +Defined in: [src/intelligence/index.ts:414](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L414) The resolved project id. @@ -3172,7 +3172,7 @@ The resolved project id. > `readonly` **effort**: [`EffortSettings`](#effortsettings) -Defined in: [src/intelligence/index.ts:410](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L410) +Defined in: [src/intelligence/index.ts:416](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L416) The resolved effort settings. @@ -3182,7 +3182,7 @@ The resolved effort settings. > **traceRun**\<`T`\>(`meta`, `fn`): `Promise`\<`T`\> -Defined in: [src/intelligence/index.ts:416](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L416) +Defined in: [src/intelligence/index.ts:422](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L422) Run `fn` under a trace, export one span best-effort, and return whatever `fn` returns. Telemetry-export failures are swallowed; an error THROWN by @@ -3212,7 +3212,7 @@ Run `fn` under a trace, export one span best-effort, and return whatever > **recordTrace**(`events`, `meta?`): `string` -Defined in: [src/intelligence/index.ts:426](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L426) +Defined in: [src/intelligence/index.ts:432](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L432) Export a run's full loop topology — the ordered `LoopTraceEvent` stream a `runLoop`/`Supervisor` run emits — as a nested OTLP span tree (loop → round → @@ -3240,7 +3240,7 @@ readonly [`LoopTraceEvent`](runtime.md#looptraceevent)[] > **exportRunRecord**(`record`): `string` -Defined in: [src/intelligence/index.ts:434](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L434) +Defined in: [src/intelligence/index.ts:440](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L440) Send one typed [RunRecord](#runrecord) — the run's flat span (input/output/outcome/ usage/model/provider, redacted) plus, when `loopEvents` are present, the @@ -3262,7 +3262,7 @@ Best-effort: export failures are swallowed. Returns the record's `traceId`. > **freshRunId**(): `string` -Defined in: [src/intelligence/index.ts:436](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L436) +Defined in: [src/intelligence/index.ts:442](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L442) Mint a fresh run id (`run-`). @@ -3274,7 +3274,7 @@ Mint a fresh run id (`run-`). > **freshTraceId**(): `string` -Defined in: [src/intelligence/index.ts:438](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L438) +Defined in: [src/intelligence/index.ts:444](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L444) Mint a fresh 32-hex trace id. @@ -3286,7 +3286,7 @@ Mint a fresh 32-hex trace id. > **doctor**(): [`DoctorReport`](#doctorreport) -Defined in: [src/intelligence/index.ts:444](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L444) +Defined in: [src/intelligence/index.ts:450](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L450) Network-free readiness report: which adoption modes are reachable given this config. Observe is always reachable; Recommend needs outcomes; PR @@ -3300,7 +3300,7 @@ needs checks + surfaces + repo. > **flush**(): `Promise`\<`void`\> -Defined in: [src/intelligence/index.ts:446](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L446) +Defined in: [src/intelligence/index.ts:452](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L452) Flush any pending export spans. Best-effort; resolves even if export fails. @@ -3312,7 +3312,7 @@ Flush any pending export spans. Best-effort; resolves even if export fails. ### ModeReadiness -Defined in: [src/intelligence/index.ts:450](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L450) +Defined in: [src/intelligence/index.ts:456](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L456) One mode's readiness verdict. @@ -3322,13 +3322,13 @@ One mode's readiness verdict. > **ready**: `boolean` -Defined in: [src/intelligence/index.ts:451](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L451) +Defined in: [src/intelligence/index.ts:457](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L457) ##### missing > **missing**: `string`[] -Defined in: [src/intelligence/index.ts:453](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L453) +Defined in: [src/intelligence/index.ts:459](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L459) Inputs this mode still needs, when not ready. Empty when ready. @@ -3336,7 +3336,7 @@ Inputs this mode still needs, when not ready. Empty when ready. ### DoctorReport -Defined in: [src/intelligence/index.ts:457](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L457) +Defined in: [src/intelligence/index.ts:463](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L463) The `doctor()` readiness report — Mode-readiness without any network call. @@ -3346,19 +3346,19 @@ The `doctor()` readiness report — Mode-readiness without any network call. > **project**: `string` -Defined in: [src/intelligence/index.ts:458](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L458) +Defined in: [src/intelligence/index.ts:464](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L464) ##### effort > **effort**: [`EffortSettings`](#effortsettings) -Defined in: [src/intelligence/index.ts:459](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L459) +Defined in: [src/intelligence/index.ts:465](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L465) ##### exportConfigured > **exportConfigured**: `boolean` -Defined in: [src/intelligence/index.ts:461](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L461) +Defined in: [src/intelligence/index.ts:467](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L467) True when an OTLP endpoint is configured (export will actually ship). @@ -3366,7 +3366,7 @@ True when an OTLP endpoint is configured (export will actually ship). > **modes**: `object` -Defined in: [src/intelligence/index.ts:462](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L462) +Defined in: [src/intelligence/index.ts:468](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L468) ###### observe @@ -3424,19 +3424,19 @@ Defined in: [src/intelligence/optimization-receipt.ts:29](https://github.com/tan Defined in: [src/intelligence/optimization-receipt.ts:30](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/optimization-receipt.ts#L30) -##### model? +##### models? -> `optional` **model?**: `object` +> `optional` **models?**: `object` Defined in: [src/intelligence/optimization-receipt.ts:31](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/optimization-receipt.ts#L31) -###### role +###### candidate? -> **role**: `"candidate"` +> `optional` **candidate?**: `AgentProfileModelHints` -###### identity +###### optimizer? -> **identity**: `AgentProfileModelHints` +> `optional` **optimizer?**: `string` ##### usage @@ -3444,13 +3444,13 @@ Defined in: [src/intelligence/optimization-receipt.ts:31](https://github.com/tan Defined in: [src/intelligence/optimization-receipt.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/optimization-receipt.ts#L35) -###### evaluations +###### optimizerEvaluations -> **evaluations**: `number` +> **optimizerEvaluations**: `number` -###### tokens? +###### optimizerTokens? -> `optional` **tokens?**: `OptimizationTokenUsage` +> `optional` **optimizerTokens?**: `OptimizationTokenUsage` ##### cost @@ -3458,17 +3458,17 @@ Defined in: [src/intelligence/optimization-receipt.ts:35](https://github.com/tan Defined in: [src/intelligence/optimization-receipt.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/optimization-receipt.ts#L39) -###### totalUsd +###### optimization -> **totalUsd**: `number` +> **optimization**: [`OptimizationReceiptCost`](#optimizationreceiptcost) -###### accountingComplete +###### finalTest -> **accountingComplete**: `boolean` +> **finalTest**: [`OptimizationReceiptCost`](#optimizationreceiptcost) -###### incompleteReasons +###### total -> **incompleteReasons**: `string`[] +> **total**: [`OptimizationReceiptCost`](#optimizationreceiptcost) ##### invocation @@ -3510,6 +3510,32 @@ Defined in: [src/intelligence/optimization-receipt.ts:52](https://github.com/tan *** +### OptimizationReceiptCost + +Defined in: [src/intelligence/optimization-receipt.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/optimization-receipt.ts#L55) + +#### Properties + +##### totalUsd + +> **totalUsd**: `number` + +Defined in: [src/intelligence/optimization-receipt.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/optimization-receipt.ts#L56) + +##### accountingComplete + +> **accountingComplete**: `boolean` + +Defined in: [src/intelligence/optimization-receipt.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/optimization-receipt.ts#L57) + +##### incompleteReasons + +> **incompleteReasons**: `string`[] + +Defined in: [src/intelligence/optimization-receipt.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/optimization-receipt.ts#L58) + +*** + ### AgentImprovementProfileReplacement Defined in: [src/intelligence/profile-activation.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/profile-activation.ts#L37) @@ -3862,7 +3888,7 @@ Defined in: [src/intelligence/with-intelligence.ts:83](https://github.com/tangle > **project**: `string` -Defined in: [src/intelligence/index.ts:301](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L301) +Defined in: [src/intelligence/index.ts:307](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L307) Stable project id — the tenant dimension every trace is tagged with. @@ -3874,7 +3900,7 @@ Stable project id — the tenant dimension every trace is tagged with. > `optional` **apiKey?**: `string` -Defined in: [src/intelligence/index.ts:303](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L303) +Defined in: [src/intelligence/index.ts:309](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L309) Bearer key for the Intelligence ingest. Reads `TANGLE_API_KEY` when omitted. @@ -3886,7 +3912,7 @@ Bearer key for the Intelligence ingest. Reads `TANGLE_API_KEY` when omitted. > `optional` **effort?**: [`EffortTier`](#efforttier) \| \{ `tier`: [`EffortTier`](#efforttier); `overrides?`: `Partial`\<[`EffortSettings`](#effortsettings)\>; \} -Defined in: [src/intelligence/index.ts:305](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L305) +Defined in: [src/intelligence/index.ts:311](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L311) Effort tier (default `'standard'`) plus optional per-field overrides. @@ -3898,7 +3924,7 @@ Effort tier (default `'standard'`) plus optional per-field overrides. > `optional` **baseUrl?**: `string` -Defined in: [src/intelligence/index.ts:313](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L313) +Defined in: [src/intelligence/index.ts:319](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L319) The ONE Tangle Intelligence base URL — both the send (OTLP `/v1/otlp`) and receive (`/v1/profiles/:target/composed`) paths derive from it. Reads @@ -3914,7 +3940,7 @@ key the ingest requires); absent a key, export is a no-op. > `optional` **redact?**: `false` \| [`Redactor`](#redactor) -Defined in: [src/intelligence/index.ts:319](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L319) +Defined in: [src/intelligence/index.ts:325](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L325) Redaction hook run over every exported input/output. A function replaces the default scrubber; `false` opts out entirely (raw fidelity, caller has @@ -3928,7 +3954,7 @@ sanitized upstream); omitted ⇒ the built-in `defaultRedactor`. > `optional` **surfaces?**: `string`[] -Defined in: [src/intelligence/index.ts:321](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L321) +Defined in: [src/intelligence/index.ts:327](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L327) Mutable surfaces a later PR mode would edit. Recorded for `doctor()` only. @@ -3940,7 +3966,7 @@ Mutable surfaces a later PR mode would edit. Recorded for `doctor()` only. > `optional` **checks?**: `string`[] -Defined in: [src/intelligence/index.ts:323](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L323) +Defined in: [src/intelligence/index.ts:329](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L329) Verification checks a later PR mode would gate on. Recorded for `doctor()` only. @@ -3952,7 +3978,7 @@ Verification checks a later PR mode would gate on. Recorded for `doctor()` only. > `optional` **repo?**: [`RepoConfig`](#repoconfig) -Defined in: [src/intelligence/index.ts:325](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L325) +Defined in: [src/intelligence/index.ts:331](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L331) Repo access a later PR mode would need. Recorded for `doctor()` only. @@ -3964,7 +3990,7 @@ Repo access a later PR mode would need. Recorded for `doctor()` only. > `optional` **profile?**: `AgentProfile` -Defined in: [src/intelligence/index.ts:327](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L327) +Defined in: [src/intelligence/index.ts:333](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L333) Full canonical profile used for this agent. Exported redacted with a stable hash. @@ -3976,7 +4002,7 @@ Full canonical profile used for this agent. Exported redacted with a stable hash > `optional` **commitSha?**: `string` -Defined in: [src/intelligence/index.ts:329](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L329) +Defined in: [src/intelligence/index.ts:335](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L335) Commit that produced the running agent, when known. @@ -3988,7 +4014,7 @@ Commit that produced the running agent, when known. > `optional` **runtimeTelemetry?**: [`RuntimeTelemetryOptions`](index.md#runtimetelemetryoptions) -Defined in: [src/intelligence/index.ts:331](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L331) +Defined in: [src/intelligence/index.ts:337](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L337) Runtime-event payload policy. Tool inputs/results remain off unless explicitly enabled. @@ -4000,7 +4026,7 @@ Runtime-event payload policy. Tool inputs/results remain off unless explicitly e > `optional` **payloadAttributes?**: `"metadata"` \| `"full"` -Defined in: [src/intelligence/index.ts:338](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L338) +Defined in: [src/intelligence/index.ts:344](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L344) Payloads are metadata-only by default: the run span carries a stable hash and UTF-8 byte count, but not the redacted content. Set `full` only when @@ -4312,7 +4338,7 @@ Defined in: [src/intelligence/improvement-surfaces.ts:54](https://github.com/tan > **UsageClass** = `"inference"` \| `"intelligence"` -Defined in: [src/intelligence/index.ts:209](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L209) +Defined in: [src/intelligence/index.ts:215](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L215) Usage class for billing. Base-stream tokens bill `'inference'`; every intelligence spawn (analyst, corpus, loop) bills `'intelligence'`. The @@ -5302,7 +5328,7 @@ so exact replacement requires a reset record followed by a set record. > **createIntelligenceClient**(`config`): [`IntelligenceClient`](#intelligenceclient) -Defined in: [src/intelligence/index.ts:528](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L528) +Defined in: [src/intelligence/index.ts:534](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/index.ts#L534) Create an Observe-mode Intelligence client. Resolves effort, the base URL, and the redactor up front; the exporter is built lazily and is `undefined` when no @@ -5321,11 +5347,31 @@ and best-effort export must never spam an unauthenticated plane). *** +### createOptimizationActivationReceipt() + +> **createOptimizationActivationReceipt**(`improvement`): [`OptimizationActivationReceipt`](#optimizationactivationreceipt) \| `undefined` + +Defined in: [src/intelligence/optimization-receipt.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/optimization-receipt.ts#L64) + +Build a detached receipt only for methods backed by an identified external optimizer. + +#### Parameters + +##### improvement + +[`ImproveMethodResult`](index.md#improvemethodresult) + +#### Returns + +[`OptimizationActivationReceipt`](#optimizationactivationreceipt) \| `undefined` + +*** + ### optimizationActivationReceiptFromMetadata() > **optimizationActivationReceiptFromMetadata**(`metadata`): [`OptimizationActivationReceipt`](#optimizationactivationreceipt) \| `undefined` -Defined in: [src/intelligence/optimization-receipt.ts:122](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/optimization-receipt.ts#L122) +Defined in: [src/intelligence/optimization-receipt.ts:130](https://github.com/tangle-network/agent-runtime/blob/main/src/intelligence/optimization-receipt.ts#L130) Read and verify the optimizer evidence carried by a measured proposal. @@ -5506,7 +5552,7 @@ Cycle-safe (a seen-set short-circuits self-referential payloads to > **resolveRedactor**(`redact`): [`Redactor`](#redactor) -Defined in: [src/redact.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/redact.ts#L95) +Defined in: [src/redact.ts:113](https://github.com/tangle-network/agent-runtime/blob/main/src/redact.ts#L113) Resolve the redactor a client uses. A caller-supplied hook handles domain-specific values first, then the built-in scrubber still removes diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index c99d2b6e..292cb3ae 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.104.0` and `@tangle-network/agent-eval@0.126.5` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.104.0` and `@tangle-network/agent-eval@0.126.6` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -15,7 +15,7 @@ Every subpath this package declares in `package.json` `exports`. Reach for these ### Root — task lifecycle, conversation, RSI verbs, observability -Import from `@tangle-network/agent-runtime` — 386 exports. +Import from `@tangle-network/agent-runtime` — 389 exports. | Symbol | Kind | Summary | |---|---|---| @@ -193,6 +193,7 @@ Import from `@tangle-network/agent-runtime` — 386 exports. | `ConversationJournalEntry` | interface | Durable conversation transcript — survives a driver process crash mid-run. | | `D1DatabaseLike` | interface | Structural type matching the surface of `D1Database` we depend on, so the | | `DriverLoopGeneratorOptions` | interface | `driverLoopGenerator` — the driver→worker `CandidateGenerator`: the build | +| `ImproveCandidateValidationInput` | interface | Exact materialized profile presented for validation before any candidate run. | | `ImproveCost` | interface | Normalized spend reported for one Runtime improvement run. | | `ImproveLineage` | interface | Optimizer ancestry sealed into downstream candidate experiments. | | `ImproveProfileComponents` | interface | Caller-owned mapping for optimizing several profile fields as one candidate. | @@ -252,7 +253,7 @@ Import from `@tangle-network/agent-runtime` — 386 exports. | `VerifiedAgentCandidateTaskOutcome` | type | Branded task outcome that has survived independent evaluator verification. | | `Verifier` | type | Verifies the edited worktree. Sync or async; throws only on a setup fault | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidatePreparationEvidence`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateVerificationPorts`, `AgentCandidateWorkspaceArchiveLimits`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgenticGeneratorShotReceipt`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `BackendCallPolicy`, `CanonicalCandidateDocument`, `CaptureAgentCandidateWorkspaceOptions`, `CapturedAgentCandidateWorkspace`, `ChatTurnHooks`, `ChatTurnResult`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `CreateAgentCandidateWorkspacePortOptions`, `CreateKnowledgeImprovementActivationExecutorOptions`, `CreateProtectedAgentCandidateModelPortOptions`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `DisposePreparedAgentCandidateOptions`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `ExactProcessCandidateExecutorOptions`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `HaltContext`, `HaltSignal`, `ImproveCodeOptions`, `ImproveCodeResult`, `ImprovementCodeCandidate`, `ImprovementProfileCandidate`, `ImproveMethodContext`, `ImproveMethodResult`, `ImproveSkillsOptions`, `KnowledgeImprovementActivationExecutor`, `KnowledgeImprovementCandidatePair`, `KnowledgeImprovementExperimentBundles`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `OtelAttribute`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `RecoverExpiredAgentCandidateOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolveAgentBackendOptions`, `ResolvedAgentCandidateContainer`, `ResolvedChatModel`, `RunChatTurnInput`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeEventOtelOptions`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunHandle`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeTelemetryOptions`, `RunToolLoopOptions`, `SanitizedKnowledgeReadinessReport`, `StreamToolLoopOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `ToAnalystFindingsOptions`, `ToolLoopResult`, `VerifiedAgentCandidate`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ControlDecision`, `ConversationStreamEvent`, `DeepReadonly`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `ImprovementCandidate`, `ImproveMethodSource`, `ImproveOptimizationRunOptions`, `ImproveProfileSurface`, `ImproveResult`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeStreamEvent`, `StreamToolLoopYield`, `SupervisedKnowledgeUpdater`, `ToolLoopEvent`, `TurnOrder`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidatePreparationEvidence`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateVerificationPorts`, `AgentCandidateWorkspaceArchiveLimits`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgenticGeneratorShotReceipt`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `BackendCallPolicy`, `CanonicalCandidateDocument`, `CaptureAgentCandidateWorkspaceOptions`, `CapturedAgentCandidateWorkspace`, `ChatTurnHooks`, `ChatTurnResult`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `CreateAgentCandidateWorkspacePortOptions`, `CreateKnowledgeImprovementActivationExecutorOptions`, `CreateProtectedAgentCandidateModelPortOptions`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `DisposePreparedAgentCandidateOptions`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `ExactProcessCandidateExecutorOptions`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `HaltContext`, `HaltSignal`, `ImproveCodeOptions`, `ImproveCodeResult`, `ImprovementCodeCandidate`, `ImprovementProfileCandidate`, `ImproveMethodContext`, `ImproveMethodResult`, `ImproveSkillsOptions`, `KnowledgeImprovementActivationExecutor`, `KnowledgeImprovementCandidatePair`, `KnowledgeImprovementExperimentBundles`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `OfficialSensitiveCandidateInput`, `OtelAttribute`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `RecoverExpiredAgentCandidateOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolveAgentBackendOptions`, `ResolvedAgentCandidateContainer`, `ResolvedChatModel`, `RunChatTurnInput`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeEventOtelOptions`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunHandle`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeTelemetryOptions`, `RunToolLoopOptions`, `SanitizedKnowledgeReadinessReport`, `StreamToolLoopOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `ToAnalystFindingsOptions`, `ToolLoopResult`, `VerifiedAgentCandidate`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ControlDecision`, `ConversationStreamEvent`, `DeepReadonly`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `ImproveCandidateValidator`, `ImprovementCandidate`, `ImproveMethodSource`, `ImproveOptimizationRunOptions`, `ImproveProfileSurface`, `ImproveResult`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeStreamEvent`, `StreamToolLoopYield`, `SupervisedKnowledgeUpdater`, `ToolLoopEvent`, `TurnOrder`. ### Vertical agent — manifest + surface proposal source @@ -333,7 +334,7 @@ Import from `@tangle-network/agent-runtime/conversation` — 53 exports. ### Intelligence SDK — Observe + provable-OFF billing -Import from `@tangle-network/agent-runtime/intelligence` — 143 exports. +Import from `@tangle-network/agent-runtime/intelligence` — 145 exports. | Symbol | Kind | Summary | |---|---|---| @@ -352,6 +353,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 143 exports. | `createCertifiedPromptSource` | function | Create the cached certified-prompt source — the ONE module-scope-cache + | | `createExactProcessCandidateExperimentExecutor` | function | Execute one signed experiment cell through any declared exact-process provider. | | `createIntelligenceClient` | function | Create an Observe-mode Intelligence client. Resolves effort, the base URL, and | +| `createOptimizationActivationReceipt` | function | Build a detached receipt only for methods backed by an identified external optimizer. | | `createProtectedExactProcessCandidateExperimentExecutor` | function | Compose host-owned execution ports with protected model access for one exact-process run. | | `defaultRedactor` | function | The built-in redactor. Walks objects and arrays; replaces values under | | `executeAgentCandidateExperimentCell` | function | Execute one exact arm, task, repetition, seed, and attempt through Runtime. | @@ -448,7 +450,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 143 exports. | `SubmitAgentImprovementProposalOutcome` | type | Typed result for proposal submission. A successful result contains the | | `UsageClass` | type | Usage class for billing. Base-stream tokens bill `'inference'`; every | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentCandidateExperimentCellPlacement`, `AgentImprovementActivationResultStore`, `AgentImprovementActivationTargetPlan`, `AgentImprovementActivationTransitionInput`, `AgentImprovementProfileReplacement`, `AgentImprovementProposal`, `AgentImprovementTargetProfileDiffOptions`, `CreateAgentImprovementActivationOptions`, `CreateAgentImprovementActivationResultOptions`, `CreateAgentImprovementProposalOptions`, `CreateExactProcessCandidateExperimentExecutorOptions`, `ExactProcessCandidateExperimentExecution`, `ExactProcessCandidateExperimentExecutor`, `ExecuteAgentCandidateExperimentCellOptions`, `ExecuteAgentImprovementActivationInput`, `ExecuteAgentImprovementActivationOptions`, `OptimizationActivationReceipt`, `ProposeAgentImprovementOptions`, `ProposeAgentImprovementResult`, `PullCertifiedOptions`, `ReviewAgentImprovementInput`, `RunAgentCandidateExperimentOptions`, `RunAgentCandidateExperimentResult`, `VerifyCandidateExecutionEvidenceOptions`, `AgentImprovementActivationIntent`, `AgentImprovementActivationOutcome`, `AgentImprovementActivationTargetIdentity`, `AgentImprovementProfileActivationPreparation`, `AgentImprovementProfileActivationTarget`, `AgentImprovementProfileSurface`, `AgentImprovementProfileTargetState`, `AgentImprovementProfileTargetTransition`, `AgentImprovementReviewDecision`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentCandidateExperimentCellPlacement`, `AgentImprovementActivationResultStore`, `AgentImprovementActivationTargetPlan`, `AgentImprovementActivationTransitionInput`, `AgentImprovementProfileReplacement`, `AgentImprovementProposal`, `AgentImprovementTargetProfileDiffOptions`, `CreateAgentImprovementActivationOptions`, `CreateAgentImprovementActivationResultOptions`, `CreateAgentImprovementProposalOptions`, `CreateExactProcessCandidateExperimentExecutorOptions`, `ExactProcessCandidateExperimentExecution`, `ExactProcessCandidateExperimentExecutor`, `ExecuteAgentCandidateExperimentCellOptions`, `ExecuteAgentImprovementActivationInput`, `ExecuteAgentImprovementActivationOptions`, `OptimizationActivationReceipt`, `OptimizationReceiptCost`, `ProposeAgentImprovementOptions`, `ProposeAgentImprovementResult`, `PullCertifiedOptions`, `ReviewAgentImprovementInput`, `RunAgentCandidateExperimentOptions`, `RunAgentCandidateExperimentResult`, `VerifyCandidateExecutionEvidenceOptions`, `AgentImprovementActivationIntent`, `AgentImprovementActivationOutcome`, `AgentImprovementActivationTargetIdentity`, `AgentImprovementProfileActivationPreparation`, `AgentImprovementProfileActivationTarget`, `AgentImprovementProfileSurface`, `AgentImprovementProfileTargetState`, `AgentImprovementProfileTargetTransition`, `AgentImprovementReviewDecision`. ### Recursive atom + loop kernel (alias of ./runtime) diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 6b1638e4..67754ace 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -6,8 +6,8 @@ Run pnpm docs:freshness after editing this file. --> > **Version 0.104.0.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. -> Agent Eval must satisfy `>=0.126.5 <0.127.0`. -> Sandbox must satisfy `>=0.12.0 <1.0.0`. +> `agent-eval` must satisfy `>=0.126.6 <0.127.0`. +> `sandbox` must satisfy `>=0.12.0 <1.0.0`. > Portable profile and tool-part types come from `@tangle-network/agent-interface` `>=0.32.0 <0.33.0`. > > **`./loops` is the runtime barrel**: `package.json` maps it to `src/runtime/index.ts`. Everything below labelled `/loops` is the recursive-atom + loop-kernel surface. @@ -103,7 +103,8 @@ A general "loop" primitive is the single most common modelling error in this rep | Pick / register a leaf backend, or bring your own agent | `createExecutor({ backend })` / `createExecutorRegistry()` / implement `Executor`: `/loops` | a per-vendor adapter or closed `inline\|sandbox\|cli` switch (won't report through the `UsageEvent` channel) | | Optimize text or named components with upstream GEPA | `officialGepa({ recipe, ... })`, passed as `improve(...).method` from root `.` | a local GEPA approximation, prompt mutation loop, or silent fallback when Python is unavailable | | Optimize one text surface with Microsoft SkillOpt | `officialSkillOpt({ trainer, optimizer, ... })`, passed as `improve(...).method` from root `.` | Runtime-owned SkillOpt search or a silent local fallback | -| Improve one profile coordinate | `improve(profile, { surface, executionRef, method, trainScenarios, selectionScenarios, testScenarios, judges, agent, costCeiling })` from root `.`; `executionRef` binds saved work to executable behavior, `agent` receives the exact complete candidate profile, and `costCeiling` limits the whole run | an implicit per-surface optimizer, a method that sees final-test cases, an unmeasured profile mutation, or separate optimizer and final-test spend limits | +| Improve one profile coordinate | `improve(profile, { surface, executionRef, method, trainScenarios, selectionScenarios, testScenarios, judges, agent, costCeiling })` from root `.`; `executionRef` binds saved work to executable behavior, `agent` receives the exact complete candidate profile, and the total-cost option limits the whole run | an implicit per-surface optimizer, a method that sees final-test cases, an unmeasured profile mutation, or separate optimizer and final-test spend limits | +| Inspect observed optimizer package, model, usage, cost, and resumed-run evidence before proposing a change | `createOptimizationActivationReceipt(result)` from `/intelligence` | reconstructing optimizer evidence from logs or trusting caller-authored metadata | | Compare complete optimization methods directly | `compareOptimizationMethods(...)` from `agent-eval/campaign` | comparing one method's training score to another method's final score | | Improve repository code | `improve({ surface: 'code', code, scenarios, judge, agent, budget })` from root `.` | passing code through a text optimizer or managing candidate worktrees in product code | | Decide ship/hold on a candidate (campaign context) | `defaultProductionGate({ holdoutScenarios, deltaThreshold })`; compose with `heldOutGate` / `composeGate`: `agent-eval/contract` | a raw `h1>h0` point comparison on the training set | diff --git a/package.json b/package.json index 36582c44..ccbef767 100644 --- a/package.json +++ b/package.json @@ -123,7 +123,7 @@ }, "devDependencies": { "@biomejs/biome": "^2.4.15", - "@tangle-network/agent-eval": "0.126.5", + "@tangle-network/agent-eval": "0.126.6", "@tangle-network/agent-interface": "0.32.0", "@tangle-network/sandbox": "^0.12.0", "@types/node": "^25.9.3", @@ -155,7 +155,7 @@ "license": "MIT", "packageManager": "pnpm@10.28.0", "peerDependencies": { - "@tangle-network/agent-eval": ">=0.126.5 <0.127.0", + "@tangle-network/agent-eval": ">=0.126.6 <0.127.0", "@tangle-network/agent-interface": ">=0.32.0 <0.33.0", "@tangle-network/sandbox": ">=0.12.0 <1.0.0", "playwright": "^1.40.0" @@ -169,7 +169,7 @@ } }, "dependencies": { - "@tangle-network/agent-knowledge": "5.0.0", + "@tangle-network/agent-knowledge": "5.0.1", "@tangle-network/agent-profile-materialize": "0.5.1", "tar-stream": "3.2.0" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ef25cf7c..ac70df05 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: dependencies: '@tangle-network/agent-knowledge': - specifier: 5.0.0 - version: 5.0.0(typescript@5.9.3) + specifier: 5.0.1 + version: 5.0.1(typescript@5.9.3) '@tangle-network/agent-profile-materialize': specifier: 0.5.1 version: 0.5.1 @@ -22,8 +22,8 @@ importers: specifier: ^2.4.15 version: 2.4.15 '@tangle-network/agent-eval': - specifier: 0.126.5 - version: 0.126.5(typescript@5.9.3) + specifier: 0.126.6 + version: 0.126.6(typescript@5.9.3) '@tangle-network/agent-interface': specifier: 0.32.0 version: 0.32.0 @@ -61,14 +61,14 @@ importers: bench: dependencies: '@tangle-network/agent-eval': - specifier: 0.126.5 - version: 0.126.5(typescript@5.9.3) + specifier: 0.126.6 + version: 0.126.6(typescript@5.9.3) '@tangle-network/agent-interface': specifier: 0.32.0 version: 0.32.0 '@tangle-network/agent-knowledge': - specifier: 5.0.0 - version: 5.0.0(typescript@5.9.3) + specifier: 5.0.1 + version: 5.0.1(typescript@5.9.3) '@tangle-network/agent-runtime': specifier: workspace:* version: link:.. @@ -686,8 +686,8 @@ packages: '@tangle-network/agent-core@0.4.20': resolution: {integrity: sha512-gJzZh5PqPJtWW6kMEfm3IP7CGibvHdVXM4uqCAuCy+Kvg9TlVVkzXqZPRt9gU44mjdw5hDGoVzZotFsPoHAlFw==} - '@tangle-network/agent-eval@0.126.5': - resolution: {integrity: sha512-nXgG+ULn2LwRs86d6TaWLsjL4Y0SrznD9GuVhSTMr3vkeludzzqBZsBEnHd2Zo454M71TT1vqdEtbX+pzJaaKA==} + '@tangle-network/agent-eval@0.126.6': + resolution: {integrity: sha512-gPZmwcanmzGEz226R1hTle/bsxGzd5qsDM03jtBSQBJfuou6+DJm3ddujjL4xjo+ucwgXwp0z/uwDAalZV2PfQ==} engines: {node: '>=20'} hasBin: true @@ -706,8 +706,8 @@ packages: '@tangle-network/agent-interface@0.32.0': resolution: {integrity: sha512-8GUiqdr9MZ+iedkx7KVL6jbiuW6jh7CzKPS3VDXbWherJjFbuf+ULsMKmc+wnfG7LvtjJgZGr5y+cZm+JN1YDA==} - '@tangle-network/agent-knowledge@5.0.0': - resolution: {integrity: sha512-VGq9UQdDZdPwwlrIVQEIr1xeSjubS5fIUdfjFDPNqDLCIajd6fsLY4OoPGOaTbr5hvXfkjKTvseANsiSnbA7bw==} + '@tangle-network/agent-knowledge@5.0.1': + resolution: {integrity: sha512-rp9i7C67U5Fr7tLArxa5WbGVBFa3mSPT1OaLvqlZv3avQVs6sMuVMtNejWWlCiv2NOBiQ4q1ucXTo2YIuLnDGQ==} engines: {node: '>=20.19.0'} hasBin: true @@ -1773,7 +1773,7 @@ snapshots: '@tangle-network/agent-interface': 0.32.0 zod: 4.4.3 - '@tangle-network/agent-eval@0.126.5(typescript@5.9.3)': + '@tangle-network/agent-eval@0.126.6(typescript@5.9.3)': dependencies: '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) '@ax-llm/ax': 23.0.1(zod@4.4.3) @@ -1815,9 +1815,9 @@ snapshots: '@noble/hashes': 1.8.0 zod: 4.4.3 - '@tangle-network/agent-knowledge@5.0.0(typescript@5.9.3)': + '@tangle-network/agent-knowledge@5.0.1(typescript@5.9.3)': dependencies: - '@tangle-network/agent-eval': 0.126.5(typescript@5.9.3) + '@tangle-network/agent-eval': 0.126.6(typescript@5.9.3) '@tangle-network/agent-interface': 0.32.0 proper-lockfile: 4.1.2 zod: 4.4.3 diff --git a/scripts/verify-official-optimizers-consumer.mjs b/scripts/verify-official-optimizers-consumer.mjs index 94cc26aa..16c9db8b 100644 --- a/scripts/verify-official-optimizers-consumer.mjs +++ b/scripts/verify-official-optimizers-consumer.mjs @@ -8,6 +8,10 @@ import { officialGepa, officialSkillOpt, } from '@tangle-network/agent-runtime' +import { + createOptimizationActivationReceipt, + optimizationActivationReceiptFromMetadata, +} from '@tangle-network/agent-runtime/intelligence' const python = process.env.AGENT_EVAL_TEST_PYTHON?.trim() if (!python) throw new Error('AGENT_EVAL_TEST_PYTHON is required') @@ -62,14 +66,52 @@ async function runWheelVerification() { assert(firstGepa.provenance?.resumed === false, 'first GEPA run must be fresh') assert(firstGepa.provenance?.source.version === '0.1.4', 'GEPA version was not observed') assert( - firstGepa.provenance?.bridge?.version === '0.126.5', + firstGepa.provenance?.bridge?.version === '0.126.6', 'agent-eval-rpc version was not observed', ) + assert( + firstGepa.provenance?.optimizerModel === 'local-model', + 'GEPA optimizer model was not observed', + ) assert(firstGepa.decision === 'ship', 'GEPA candidate was not promoted') assert( JSON.parse(String(firstGepa.candidate.value)).k === 2, 'GEPA did not produce the expected candidate', ) + const firstGepaModelRequests = gepaModel.requests.length + const firstGepaOptimizationCostUsd = firstGepa.raw.optimizationCost.totalCostUsd + assert(firstGepaModelRequests > 0, 'fresh GEPA run did not call the optimizer model') + assert( + firstGepa.raw.optimizationCost.accountingComplete, + 'fresh GEPA run has incomplete optimizer cost accounting', + ) + assert( + approximatelyEqual( + firstGepaOptimizationCostUsd, + localModelCostUsd(firstGepaModelRequests), + ), + `fresh GEPA cost ${firstGepaOptimizationCostUsd} did not match ${firstGepaModelRequests} optimizer requests`, + ) + const activationReceipt = createOptimizationActivationReceipt(firstGepa) + assert(activationReceipt, 'GEPA result did not produce an activation receipt') + assert( + activationReceipt.models?.optimizer === 'local-model', + 'activation receipt omitted the optimizer model', + ) + assert( + approximatelyEqual( + activationReceipt.cost.optimization.totalUsd, + firstGepaOptimizationCostUsd, + ), + 'activation receipt changed optimizer cost', + ) + const parsedReceipt = optimizationActivationReceiptFromMetadata({ + optimizationReceipt: activationReceipt, + }) + assert( + parsedReceipt?.digest === activationReceipt.digest, + 'activation receipt did not survive public parsing', + ) const resumedGepa = await runGepa({ runDir: gepaRunDir, @@ -77,10 +119,31 @@ async function runWheelVerification() { resume: 'required', }) assert(resumedGepa.provenance?.resumed === true, 'second GEPA run did not restore state') + assert( + resumedGepa.provenance?.optimizerModel === 'local-model', + 'resumed GEPA run omitted the optimizer model', + ) assert( resumedGepa.provenance?.compatibleRunId === firstGepa.provenance?.compatibleRunId, 'resumed GEPA run changed compatible identity', ) + const resumedGepaModelRequests = gepaModel.requests.length + const resumedGepaOptimizationCostUsd = resumedGepa.raw.optimizationCost.totalCostUsd + assert( + resumedGepa.raw.optimizationCost.accountingComplete, + 'resumed GEPA run has incomplete optimizer cost accounting', + ) + assert( + resumedGepaOptimizationCostUsd >= firstGepaOptimizationCostUsd, + `resumed GEPA cost reset from ${firstGepaOptimizationCostUsd} to ${resumedGepaOptimizationCostUsd}`, + ) + assert( + approximatelyEqual( + resumedGepaOptimizationCostUsd, + localModelCostUsd(resumedGepaModelRequests), + ), + `resumed GEPA cost ${resumedGepaOptimizationCostUsd} did not preserve all ${resumedGepaModelRequests} optimizer requests`, + ) let incompatibleError try { @@ -148,8 +211,14 @@ async function runWheelVerification() { runtimeImport: '@tangle-network/agent-runtime', gepa: { bridge: firstGepa.provenance.bridge.version, + optimizerModel: firstGepa.provenance.optimizerModel, + receipt: activationReceipt.digest, evaluations: firstGepa.provenance.evaluationCount, resumed: resumedGepa.provenance.resumed, + optimizationCostUsd: { + fresh: firstGepaOptimizationCostUsd, + resumed: resumedGepaOptimizationCostUsd, + }, source: firstGepa.provenance.source.version, concurrent: { completed: completedConcurrent.length, @@ -221,7 +290,7 @@ async function runOmniVerification() { 'source GEPA revision was not observed', ) assert( - result.provenance?.bridge?.version === '0.126.5', + result.provenance?.bridge?.version === '0.126.6', 'Omni agent-eval-rpc version was not observed', ) assert(result.decision === 'ship', 'Omni candidate was not promoted') @@ -435,6 +504,15 @@ function digest(value) { return `sha256:${hex}` } +function localModelCostUsd(requestCount) { + return requestCount * ((11 * 1) / 1_000_000 + (13 * 2) / 1_000_000) +} + +function approximatelyEqual(left, right) { + const tolerance = Number.EPSILON * Math.max(1, Math.abs(left), Math.abs(right)) * 8 + return Math.abs(left - right) <= tolerance +} + function assert(condition, message) { if (!condition) throw new Error(message) } diff --git a/scripts/verify-official-optimizers.mjs b/scripts/verify-official-optimizers.mjs index c260400b..d0f55d59 100644 --- a/scripts/verify-official-optimizers.mjs +++ b/scripts/verify-official-optimizers.mjs @@ -12,8 +12,8 @@ import { tmpdir } from 'node:os' import { dirname, join, resolve } from 'node:path' import { fileURLToPath } from 'node:url' -const agentEvalVersion = '0.126.5' -const agentKnowledgeVersion = '5.0.0' +const agentEvalVersion = '0.126.6' +const agentKnowledgeVersion = '5.0.1' const gepaVersion = '0.1.4' const gepaSourceRevision = 'f919db0a622e2e9f9204779b81fe00cc1b2d808f' const skillOptRevision = '61735e3922efc2b90c6d6cab561e62e98452ca90' diff --git a/src/improvement/improve-types.ts b/src/improvement/improve-types.ts index 8175fa17..c1ae0bf5 100644 --- a/src/improvement/improve-types.ts +++ b/src/improvement/improve-types.ts @@ -69,6 +69,17 @@ export type ImproveProfileAgent = ( >[2], ) => Promise +/** Exact materialized profile presented for validation before any candidate run. */ +export interface ImproveCandidateValidationInput { + profile: ReadonlyAgentProfile + surface: ImproveProfileSurface + candidateSurface: MutableSurface + value: unknown + isBaseline: boolean +} + +export type ImproveCandidateValidator = (input: ImproveCandidateValidationInput) => void + export type ImproveOptimizationRunOptions = Omit< NonNullable['optimizationRunOptions']>, 'dispatchRef' @@ -95,6 +106,8 @@ export type ImproveMethodOptions = Omit< method: ImproveMethodSource /** Runs the exact complete profile materialized from one candidate surface. */ agent: ImproveProfileAgent + /** Reject a materialized profile before it reaches the agent callback. */ + validateCandidate?: ImproveCandidateValidator /** Trace or analyst findings available to a method factory. */ findings?: readonly unknown[] /** Select the exact inline skill document for `surface: 'skills'`. */ diff --git a/src/improvement/improve.ts b/src/improvement/improve.ts index 78f34c0d..4f7cf00d 100644 --- a/src/improvement/improve.ts +++ b/src/improvement/improve.ts @@ -24,6 +24,8 @@ import type { import { runMethodImprovement } from './method-execution' export type { + ImproveCandidateValidationInput, + ImproveCandidateValidator, ImproveCodeOptions, ImproveCodeResult, ImproveCodeRunOptions, diff --git a/src/improvement/index.ts b/src/improvement/index.ts index 303e501f..74e7d785 100644 --- a/src/improvement/index.ts +++ b/src/improvement/index.ts @@ -30,6 +30,8 @@ export { toAnalystFindings, } from './findings' export { + type ImproveCandidateValidationInput, + type ImproveCandidateValidator, type ImproveCodeOptions, type ImproveCodeResult, type ImproveCodeRunOptions, @@ -59,6 +61,7 @@ export { type OfficialGepaOptions, type OfficialOptimizerContextOptions, OfficialOptimizerUnavailableError, + type OfficialSensitiveCandidateInput, type OfficialSkillOptOptions, officialGepa, officialSkillOpt, diff --git a/src/improvement/method-controls.ts b/src/improvement/method-controls.ts new file mode 100644 index 00000000..fa1fd711 --- /dev/null +++ b/src/improvement/method-controls.ts @@ -0,0 +1,32 @@ +import type { OptimizationMethod, Scenario } from '@tangle-network/agent-eval/campaign' +import type { ImproveCandidateValidationInput } from './improve-types' + +const methodRuntimeControls = Symbol('agent-runtime.improvement.method-runtime-controls') + +export interface MethodRuntimeControls { + costAttribution: 'optimizer-run' + validateCandidate: (input: ImproveCandidateValidationInput) => void +} + +type ControlledOptimizationMethod = OptimizationMethod< + TScenario, + TArtifact +> & { + [methodRuntimeControls]?: MethodRuntimeControls +} + +export function withMethodRuntimeControls( + method: OptimizationMethod, + controls: MethodRuntimeControls, +): OptimizationMethod { + return Object.freeze({ + ...method, + [methodRuntimeControls]: Object.freeze({ ...controls }), + }) +} + +export function methodRuntimeControlsOf( + method: OptimizationMethod, +): MethodRuntimeControls | undefined { + return (method as ControlledOptimizationMethod)[methodRuntimeControls] +} diff --git a/src/improvement/method-cost.test.ts b/src/improvement/method-cost.test.ts index 4ebcc977..66d23561 100644 --- a/src/improvement/method-cost.test.ts +++ b/src/improvement/method-cost.test.ts @@ -2,23 +2,38 @@ import { CostLedger } from '@tangle-network/agent-eval' import type { OptimizationMethodInput, Scenario } from '@tangle-network/agent-eval/campaign' import { canonicalCandidateDigest } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' -import { assertMethodCostRecorded, methodInputWithScopedCost } from './method-cost' +import { + assertMethodCostRecorded, + type MethodCostAttribution, + methodInputWithScopedCost, + methodInvocationCostLedger, +} from './method-cost' const evaluationRef = canonicalCandidateDigest({ fixture: 'shared-cost-ledger' }) -function scopedInput(ledger: CostLedger, invocationId: string) { +function scopedInput( + ledger: CostLedger, + invocationId: string, + costAttribution: MethodCostAttribution = 'invocation', +) { return methodInputWithScopedCost( { costLedger: ledger } as unknown as OptimizationMethodInput, { evaluationRef, invocationId }, + costAttribution, ) } -async function recordCost(input: ReturnType, cost: number): Promise { +async function recordCost( + input: ReturnType, + cost: number, + optimizerRun?: string, +): Promise { const paid = await input.costLedger.runPaidCall({ channel: 'driver', phase: 'optimizer', actor: 'optimizer-model', model: 'fixture', + ...(optimizerRun ? { tags: { optimizerRun } } : {}), maximumCharge: { externallyEnforcedMaximumUsd: cost }, execute: async () => 'candidate', receipt: () => ({ @@ -53,7 +68,10 @@ describe('optimizer cost reconciliation', () => { expect(() => assertMethodCostRecorded( 'fixture', - { totalCostUsd: 0.25, accountingComplete: true, incompleteReasons: [] }, + { + cost: { totalCostUsd: 0.25, accountingComplete: true, incompleteReasons: [] }, + }, + ledger, ledger, 1, ), @@ -66,7 +84,10 @@ describe('optimizer cost reconciliation', () => { expect(() => assertMethodCostRecorded( 'fixture', - { totalCostUsd: 0.25, accountingComplete: true, incompleteReasons: [] }, + { + cost: { totalCostUsd: 0.25, accountingComplete: true, incompleteReasons: [] }, + }, + ledger, ledger, 1, ), @@ -80,11 +101,14 @@ describe('optimizer cost reconciliation', () => { assertMethodCostRecorded( 'fixture', { - totalCostUsd: 0, - accountingComplete: false, - incompleteReasons: ['provider receipt unavailable'], + cost: { + totalCostUsd: 0, + accountingComplete: false, + incompleteReasons: ['provider receipt unavailable'], + }, }, ledger, + ledger, 1, ), ).toThrow(/incomplete cost accounting under costCeiling; refusing final scoring/) @@ -108,4 +132,98 @@ describe('optimizer cost reconciliation', () => { expect(ledger.summary().totalCalls).toBe(2) expect(ledger.summary().totalCostUsd).toBeCloseTo(0.3) }) + + it('does not treat arbitrary method provenance as an official optimizer run', async () => { + const ledger = new CostLedger({ costCeilingUsd: 1 }) + const scope = { evaluationRef, invocationId: 'third-party-invocation' } + const input = methodInputWithScopedCost( + { costLedger: ledger } as unknown as OptimizationMethodInput, + scope, + ) + await recordCost(input, 0.1) + + expect(() => + assertMethodCostRecorded( + 'third-party', + { + cost: { totalCostUsd: 0.1, accountingComplete: true, incompleteReasons: [] }, + provenance: { + source: { + kind: 'package', + evidence: 'observed', + package: 'third-party-optimizer', + version: '1.0.0', + }, + runId: 'third-party-run-without-optimizer-tag', + resumed: false, + evaluationCount: 1, + artifactDir: '/tmp/third-party', + }, + }, + input.costLedger, + methodInvocationCostLedger(ledger, scope), + 1, + ), + ).not.toThrow() + }) + + it('retains compatible-run spend across resumed invocations', async () => { + const ledger = new CostLedger({ costCeilingUsd: 1 }) + const runId = 'compatible-upstream-run' + const first = scopedInput(ledger, 'invocation-first', 'optimizer-run') + await recordCost(first, 0.1, runId) + + const secondScope = { evaluationRef, invocationId: 'invocation-resumed' } + const resumed = methodInputWithScopedCost( + { costLedger: ledger } as unknown as OptimizationMethodInput, + secondScope, + 'optimizer-run', + ) + const resumedInvocation = methodInvocationCostLedger(ledger, secondScope) + + expect(resumed.costLedger.summary()).toMatchObject({ totalCalls: 0, totalCostUsd: 0 }) + expect( + resumed.costLedger.summary({ + phase: 'optimizer', + tags: { optimizerRun: runId }, + }), + ).toMatchObject({ totalCalls: 1, totalCostUsd: 0.1 }) + + await recordCost(resumed, 0.2, runId) + expect(resumed.costLedger.summary({ phase: 'optimizer' })).toMatchObject({ + totalCalls: 1, + totalCostUsd: 0.2, + }) + const compatibleSummary = resumed.costLedger.summary({ + phase: 'optimizer', + tags: { optimizerRun: runId }, + }) + expect(compatibleSummary.totalCalls).toBe(2) + expect(compatibleSummary.totalCostUsd).toBeCloseTo(0.3) + expect(() => + assertMethodCostRecorded( + 'fixture', + { + cost: { totalCostUsd: 0.3, accountingComplete: true, incompleteReasons: [] }, + provenance: { + source: { + kind: 'package', + evidence: 'observed', + package: 'fixture', + version: '1.0.0', + }, + runId, + resumed: true, + evaluationCount: 1, + artifactDir: '/tmp/fixture', + }, + }, + resumed.costLedger, + resumedInvocation, + 1, + 'optimizer-run', + ), + ).not.toThrow() + expect(resumedInvocation.summary()).toMatchObject({ totalCalls: 1, totalCostUsd: 0.2 }) + }) }) diff --git a/src/improvement/method-cost.ts b/src/improvement/method-cost.ts index 6a3629d2..fe9e1e6b 100644 --- a/src/improvement/method-cost.ts +++ b/src/improvement/method-cost.ts @@ -1,7 +1,7 @@ import type { - ComparisonCost, CostLedgerHandle, OptimizationMethodInput, + OptimizationMethodResult, Scenario, } from '@tangle-network/agent-eval/campaign' import type { Sha256Digest } from '@tangle-network/agent-interface' @@ -18,23 +18,39 @@ interface MethodCostScope { invocationId: string } +export type MethodCostAttribution = 'invocation' | 'optimizer-run' + export function methodInputWithScopedCost( input: OptimizationMethodInput, scope: MethodCostScope, + costAttribution: MethodCostAttribution = 'invocation', ): OptimizationMethodInput { return Object.freeze({ ...input, - costLedger: scopedCostLedger(input.costLedger, scope), + costLedger: scopedCostLedger(input.costLedger, scope, costAttribution), }) } +export function methodInvocationCostLedger( + ledger: CostLedgerHandle, + scope: MethodCostScope, +): CostLedgerHandle { + return scopedCostLedger(ledger, scope, 'invocation') +} + export function assertMethodCostRecorded( methodName: string, - cost: ComparisonCost, - ledger: CostLedgerHandle, + result: Pick, + compatibleLedger: CostLedgerHandle, + invocationLedger: CostLedgerHandle, costCeiling: number | undefined, + costAttribution: MethodCostAttribution = 'invocation', ): void { - const observed = ledger.summary() + const { cost } = result + const observed = + costAttribution === 'optimizer-run' && result.provenance?.runId + ? compatibleLedger.summary({ tags: { optimizerRun: result.provenance.runId } }) + : invocationLedger.summary() if (costCeiling !== undefined && !cost.accountingComplete) { throw new ConfigError( `improve(): method '${methodName}' returned incomplete cost accounting under costCeiling; refusing final scoring`, @@ -57,32 +73,50 @@ export function assertMethodCostRecorded( } } -function scopedCostLedger(parent: CostLedgerHandle, scope: MethodCostScope): CostLedgerHandle { - const tags = { +function scopedCostLedger( + parent: CostLedgerHandle, + scope: MethodCostScope, + costAttribution: MethodCostAttribution, +): CostLedgerHandle { + const evaluationTags = { [EVALUATION_TAG]: scope.evaluationRef, + } + const invocationTags = { + ...evaluationTags, [INVOCATION_TAG]: scope.invocationId, } + const readTags = (filter: LedgerFilter): Record => + costAttribution === 'optimizer-run' && hasOptimizerRunFilter(filter) + ? evaluationTags + : invocationTags return { costCeilingUsd: parent.costCeilingUsd, runPaidCall: (input) => parent.runPaidCall({ ...input, - tags: { ...(input.tags ?? {}), ...tags }, + tags: { ...(input.tags ?? {}), ...invocationTags }, }), reconcile: (...args) => parent.reconcile(...args), - list: (filter) => parent.list(withTags(filter, tags)), - listPending: (filter) => parent.listPending?.(withTags(filter, tags)) ?? [], - summary: (filter) => parent.summary(withTags(filter, tags)), + list: (filter) => parent.list(withTags(filter, readTags(filter))), + listPending: (filter) => parent.listPending?.(withTags(filter, readTags(filter))) ?? [], + summary: (filter) => parent.summary(withTags(filter, readTags(filter))), waitForIdle: (options: LedgerWaitOptions = {}) => parent.waitForIdle?.({ ...options, - filter: withTags(options.filter, tags), - }) ?? Promise.resolve(parent.summary(withTags(options.filter, tags)).pendingCalls === 0), + filter: withTags(options.filter, readTags(options.filter)), + }) ?? + Promise.resolve( + parent.summary(withTags(options.filter, readTags(options.filter))).pendingCalls === 0, + ), markCompleted: (count) => parent.markCompleted(count), costPerCompletedTask: () => parent.costPerCompletedTask(), } } +function hasOptimizerRunFilter(filter: LedgerFilter): boolean { + return typeof filter?.tags?.optimizerRun === 'string' && filter.tags.optimizerRun.length > 0 +} + function withTags(filter: LedgerFilter, tags: Record): LedgerFilter { return { ...(filter ?? {}), diff --git a/src/improvement/method-execution.ts b/src/improvement/method-execution.ts index a80c4826..dda98cd1 100644 --- a/src/improvement/method-execution.ts +++ b/src/improvement/method-execution.ts @@ -14,13 +14,19 @@ import { canonicalCandidateDigest, immutableCandidateValue } from '../candidate- import { ConfigError } from '../errors' import { copyImproveCost } from './improve-result' import type { + ImproveCandidateValidationInput, ImproveMethodContext, ImproveMethodOptions, ImproveMethodResult, ImproveMethodSource, ImprovementProfileCandidate, } from './improve-types' -import { assertMethodCostRecorded, methodInputWithScopedCost } from './method-cost' +import { methodRuntimeControlsOf } from './method-controls' +import { + assertMethodCostRecorded, + methodInputWithScopedCost, + methodInvocationCostLedger, +} from './method-cost' import { buildMethodEvaluationIdentity } from './method-identity' import { assertCandidateSurfaceKind, @@ -71,6 +77,7 @@ export async function runMethodImprovement() + const materializeProfile = ( + candidateSurface: Parameters[0], + ): ReturnType => { + const candidate = rawMaterializeProfile(candidateSurface) + const candidateDigest = canonicalCandidateDigest(candidateSurface) + if (!validatedCandidates.has(candidateDigest)) { + const prepared = prepareProfileSurface(candidate, surface, skills, profileComponents) + const validationInput: ImproveCandidateValidationInput = Object.freeze({ + profile: candidate, + surface, + candidateSurface: immutableCandidateValue(candidateSurface), + value: immutableCandidateValue(prepared.value), + isBaseline: candidateDigest === baselineSurfaceDigest, + }) + methodControls?.validateCandidate(validationInput) + validateCandidate?.(validationInput) + validatedCandidates.add(candidateDigest) + } + return candidate + } + materializeProfile(baselineSurface) const measuredMethod: OptimizationMethod = { ...method, async optimize(input) { - const scopedInput = methodInputWithScopedCost(input, { + const costScope = { evaluationRef, invocationId: runtimeInvocationId, - }) + } + const scopedInput = methodInputWithScopedCost( + input, + costScope, + methodControls?.costAttribution, + ) + const invocationLedger = methodInvocationCostLedger(input.costLedger, costScope) const result = await method.optimize(scopedInput) assertMethodCostRecorded( method.name, - result.cost, + result, scopedInput.costLedger, + invocationLedger, comparisonOptions.costCeiling, + methodControls?.costAttribution, ) materializeProfile(result.winnerSurface) return result diff --git a/src/improvement/method-identity.test.ts b/src/improvement/method-identity.test.ts index bfc43372..f0ac7fd4 100644 --- a/src/improvement/method-identity.test.ts +++ b/src/improvement/method-identity.test.ts @@ -62,6 +62,7 @@ describe('method evaluation identity', () => { findings?: readonly unknown[] trainScenarios?: IdentityScenario[] judges?: JudgeConfig[] + validateCandidate?: () => void reps?: number optimizationRunOptions?: { reps?: number } }) => @@ -93,6 +94,7 @@ describe('method evaluation identity', () => { ).not.toBe(baseline) expect(evaluationRef({ optimizationRunOptions: { reps: 2 } })).not.toBe(baseline) expect(evaluationRef({ reps: 2 })).not.toBe(baseline) + expect(evaluationRef({ validateCandidate: () => undefined })).not.toBe(baseline) expect(evaluationRef({ findings: [{ issue: 'different failure' }] })).not.toBe(baseline) expect(evaluationRef({})).toBe(baseline) }) diff --git a/src/improvement/method-identity.ts b/src/improvement/method-identity.ts index ae52ed1e..ea9b8cd2 100644 --- a/src/improvement/method-identity.ts +++ b/src/improvement/method-identity.ts @@ -7,6 +7,7 @@ import { import type { Sha256Digest } from '@tangle-network/agent-interface' import { canonicalCandidateDigest } from '../candidate-execution/digest' import type { + ImproveCandidateValidator, ImproveOptimizationRunOptions, ImproveProfileSurface, ImproveSkillsOptions, @@ -32,6 +33,7 @@ export function buildMethodEvaluationIdentity { expect(result.candidate.profile?.prompt?.systemPrompt).toBe('improved prompt') }) + it('changes upstream identity when feedback transformation logic changes', async () => { + const describeArtifactA = (artifact: Artifact) => ({ answer: artifact.text }) + const describeArtifactB = (artifact: Artifact) => ({ output: artifact.text }) + const rawRedactor = (value: unknown) => value + const observe = async (options: { + describeArtifact: (artifact: Artifact) => unknown + redact: typeof rawRedactor | false + }) => { + const root = runDir() + const observedInputPath = join(root, 'input.json') + await improve(profile, { + ...commonOptions( + officialGepa({ + objective: 'Improve the agent prompt.', + recipe: { + kind: 'engine', + run: { engine: 'gepa', maxEvaluations: 1, maxProposerCostUsd: 1 }, + }, + optimizer: testOptimizer, + describeArtifact: options.describeArtifact, + redact: options.redact, + runner: fakeRunner('gepa', observedInputPath), + }), + ), + runDir: join(root, 'run'), + }) + const observed = JSON.parse(readFileSync(observedInputPath, 'utf8')) as { + evaluationId: string + } + return observed.evaluationId + } + + const baseline = await observe({ describeArtifact: describeArtifactA, redact: false }) + expect(await observe({ describeArtifact: describeArtifactB, redact: false })).not.toBe(baseline) + expect(await observe({ describeArtifact: describeArtifactA, redact: rawRedactor })).not.toBe( + baseline, + ) + }) + + it('changes saved-work identity when built-in redaction behavior changes', () => { + const first = optimizerRedactionPolicyRef(undefined, { + maxDepth: 32, + patterns: ['credential'], + }) + const second = optimizerRedactionPolicyRef(undefined, { + maxDepth: 64, + patterns: ['credential'], + }) + + expect(second).not.toBe(first) + }) + it('passes the official Omni recipe through unchanged', async () => { const root = runDir() const observedInputPath = join(root, 'observed-omni.json') @@ -575,34 +624,116 @@ describe('official optimizer methods', () => { }) }) - it('allows a reviewed sensitive profile surface only with explicit approval', () => { - const mcp = { + it('rejects an unauthorized executable candidate before agent dispatch', async () => { + const root = runDir() + const candidate = JSON.stringify({ + local: { + transport: 'stdio' as const, + command: 'echo', + args: ['unauthorized'], + }, + }) + let agentCalls = 0 + const reviewed: Array<{ + isBaseline: boolean + command: string | undefined + sensitivePaths: readonly string[] + }> = [] + + await expect( + improve( + { name: 'optimizer-fixture', mcp: {} }, + { + ...commonOptions( + officialGepa({ + objective: 'Improve the MCP profile.', + recipe: { + kind: 'engine', + run: { engine: 'gepa', maxEvaluations: 1, maxProposerCostUsd: 1 }, + }, + optimizer: testOptimizer, + authorizeSensitiveCandidate: (input) => { + reviewed.push({ + isBaseline: input.isBaseline, + command: input.profile.mcp?.local?.command, + sensitivePaths: input.sensitivePaths, + }) + return input.isBaseline + }, + runner: fakeRunner('gepa', join(root, 'input.json'), { + exampleId: 'train', + responsePath: join(root, 'response.json'), + candidate, + }), + }), + ), + runDir: join(root, 'run'), + surface: 'mcp', + agent: async () => { + agentCalls += 1 + return { text: 'must not run' } + }, + }, + ), + ).rejects.toThrow(/callback failed: 500/) + expect(agentCalls).toBe(0) + expect(reviewed).toEqual([ + { isBaseline: true, command: undefined, sensitivePaths: ['$'] }, + { isBaseline: false, command: 'echo', sensitivePaths: ['$'] }, + ]) + }) + + it('authorizes every exact sensitive candidate through a callback', async () => { + const root = runDir() + const candidate = JSON.stringify({ remote: { transport: 'http' as const, url: 'https://public.example.test/mcp', }, - } - const method = officialGepa({ - objective: 'Improve the MCP profile.', - recipe: { - kind: 'engine', - run: { engine: 'gepa', maxEvaluations: 1, maxProposerCostUsd: 1 }, - }, - optimizer: testOptimizer, - approveSensitiveProfileSurface: true, - runner: failingRunner('runner must not start'), }) - - expect(() => - method({ - profile: { name: 'optimizer-fixture', mcp }, - evaluationRef, + const reviewed: ReadonlyAgentProfile[] = [] + const result = await improve( + { name: 'optimizer-fixture', mcp: {} }, + { + ...commonOptions( + officialGepa({ + objective: 'Improve the MCP profile.', + recipe: { + kind: 'engine', + run: { engine: 'gepa', maxEvaluations: 1, maxProposerCostUsd: 1 }, + }, + optimizer: testOptimizer, + authorizeSensitiveCandidate: (input) => { + reviewed.push(input.profile) + if (input.isBaseline) { + return ( + input.surface === 'mcp' && + input.sensitivePaths.includes('$') && + Object.keys(input.profile.mcp ?? {}).length === 0 + ) + } + return ( + input.surface === 'mcp' && + input.sensitivePaths.includes('$') && + input.sensitivePaths.includes('$.remote.url') && + input.profile.mcp?.remote?.url === 'https://public.example.test/mcp' + ) + }, + runner: fakeRunner('gepa', join(root, 'input.json'), { + exampleId: 'train', + responsePath: join(root, 'response.json'), + candidate, + }), + }), + ), + runDir: join(root, 'run'), surface: 'mcp', - baselineSurface: JSON.stringify(mcp), - baselineValue: mcp, - findings: [], - }), - ).not.toThrow() + }, + ) + + expect(reviewed).toHaveLength(2) + expect(reviewed.every(Object.isFrozen)).toBe(true) + expect(result.candidate.profile.mcp?.remote?.url).toBe('https://public.example.test/mcp') }) it.each([ diff --git a/src/improvement/official-optimizers.ts b/src/improvement/official-optimizers.ts index 5ee103c0..51a11041 100644 --- a/src/improvement/official-optimizers.ts +++ b/src/improvement/official-optimizers.ts @@ -10,12 +10,22 @@ import { } from '@tangle-network/agent-eval/campaign' import { canonicalCandidateDigest } from '../candidate-execution/digest' import { ConfigError } from '../errors' -import { defaultRedactor, type Redactor, resolveRedactor } from '../redact' -import type { ImproveMethodContext, ImproveMethodFactory } from './improve' +import { + defaultRedactor, + defaultRedactorIdentityMaterial, + type Redactor, + resolveRedactor, +} from '../redact' +import type { + ImproveCandidateValidationInput, + ImproveMethodContext, + ImproveMethodFactory, +} from './improve' +import { withMethodRuntimeControls } from './method-controls' const defaultMaxFindingsChars = 50_000 const pythonClientDocs = 'https://github.com/tangle-network/agent-eval/tree/main/clients/python' -const bridgeInstall = '`python -m pip install "agent-eval-rpc==0.126.5"`' +const bridgeInstall = '`python -m pip install "agent-eval-rpc==0.126.6"`' const gepaWheelInstall = '`python -m pip install "gepa[full]==0.1.4"`' const gepaSourceInstall = '`python -m pip install "gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f"`' @@ -37,13 +47,13 @@ export interface OfficialOptimizerContextOptions { * that has already been reviewed. */ redact?: Redactor | false - /** - * Confirm that structurally sensitive candidate fields contain only safe - * references or public values. Required for fields such as MCP env, headers, - * URLs, metadata, and extensions because candidate bytes cannot be redacted - * without changing what the optimizer measures. - */ - approveSensitiveProfileSurface?: boolean + /** Authorize one exact candidate containing structurally sensitive fields. + * The callback must return true for every accepted baseline and candidate. */ + authorizeSensitiveCandidate?: (input: OfficialSensitiveCandidateInput) => boolean +} + +export interface OfficialSensitiveCandidateInput extends ImproveCandidateValidationInput { + sensitivePaths: readonly string[] } /** Official GEPA configuration plus bounded Runtime findings context. */ @@ -104,7 +114,7 @@ export function officialGepa { - assertSafeOptimizerSurface('officialGepa', context, approveSensitiveProfileSurface) - return withDependencyHelp( + const externalEvaluationRef = optimizerEvidencePolicyRef({ + runtimeEvaluationRef: context.evaluationRef, + redactionPolicyRef, + describeScenario, + describeArtifact, + authorizeSensitiveCandidate, + }) + const method = withDependencyHelp( 'gepa', - context.evaluationRef, + externalEvaluationRef, redactor, redactionPolicyRef, gepaOptimizationMethod({ ...config, objective, - evaluationId: context.evaluationRef, + evaluationId: externalEvaluationRef, background: methodBackground({ context, background, @@ -154,6 +170,11 @@ export function officialGepa + assertSafeOptimizerCandidate('officialGepa', input, authorizeSensitiveCandidate), + }) } } @@ -171,7 +192,7 @@ export function officialSkillOpt< describeScenario, describeArtifact, redact, - approveSensitiveProfileSurface = false, + authorizeSensitiveCandidate, ...config } = options const redactor = resolveRedactor(redact) @@ -179,16 +200,22 @@ export function officialSkillOpt< assertMaxFindingsChars('officialSkillOpt', maxFindingsChars) const objective = redactOptimizerText('officialSkillOpt', 'objective', config.objective, redactor) return (context) => { - assertSafeOptimizerSurface('officialSkillOpt', context, approveSensitiveProfileSurface) - return withDependencyHelp( + const externalEvaluationRef = optimizerEvidencePolicyRef({ + runtimeEvaluationRef: context.evaluationRef, + redactionPolicyRef, + describeScenario, + describeArtifact, + authorizeSensitiveCandidate, + }) + const method = withDependencyHelp( 'skillopt', - context.evaluationRef, + externalEvaluationRef, redactor, redactionPolicyRef, skillOptOptimizationMethod({ ...config, objective, - evaluationId: context.evaluationRef, + evaluationId: externalEvaluationRef, background: methodBackground({ context, background, @@ -221,6 +248,11 @@ export function officialSkillOpt< : {}), }), ) + return withMethodRuntimeControls(method, { + costAttribution: 'optimizer-run', + validateCandidate: (input) => + assertSafeOptimizerCandidate('officialSkillOpt', input, authorizeSensitiveCandidate), + }) } } @@ -279,34 +311,58 @@ function methodBackground(options: { return sections.join('\n\n') } -function assertSafeOptimizerSurface( +function assertSafeOptimizerCandidate( label: string, - context: ImproveMethodContext, - approveSensitiveProfileSurface: boolean, + input: ImproveCandidateValidationInput, + authorizeSensitiveCandidate: ((input: OfficialSensitiveCandidateInput) => boolean) | undefined, ): void { - const redactedValue = defaultRedactor(context.baselineValue) - const redactedSurface = defaultRedactor(context.baselineSurface) + const redactedValue = defaultRedactor(input.value) + const redactedSurface = defaultRedactor(input.candidateSurface) if ( - !isDeepStrictEqual(context.baselineValue, redactedValue) || - !isDeepStrictEqual(context.baselineSurface, redactedSurface) + !isDeepStrictEqual(input.value, redactedValue) || + !isDeepStrictEqual(input.candidateSurface, redactedSurface) ) { throw new ConfigError( `${label}: the selected profile surface contains a common credential or private value. ` + 'Store live credentials as provider references, or remove private data before starting an external optimizer.', ) } - const sensitivePaths = sensitiveProfileSurfacePaths(context.baselineValue) - if (!approveSensitiveProfileSurface && sensitivePaths.length > 0) { + const sensitivePaths = sensitiveProfileSurfacePaths(input) + if (sensitivePaths.length === 0) return + let authorized = false + if (authorizeSensitiveCandidate) { + try { + authorized = + authorizeSensitiveCandidate( + Object.freeze({ + ...input, + sensitivePaths: Object.freeze([...sensitivePaths]), + }), + ) === true + } catch (cause) { + throw new ConfigError(`${label}: sensitive candidate authorization failed`, { cause }) + } + } + if (!authorized) { throw new ConfigError( `${label}: the selected profile surface contains fields that may carry private values: ` + `${sensitivePaths.slice(0, 8).join(', ')}. Remove them, replace values with safe references, ` + - 'or set approveSensitiveProfileSurface: true after reviewing the exact candidate bytes.', + 'or authorize the exact profile with authorizeSensitiveCandidate.', ) } } -function sensitiveProfileSurfacePaths(value: unknown): string[] { - const paths: string[] = [] +function sensitiveProfileSurfacePaths(input: ImproveCandidateValidationInput): string[] { + const paths = new Set() + if ( + input.surface === 'tools' || + input.surface === 'mcp' || + input.surface === 'hooks' || + input.surface === 'subagents' || + input.surface === 'agent-profile' + ) { + paths.add('$') + } const seen = new WeakSet() const visit = (current: unknown, path: string): void => { if (current === null || typeof current !== 'object') return @@ -321,14 +377,14 @@ function sensitiveProfileSurfacePaths(value: unknown): string[] { for (const [key, child] of Object.entries(current as Record)) { const childPath = `${path}.${key}` if (['env', 'headers', 'url', 'metadata', 'extensions'].includes(key.toLowerCase())) { - paths.push(childPath) + paths.add(childPath) continue } visit(child, childPath) } } - visit(value, '$') - return paths + visit(input.value, '$') + return [...paths] } function redactOptimizerEvidence( @@ -365,15 +421,43 @@ function redactJudgeScore(score: JudgeScore, redactor: Redactor): JudgeScore { } } -function optimizerRedactionPolicyRef(redact: Redactor | false | undefined): string { - if (redact === undefined) return 'default-redactor' +export function optimizerRedactionPolicyRef( + redact: Redactor | false | undefined, + builtInIdentity: unknown = defaultRedactorIdentityMaterial(), +): string { if (redact === false) return 'caller-approved-raw' return canonicalCandidateDigest({ - kind: 'caller-redactor', - source: Function.prototype.toString.call(redact), + kind: redact === undefined ? 'default-redactor' : 'caller-redactor-with-default', + builtIn: builtInIdentity, + ...(redact === undefined + ? {} + : { + callerSource: Function.prototype.toString.call(redact), + composition: Function.prototype.toString.call(resolveRedactor), + }), + }) +} + +function optimizerEvidencePolicyRef(input: { + runtimeEvaluationRef: ImproveMethodContext['evaluationRef'] + redactionPolicyRef: string + describeScenario: unknown + describeArtifact: unknown + authorizeSensitiveCandidate: unknown +}): ReturnType { + return canonicalCandidateDigest({ + runtimeEvaluationRef: input.runtimeEvaluationRef, + redactionPolicyRef: input.redactionPolicyRef, + describeScenario: callbackSource(input.describeScenario), + describeArtifact: callbackSource(input.describeArtifact), + authorizeSensitiveCandidate: callbackSource(input.authorizeSensitiveCandidate), }) } +function callbackSource(callback: unknown): string | null { + return typeof callback === 'function' ? Function.prototype.toString.call(callback) : null +} + function withDependencyHelp( optimizer: 'gepa' | 'skillopt', evaluationRef: ImproveMethodContext['evaluationRef'], diff --git a/src/improvement/official-packages.test.ts b/src/improvement/official-packages.test.ts index 03023c80..8fd6e93c 100644 --- a/src/improvement/official-packages.test.ts +++ b/src/improvement/official-packages.test.ts @@ -160,7 +160,7 @@ print(json.dumps({ } expect(inspected.runtime.bridge).toMatchObject({ package: 'agent-eval-rpc', - version: '0.126.5', + version: '0.126.6', sourceSha256: expect.stringMatching(/^[a-f0-9]{64}$/), }) expect(inspected.runtime.optimizer).toMatchObject({ @@ -229,7 +229,7 @@ print(json.dumps({ } expect(inspected.runtime.bridge).toMatchObject({ package: 'agent-eval-rpc', - version: '0.126.5', + version: '0.126.6', sourceSha256: expect.stringMatching(/^[a-f0-9]{64}$/), }) expect(inspected.runtime.optimizer).toMatchObject({ diff --git a/src/improvement/profile-surface.ts b/src/improvement/profile-surface.ts index f9262bc4..13f93114 100644 --- a/src/improvement/profile-surface.ts +++ b/src/improvement/profile-surface.ts @@ -77,12 +77,10 @@ export function prepareProfileSurface( } case 'rollout-policy': { const policy = structuralRolloutPolicyFromProfile(profile) - if (!policy) { - throw new ConfigError( - "improve(): surface 'rollout-policy' requires an existing structural rollout policy", - ) + return { + surface: policy ? serializeRolloutPolicy(policy) : '', + value: policy ?? null, } - return { surface: serializeRolloutPolicy(policy), value: policy } } case 'code': throw new ConfigError( diff --git a/src/intelligence/index.ts b/src/intelligence/index.ts index f0424456..9b7b4527 100644 --- a/src/intelligence/index.ts +++ b/src/intelligence/index.ts @@ -180,8 +180,14 @@ export { buildAgentImprovementActivationTargets, isAgentImprovementProfileSurface, } from './improvement-surfaces' -export type { OptimizationActivationReceipt } from './optimization-receipt' -export { optimizationActivationReceiptFromMetadata } from './optimization-receipt' +export type { + OptimizationActivationReceipt, + OptimizationReceiptCost, +} from './optimization-receipt' +export { + createOptimizationActivationReceipt, + optimizationActivationReceiptFromMetadata, +} from './optimization-receipt' export type { AgentImprovementProfileActivationPreparation, AgentImprovementProfileActivationTarget, diff --git a/src/intelligence/intelligence.test.ts b/src/intelligence/intelligence.test.ts index 9f4f4bfc..3e39ec05 100644 --- a/src/intelligence/intelligence.test.ts +++ b/src/intelligence/intelligence.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { defaultRedactorIdentityMaterial } from '../redact' import type { LoopTraceEvent } from '../runtime/types' import { compileEffort, @@ -131,6 +132,28 @@ describe('resolveEffort', () => { }) describe('defaultRedactor', () => { + it('exposes every built-in behavior input used to identify saved optimizer work', () => { + const identity = defaultRedactorIdentityMaterial() as { + marker: string + maxDepth: number + valuePatterns: unknown[] + secretKeyPattern: { source: string; flags: string } + secretAssignmentPattern: { source: string; flags: string } + scrubString: string + walk: string + defaultRedactor: string + } + + expect(identity.marker).toBe('[redacted]') + expect(identity.maxDepth).toBe(32) + expect(identity.valuePatterns).toHaveLength(6) + expect(identity.secretKeyPattern.source).toContain('api[-_]?key') + expect(identity.secretAssignmentPattern.flags).toContain('g') + expect(identity.scrubString).toContain('secretAssignmentPattern') + expect(identity.walk).toContain('maxDepth') + expect(identity.defaultRedactor).toContain('walk') + }) + it('strips api keys, bearer tokens, emails, and secret-keyed values', () => { const redacted = defaultRedactor({ message: 'contact me at alice@acme.com', diff --git a/src/intelligence/optimization-receipt.ts b/src/intelligence/optimization-receipt.ts index c5f0dd6e..dbe4bc45 100644 --- a/src/intelligence/optimization-receipt.ts +++ b/src/intelligence/optimization-receipt.ts @@ -28,18 +28,18 @@ export interface OptimizationActivationReceipt { bridge?: OptimizationPackageSource modules?: OptimizationProvenance['modules'] python?: OptimizationProvenance['python'] - model?: { - role: 'candidate' - identity: NonNullable + models?: { + candidate?: NonNullable + optimizer?: string } usage: { - evaluations: number - tokens?: OptimizationTokenUsage + optimizerEvaluations: number + optimizerTokens?: OptimizationTokenUsage } cost: { - totalUsd: number - accountingComplete: boolean - incompleteReasons: string[] + optimization: OptimizationReceiptCost + finalTest: OptimizationReceiptCost + total: OptimizationReceiptCost } invocation: { runtimeInvocationId: string @@ -52,6 +52,12 @@ export interface OptimizationActivationReceipt { digest: Sha256Digest } +export interface OptimizationReceiptCost { + totalUsd: number + accountingComplete: boolean + incompleteReasons: string[] +} + export const optimizationReceiptMetadataKey = 'optimizationReceipt' /** Build a detached receipt only for methods backed by an identified external optimizer. */ @@ -60,6 +66,8 @@ export function createOptimizationActivationReceipt( ): OptimizationActivationReceipt | undefined { const provenance = improvement.provenance if (!provenance) return undefined + const optimizerModel = provenance.optimizerModel + const candidateModel = improvement.candidate.profile.model return canonicalCandidateDocument({ kind: 'optimization-activation-receipt', @@ -68,22 +76,22 @@ export function createOptimizationActivationReceipt( ...(provenance.bridge ? { bridge: provenance.bridge } : {}), ...(provenance.modules ? { modules: provenance.modules } : {}), ...(provenance.python ? { python: provenance.python } : {}), - ...(improvement.candidate.profile.model + ...(candidateModel || optimizerModel ? { - model: { - role: 'candidate', - identity: improvement.candidate.profile.model, + models: { + ...(candidateModel ? { candidate: candidateModel } : {}), + ...(optimizerModel ? { optimizer: optimizerModel } : {}), }, } : {}), usage: { - evaluations: provenance.evaluationCount, - ...(provenance.tokenUsage ? { tokens: provenance.tokenUsage } : {}), + optimizerEvaluations: provenance.evaluationCount, + ...(provenance.tokenUsage ? { optimizerTokens: provenance.tokenUsage } : {}), }, cost: { - totalUsd: improvement.cost.totalCostUsd, - accountingComplete: improvement.cost.accountingComplete, - incompleteReasons: improvement.cost.incompleteReasons, + optimization: receiptCost(improvement.raw.optimizationCost), + finalTest: receiptCost(improvement.raw.testCost), + total: receiptCost(improvement.raw.totalCost), }, invocation: { runtimeInvocationId: improvement.lineage.invocationId, @@ -139,7 +147,7 @@ function parseOptimizationActivationReceipt( (value.bridge !== undefined && !isPackageSource(value.bridge)) || !isModules(value.modules) || !isPythonRuntime(value.python) || - !isCandidateModel(value.model) || + !isModels(value.models) || !isUsage(value.usage) || !isCost(value.cost) || !isInvocation(value.invocation) || @@ -194,19 +202,21 @@ function isPythonRuntime(value: unknown): boolean { ) } -function isCandidateModel(value: unknown): boolean { +function isModels(value: unknown): boolean { return ( value === undefined || (isRecord(value) && - value.role === 'candidate' && - agentProfileModelHintsSchema.safeParse(value.identity).success) + (value.candidate === undefined || + agentProfileModelHintsSchema.safeParse(value.candidate).success) && + isOptionalNonEmptyString(value.optimizer) && + (value.candidate !== undefined || value.optimizer !== undefined)) ) } function isUsage(value: unknown): boolean { - if (!isRecord(value) || !isNonNegativeInteger(value.evaluations)) return false - if (value.tokens === undefined) return true - const tokens = value.tokens + if (!isRecord(value) || !isNonNegativeInteger(value.optimizerEvaluations)) return false + if (value.optimizerTokens === undefined) return true + const tokens = value.optimizerTokens if (!isRecord(tokens)) return false const inputTokens = tokens.inputTokens const outputTokens = tokens.outputTokens @@ -241,6 +251,24 @@ function isUsage(value: unknown): boolean { } function isCost(value: unknown): boolean { + if ( + !isRecord(value) || + !isCostPart(value.optimization) || + !isCostPart(value.finalTest) || + !isCostPart(value.total) + ) { + return false + } + const optimization = value.optimization as unknown as OptimizationReceiptCost + const finalTest = value.finalTest as unknown as OptimizationReceiptCost + const total = value.total as unknown as OptimizationReceiptCost + return ( + approximatelyEqual(total.totalUsd, optimization.totalUsd + finalTest.totalUsd) && + total.accountingComplete === (optimization.accountingComplete && finalTest.accountingComplete) + ) +} + +function isCostPart(value: unknown): boolean { return ( isRecord(value) && typeof value.totalUsd === 'number' && @@ -252,6 +280,23 @@ function isCost(value: unknown): boolean { ) } +function receiptCost(value: { + totalCostUsd: number + accountingComplete: boolean + incompleteReasons: string[] +}): OptimizationReceiptCost { + return { + totalUsd: value.totalCostUsd, + accountingComplete: value.accountingComplete, + incompleteReasons: [...value.incompleteReasons], + } +} + +function approximatelyEqual(left: number, right: number): boolean { + const tolerance = Number.EPSILON * Math.max(1, Math.abs(left), Math.abs(right)) * 8 + return Math.abs(left - right) <= tolerance +} + function isInvocation(value: unknown): boolean { return ( isRecord(value) && diff --git a/src/redact.ts b/src/redact.ts index 6942ab53..9cf0ccdc 100644 --- a/src/redact.ts +++ b/src/redact.ts @@ -86,6 +86,24 @@ function walk(value: unknown, seen: WeakSet, depth: number): unknown { return out } +/** Stable identity input for saved work that depends on built-in redaction behavior. */ +export function defaultRedactorIdentityMaterial(): unknown { + const patternIdentity = (pattern: RegExp) => ({ + source: pattern.source, + flags: pattern.flags, + }) + return { + marker: redactedMarker, + secretKeyPattern: patternIdentity(secretKeyPattern), + valuePatterns: valuePatterns.map(patternIdentity), + secretAssignmentPattern: patternIdentity(secretAssignmentPattern), + maxDepth, + scrubString: Function.prototype.toString.call(scrubString), + walk: Function.prototype.toString.call(walk), + defaultRedactor: Function.prototype.toString.call(defaultRedactor), + } +} + /** * Resolve the redactor a client uses. A caller-supplied hook handles * domain-specific values first, then the built-in scrubber still removes diff --git a/tests/knowledge-improvement-job.test.ts b/tests/knowledge-improvement-job.test.ts index 0818705e..44617e7b 100644 --- a/tests/knowledge-improvement-job.test.ts +++ b/tests/knowledge-improvement-job.test.ts @@ -1,9 +1,10 @@ import { mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { dirname, join, relative } from 'node:path' -import type { - AgentImprovementActivationResult, - Sha256Digest, +import { + type AgentImprovementActivationResult, + canonicalCandidateDigest, + type Sha256Digest, } from '@tangle-network/agent-interface' import { addSourceText, @@ -161,6 +162,7 @@ describe('runKnowledgeImprovementJob', () => { const result = await runKnowledgeImprovementJob({ root, goal: 'Add runtime job knowledge', + implementationRef: canonicalCandidateDigest({ fixture: 'runtime-job' }), runId: 'runtime-job', strict: true, budget: { maxIterations: 2, maxTokens: 1000 }, @@ -258,6 +260,9 @@ describe('runKnowledgeImprovementJob', () => { const result = await runKnowledgeImprovementJob({ root, goal: 'Add runtime job knowledge', + implementationRef: canonicalCandidateDigest({ + fixture: 'runtime-job-default-readiness', + }), runId: 'runtime-job-default-readiness', strict: true, readinessSpecs: [readinessSpec], @@ -305,6 +310,7 @@ describe('runKnowledgeImprovementJob', () => { const proposed = await runKnowledgeImprovementJob({ root, goal: 'Add runtime job knowledge', + implementationRef: canonicalCandidateDigest({ fixture: 'runtime-job-approved' }), runId: 'runtime-job-approved', strict: true, budget: { maxIterations: 2, maxTokens: 1000 }, diff --git a/tests/optimization-receipt.test.ts b/tests/optimization-receipt.test.ts index abba21e3..e8d6a832 100644 --- a/tests/optimization-receipt.test.ts +++ b/tests/optimization-receipt.test.ts @@ -45,9 +45,10 @@ const officialImprovementMethod: ImproveMethodFactory { }, bridge: { package: 'agent-eval-rpc', - version: '0.126.5', + version: '0.126.6', sourceSha256: 'b'.repeat(64), }, python: { implementation: 'CPython', version: '3.13.5' }, - model: { - role: 'candidate', - identity: { + models: { + candidate: { provider: 'test-provider', default: 'provider/model', reasoningEffort: 'high', }, + optimizer: 'optimizer-model', }, usage: { - evaluations: 7, - tokens: { inputTokens: 100, outputTokens: 30, totalTokens: 130, calls: 2 }, + optimizerEvaluations: 7, + optimizerTokens: { inputTokens: 100, outputTokens: 30, totalTokens: 130, calls: 2 }, }, cost: { - totalUsd: result.improvement.cost.totalCostUsd, - accountingComplete: true, - incompleteReasons: [], + optimization: { + totalUsd: result.improvement.raw.optimizationCost.totalCostUsd, + accountingComplete: true, + incompleteReasons: [], + }, + finalTest: { + totalUsd: result.improvement.raw.testCost.totalCostUsd, + accountingComplete: true, + incompleteReasons: [], + }, + total: { + totalUsd: result.improvement.cost.totalCostUsd, + accountingComplete: true, + incompleteReasons: [], + }, }, invocation: { runtimeInvocationId: result.improvement.lineage.invocationId, @@ -154,14 +167,23 @@ describe('optimization activation receipt', { timeout: 30_000 }, () => { } as unknown as Parameters[0]), ).toThrow(/invalid evidence/) const invalidUsageReceipt = structuredClone(receipt) - if (!invalidUsageReceipt.usage.tokens) throw new Error('expected optimizer token usage') - invalidUsageReceipt.usage.tokens.totalTokens += 1 + if (!invalidUsageReceipt.usage.optimizerTokens) + throw new Error('expected optimizer token usage') + invalidUsageReceipt.usage.optimizerTokens.totalTokens += 1 invalidUsageReceipt.digest = canonicalCandidateDigest(omitTopLevelDigest(invalidUsageReceipt)) expect(() => optimizationActivationReceiptFromMetadata({ [optimizationReceiptMetadataKey]: invalidUsageReceipt, } as unknown as Parameters[0]), ).toThrow(/invalid evidence/) + const invalidCostReceipt = structuredClone(receipt) + invalidCostReceipt.cost.total.totalUsd += 1 + invalidCostReceipt.digest = canonicalCandidateDigest(omitTopLevelDigest(invalidCostReceipt)) + expect(() => + optimizationActivationReceiptFromMetadata({ + [optimizationReceiptMetadataKey]: invalidCostReceipt, + } as unknown as Parameters[0]), + ).toThrow(/invalid evidence/) const review = reviewAgentImprovementProposal(result.proposal, { decision: 'approve', From 66780e17c958e03ae82ff8d917e0fad50bf3ecbc Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 24 Jul 2026 16:35:24 -0600 Subject: [PATCH 12/14] fix(bench): isolate concurrent optimizer evaluations --- bench/src/swe-arena/gepa-seat.mts | 117 +++++++++--- bench/src/swe-arena/gepa-seat.test.mts | 179 +++++++++++++++++- .../src/swe-arena/implementation-ref.test.mts | 64 +++++++ bench/src/swe-arena/implementation-ref.ts | 62 ++++++ bench/src/swe-arena/outer-loop.mts | 100 +++++++++- bench/src/swe-arena/proposer-fanout.mts | 46 +++-- bench/src/swe-arena/scratch-worktree.test.mts | 55 ++++++ bench/src/swe-arena/scratch-worktree.ts | 34 ++++ 8 files changed, 598 insertions(+), 59 deletions(-) create mode 100644 bench/src/swe-arena/implementation-ref.test.mts create mode 100644 bench/src/swe-arena/implementation-ref.ts create mode 100644 bench/src/swe-arena/scratch-worktree.test.mts create mode 100644 bench/src/swe-arena/scratch-worktree.ts diff --git a/bench/src/swe-arena/gepa-seat.mts b/bench/src/swe-arena/gepa-seat.mts index ed99645b..cc2c41ea 100644 --- a/bench/src/swe-arena/gepa-seat.mts +++ b/bench/src/swe-arena/gepa-seat.mts @@ -6,12 +6,11 @@ * * INNER (what GEPA's own loop calls, many times, budget-capped): the * candidate is ONE change-space file's content as a string. Each inner call - * materializes the candidate string into the seat's scratch loops worktree - * (the rest of the repo stays at the incumbent commit) and runs the EXISTING - * pre-filter smoke cell — one PUBLIC instance, the cheap path — through the - * injected `SmokeRunner`. Score = smoke resolve (1/0) + verify-pass fraction - * as a bounded tiebreak. Inner calls are capped by `maxMetricCalls` - * (default 10; each smoke costs minutes of arm time). + * gets a detached worktree at the incumbent commit, writes its own candidate, + * and runs the existing pre-filter smoke cell on one PUBLIC instance through + * the injected `SmokeRunner`. Score = smoke resolve (1/0) + verify-pass + * fraction as a bounded tiebreak. Inner calls are capped by + * `maxMetricCalls` (default 10; each smoke costs minutes of arm time). * * OUTER: GEPA's best candidate is written back to the surface file in the * scratch worktree and the seat returns `applied: true` — from there the @@ -58,6 +57,12 @@ import { officialOptimizerModel } from '../official-optimizer-config.mts' import { ACTIVATION_PREDICATE_RELPATH, type ActivationPredicate } from './activation.mts' import { changeSpaceViolations, type OuterLoopConfig } from './outer-loop.mts' import type { AuthorFn, ProposerSpec, SmokeRunner, SmokeVerdict } from './proposer-fanout.mts' +import { runOk } from './proc.ts' +import { + createDetachedWorktree, + pruneDetachedWorktrees, + removeDetachedWorktree, +} from './scratch-worktree.ts' import type { ScoreSplit } from './score-split.mts' // --------------------------------------------------------------------------- @@ -590,6 +595,8 @@ export function mechanicalActivationPredicate( export interface GepaSeatDeps { smokeRunner: SmokeRunner + runnerImplementationRef: string + judgeImplementationRef: string /** Resolved PUBLIC smoke instance (outer-loop restricts the choice to the * split's public set; re-asserted here fail-closed). */ smokeInstanceId: string @@ -605,6 +612,40 @@ export interface GepaSeatDeps { const sha256 = (s: string): string => `sha256:${createHash('sha256').update(s).digest('hex')}` +export function gepaSeatEvaluationId(input: { + smokeInstanceId: string + dispatchTimeoutMs: number + incumbentCommit: string + runnerImplementationRef: string + judgeImplementationRef: string +}): string { + if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(input.incumbentCommit)) { + throw new Error('gepa seat: incumbentCommit must be an immutable git object id') + } + const requireImplementationRef = (value: string, label: string): string => { + if (!/^sha256:[a-f0-9]{64}$/.test(value)) { + throw new Error(`gepa seat: ${label} must be an immutable sha256 reference`) + } + return value + } + const runnerRef = requireImplementationRef( + input.runnerImplementationRef, + 'runnerImplementationRef', + ) + const judgeRef = requireImplementationRef( + input.judgeImplementationRef, + 'judgeImplementationRef', + ) + return [ + 'swe-arena-gepa-seat', + `smoke=${input.smokeInstanceId}`, + `incumbent=${input.incumbentCommit}`, + `runner=${runnerRef}`, + `judge=${judgeRef}`, + `dispatchTimeoutMs=${input.dispatchTimeoutMs}`, + ].join('|') +} + /** Build the seat's `AuthorFn`. The fan-out calls it with the seat's scratch * worktree (checked out at the incumbent commit); everything this function * leaves in that worktree becomes the candidate diff. */ @@ -621,8 +662,12 @@ export function gepaSeatAuthor(config: OuterLoopConfig, deps: GepaSeatDeps): Aut const seed = await readFile(surfacePath, 'utf8').catch(() => { throw new Error(`gepa seat '${spec.name}': surface ${spec.surface} does not exist at the incumbent commit`) }) + const incumbentCommit = ( + await runOk('git', ['-C', args.worktreePath, 'rev-parse', 'HEAD']) + ).stdout.trim() const runDir = join(config.outDir, 'gepa-seat', `gen${generation}-${spec.name.replace(/[^a-zA-Z0-9_-]/g, '_')}`) await mkdir(runDir, { recursive: true }) + await pruneDetachedWorktrees(args.worktreePath) const objective = `Improve the supervisor-loop file '${spec.surface}' (returned as the COMPLETE new file content) so the ` + @@ -643,6 +688,7 @@ export function gepaSeatAuthor(config: OuterLoopConfig, deps: GepaSeatDeps): Aut runDir: `${runDir}/cost`, }) const innerScores: GepaInnerCall[] = [] + let dispatchedCalls = 0 const dispatchWithSurface = async ( surface: MutableSurface, scenario: GepaSeatScenario, @@ -651,27 +697,38 @@ export function gepaSeatAuthor(config: OuterLoopConfig, deps: GepaSeatDeps): Aut if (typeof surface !== 'string') { throw new Error(`gepa seat '${spec.name}': candidate surface must be a string`) } - if (innerScores.length >= budget) { + const call = ++dispatchedCalls + if (call > budget) { // Defense-in-depth: the adapter's callback enforces the same cap. throw new Error(`gepa seat '${spec.name}': inner-call budget ${budget} exhausted`) } - await writeFile(surfacePath, surface) - const verdict = await deps.smokeRunner({ - scratchPath: args.worktreePath, - generation, - proposer: spec, - costLedger, - }) + const candidateSha256 = sha256(surface) + const evaluationKey = `inner-${call}-${candidateSha256.slice(7, 19)}` + const candidateWorktree = join(runDir, 'candidate-worktrees', evaluationKey) + await createDetachedWorktree(args.worktreePath, incumbentCommit, candidateWorktree) + let verdict: SmokeVerdict + try { + await writeFile(join(candidateWorktree, spec.surface), surface) + verdict = await deps.smokeRunner({ + scratchPath: candidateWorktree, + generation, + proposer: spec, + evaluationKey, + costLedger, + }) + } finally { + await removeDetachedWorktree(args.worktreePath, candidateWorktree) + } if (deps.scoreSplit !== null && deps.scoreSplit.privateInstances.includes(verdict.iid)) { throw new Error( `gepa seat '${spec.name}': smoke ran PRIVATE instance ${verdict.iid} — refusing to feed its score to the bridge`, ) } innerScores.push({ - call: innerScores.length + 1, + call, scenarioId: scenario.id, smokeIid: verdict.iid, - candidateSha256: sha256(surface), + candidateSha256, composite: innerSmokeComposite(verdict), resolved: verdict.resolved, verifyPass: verdict.verifyPass ?? null, @@ -679,7 +736,7 @@ export function gepaSeatAuthor(config: OuterLoopConfig, deps: GepaSeatDeps): Aut wallS: verdict.wallS, }) log( - `gepa seat ${spec.name} inner call ${innerScores.length}/${budget}: ` + + `gepa seat ${spec.name} inner call ${call}/${budget}: ` + `composite=${innerSmokeComposite(verdict)} (${verdict.reason})`, ) return verdict @@ -687,6 +744,7 @@ export function gepaSeatAuthor(config: OuterLoopConfig, deps: GepaSeatDeps): Aut const factory: GepaMethodFactory = deps.methodFactory ?? gepaOptimizationMethod + const innerJudge = innerSmokeJudge() const optimizer = deps.optimizer ?? (deps.methodFactory @@ -709,12 +767,13 @@ export function gepaSeatAuthor(config: OuterLoopConfig, deps: GepaSeatDeps): Aut name: `gepa-seat:${spec.name}`, recipe, objective, - evaluationId: [ - 'swe-arena-gepa-seat', - `smoke=${deps.smokeInstanceId}`, - 'judge=gepa-inner-smoke', - `dispatchTimeoutMs=${config.dispatchTimeoutMs}`, - ].join('|'), + evaluationId: gepaSeatEvaluationId({ + smokeInstanceId: deps.smokeInstanceId, + dispatchTimeoutMs: config.dispatchTimeoutMs, + incumbentCommit, + runnerImplementationRef: deps.runnerImplementationRef, + judgeImplementationRef: deps.judgeImplementationRef, + }), background, describeScenario: (scenario) => ({ id: scenario.id }), ...(optimizer ? { optimizer } : {}), @@ -730,7 +789,7 @@ export function gepaSeatAuthor(config: OuterLoopConfig, deps: GepaSeatDeps): Aut trainScenarios: scenarios.train, selectionScenarios: scenarios.selection, dispatchWithSurface, - judges: [innerSmokeJudge()], + judges: [innerJudge], runDir, seed: config.round * 1000 + generation, runOptions: { @@ -749,15 +808,19 @@ export function gepaSeatAuthor(config: OuterLoopConfig, deps: GepaSeatDeps): Aut const result = await method.optimize(input) let innerRun: GepaSeatInnerRun try { + const orderedInnerScores = [...innerScores].sort((a, b) => a.call - b.call) innerRun = { seat: spec.name, engine: spec.engine, surface: spec.surface, generation, budget, - innerCallCount: innerScores.length, - innerScores, - bestComposite: innerScores.length > 0 ? Math.max(...innerScores.map((s) => s.composite)) : null, + innerCallCount: orderedInnerScores.length, + innerScores: orderedInnerScores, + bestComposite: + orderedInnerScores.length > 0 + ? Math.max(...orderedInnerScores.map((score) => score.composite)) + : null, ...completeProvenance(result.provenance, spec.name), totalCostUsd: result.cost.totalCostUsd, accountingComplete: result.cost.accountingComplete, diff --git a/bench/src/swe-arena/gepa-seat.test.mts b/bench/src/swe-arena/gepa-seat.test.mts index 246e77c9..6beb2f75 100644 --- a/bench/src/swe-arena/gepa-seat.test.mts +++ b/bench/src/swe-arena/gepa-seat.test.mts @@ -16,6 +16,7 @@ import { GEPA_INNER_RUNS_DIRNAME, GEPA_PYTHON_INSTALL_HINT, gepaBridgeScenarios, + gepaSeatEvaluationId, innerSmokeComposite, innerSmokeJudge, isGepaSeat, @@ -36,6 +37,10 @@ import { captureProposerProvenance } from './proposer-provenance.mts' import { runOk } from './proc.ts' const SURFACE = 'extensions/pi/prompts/worker-coding-system.md' +const IMPLEMENTATION_REFS = { + runnerImplementationRef: `sha256:${'a'.repeat(64)}`, + judgeImplementationRef: `sha256:${'b'.repeat(64)}`, +} as const const seat = (over: Partial = {}): ProposerSpec => ({ name: 'gepa-author', @@ -569,16 +574,36 @@ describe('fanOutLoopsGenerator with the gepa seat', () => { it('construction fails loud without the smoke runner (the inner evaluator)', () => { expect(() => fanOutLoopsGenerator(baseConfig([seat()]))).toThrow(/inner evaluator/) + expect(() => + fanOutLoopsGenerator(baseConfig([seat()]), { + smokeRunner: async () => smokeVerdict(), + smokeInstanceId: 'astropy__astropy-13033', + }), + ).toThrow(/immutable runner and judge references/) expect(() => fanOutLoopsGenerator(baseConfig([{ name: 'no-seat-kind' }]))).toThrow(/neither a harness nor an engine/) }) it('materializes each candidate into the scratch surface, applies the winner through the normal prefilter path, and records provenance', async () => { const WINNER = `${SEED}Always run the neighboring test file before finalizing.\n` const LOSER = `${SEED}delete all tests\n` - const seen: Array<{ content: string; scratch: string; hasCostLedger: boolean }> = [] - const smokeRunner: SmokeRunner = async ({ scratchPath, costLedger }) => { + const seen: Array<{ + content: string + scratch: string + evaluationKey: string + hasCostLedger: boolean + }> = [] + const smokeRunner: SmokeRunner = async ({ + scratchPath, + evaluationKey, + costLedger, + }) => { const content = await readFile(join(scratchPath, SURFACE), 'utf8') - seen.push({ content, scratch: scratchPath, hasCostLedger: costLedger !== undefined }) + seen.push({ + content, + scratch: scratchPath, + evaluationKey, + hasCostLedger: costLedger !== undefined, + }) // The winner candidate resolves; the seed gets verify-pass only; the // loser gets nothing — exercising resolve-dominates + tiebreak. if (content === WINNER) return smokeVerdict({ resolved: true, verifyPass: true }) @@ -588,6 +613,7 @@ describe('fanOutLoopsGenerator with the gepa seat', () => { const observed: { config?: unknown } = {} const config = baseConfig([seat({ maxMetricCalls: 5 })], { activationGate: true }) const gen = fanOutLoopsGenerator(config, { + ...IMPLEMENTATION_REFS, smokeRunner, smokeInstanceId: 'astropy__astropy-13033', scoreSplit: { privateInstances: ['django__django-11532'] }, @@ -599,11 +625,14 @@ describe('fanOutLoopsGenerator with the gepa seat', () => { expect(result).toMatchObject({ applied: true, label: 'gepa-author' }) expect(result.rationale).toContain('engine gepa') - // 3 inner calls (seed, loser, winner) + 1 stage-B prefilter smoke on the - // final candidate — all in the seat's scratch worktree, never the driver. + // 3 isolated inner calls (seed, loser, winner) + 1 stage-B prefilter smoke + // on the final candidate, all outside the driver worktree. expect(seen).toHaveLength(4) for (const call of seen) expect(call.scratch).not.toBe(driverWt) expect(seen.map((c) => c.content)).toEqual([SEED, LOSER, WINNER, WINNER]) + expect(new Set(seen.slice(0, 3).map((call) => call.scratch)).size).toBe(3) + expect(seen.slice(0, 3).every((call) => call.evaluationKey.startsWith('inner-'))).toBe(true) + expect(seen[3]!.evaluationKey).toBe('candidate-0') expect(seen.slice(0, 3).every((call) => call.hasCostLedger)).toBe(true) // The winner landed on the driver worktree with the mechanical predicate. expect(await readFile(join(driverWt, SURFACE), 'utf8')).toBe(WINNER) @@ -616,9 +645,14 @@ describe('fanOutLoopsGenerator with the gepa seat', () => { .filter(Boolean) expect(changed.sort()).toEqual([ACTIVATION_PREDICATE_RELPATH, SURFACE].sort()) // Budget threaded into the adapter recipe. + const incumbentCommit = await git(['rev-parse', 'HEAD'], driverWt) expect(observed.config).toMatchObject({ - evaluationId: - `swe-arena-gepa-seat|smoke=astropy__astropy-13033|judge=gepa-inner-smoke|dispatchTimeoutMs=${config.dispatchTimeoutMs}`, + evaluationId: gepaSeatEvaluationId({ + smokeInstanceId: 'astropy__astropy-13033', + dispatchTimeoutMs: config.dispatchTimeoutMs, + incumbentCommit, + ...IMPLEMENTATION_REFS, + }), recipe: { kind: 'engine', run: { engine: 'gepa', maxEvaluations: 5 } }, optimizer: testOptimizer, resume: 'if-compatible', @@ -670,6 +704,116 @@ describe('fanOutLoopsGenerator with the gepa seat', () => { expect(gen.drainPrefilterKills()).toEqual([]) }) + it('isolates concurrent candidate evaluations while preserving parallel execution', async () => { + const candidateA = `${SEED}candidate A keeps its own workspace\n` + const candidateB = `${SEED}candidate B keeps its own workspace\n` + let releaseBoth!: () => void + const bothStarted = new Promise((resolve) => { + releaseBoth = resolve + }) + let started = 0 + const innerSeen: Array<{ content: string; scratchPath: string }> = [] + const smokeRunner: SmokeRunner = async ({ + scratchPath, + evaluationKey, + }) => { + if (evaluationKey.startsWith('inner-')) { + started += 1 + if (started === 2) releaseBoth() + await bothStarted + } + const content = await readFile(join(scratchPath, SURFACE), 'utf8') + if (evaluationKey.startsWith('inner-')) { + innerSeen.push({ content, scratchPath }) + } + return smokeVerdict({ resolved: content === candidateA }) + } + const parallelFactory: GepaMethodFactory = () => ({ + name: 'parallel-gepa', + async optimize(input) { + const scenario = input.trainScenarios[0]! + await Promise.all([ + input.dispatchWithSurface(candidateA, scenario, fakeCtx), + input.dispatchWithSurface(candidateB, scenario, fakeCtx), + ]) + return { + winnerSurface: candidateA, + cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + durationMs: 1, + provenance: fullProvenance('parallel-gepa', { evaluationCount: 2 }), + } + }, + }) + const gen = fanOutLoopsGenerator(baseConfig([seat({ maxMetricCalls: 2 })]), { + ...IMPLEMENTATION_REFS, + smokeRunner, + smokeInstanceId: 'astropy__astropy-13033', + scoreSplit: null, + gepaMethodFactory: parallelFactory, + }) + + const result = await gen.generate(generatorArgs(0)) + + expect(result.applied).toBe(true) + expect(innerSeen.map((entry) => entry.content).sort()).toEqual( + [candidateA, candidateB].sort(), + ) + expect(new Set(innerSeen.map((entry) => entry.scratchPath)).size).toBe(2) + }) + + it('uses explicit immutable refs so captured runner or judge behavior cannot share resume state', async () => { + const config = baseConfig([seat()]) + const incumbentCommit = await git(['rev-parse', 'HEAD'], driverWt) + const makeRunner = (resolved: boolean): SmokeRunner => + async () => smokeVerdict({ resolved }) + const runnerV1 = makeRunner(false) + const runnerV2 = makeRunner(true) + const common = { + smokeInstanceId: 'astropy__astropy-13033', + dispatchTimeoutMs: config.dispatchTimeoutMs, + incumbentCommit, + } + const original = gepaSeatEvaluationId({ ...common, ...IMPLEMENTATION_REFS }) + const changedRunner = gepaSeatEvaluationId({ + ...common, + ...IMPLEMENTATION_REFS, + runnerImplementationRef: `sha256:${'c'.repeat(64)}`, + }) + const changedJudge = gepaSeatEvaluationId({ + ...common, + ...IMPLEMENTATION_REFS, + judgeImplementationRef: `sha256:${'d'.repeat(64)}`, + }) + const changedIncumbent = gepaSeatEvaluationId({ + ...common, + ...IMPLEMENTATION_REFS, + incumbentCommit: 'e'.repeat(40), + }) + const smokeArgs: Parameters[0] = { + scratchPath: driverWt, + generation: 0, + proposer: seat(), + evaluationKey: 'identity-test', + } + + expect(runnerV1.toString()).toBe(runnerV2.toString()) + expect((await runnerV1(smokeArgs)).resolved).toBe(false) + expect((await runnerV2(smokeArgs)).resolved).toBe(true) + expect(original).toMatch( + /^swe-arena-gepa-seat\|smoke=astropy__astropy-13033\|incumbent=[a-f0-9]{40,64}\|runner=sha256:[a-f0-9]{64}\|judge=sha256:[a-f0-9]{64}\|dispatchTimeoutMs=\d+$/, + ) + expect(changedRunner).not.toBe(original) + expect(changedJudge).not.toBe(original) + expect(changedIncumbent).not.toBe(original) + expect(() => + gepaSeatEvaluationId({ + ...common, + ...IMPLEMENTATION_REFS, + runnerImplementationRef: 'runner-v2', + }), + ).toThrow(/runnerImplementationRef must be an immutable sha256 reference/) + }) + it('records but rejects a winner when cost accounting is incomplete', async () => { const WINNER = `${SEED}Always run the neighboring test file before finalizing.\n` const incomplete: GepaMethodFactory = () => ({ @@ -689,6 +833,7 @@ describe('fanOutLoopsGenerator with the gepa seat', () => { }, }) const gen = fanOutLoopsGenerator(baseConfig([seat()]), { + ...IMPLEMENTATION_REFS, smokeRunner: async () => smokeVerdict({ resolved: true }), smokeInstanceId: 'astropy__astropy-13033', scoreSplit: null, @@ -714,9 +859,19 @@ describe('fanOutLoopsGenerator with the gepa seat', () => { const runaway: GepaMethodFactory = () => ({ name: 'runaway', async optimize(input) { - for (let i = 0; i < 4; i++) { - await input.dispatchWithSurface(`${SEED}candidate ${i}\n`, input.trainScenarios[0]!, fakeCtx) - } + const results = await Promise.allSettled( + Array.from({ length: 4 }, (_, index) => + input.dispatchWithSurface( + `${SEED}candidate ${index}\n`, + input.trainScenarios[0]!, + fakeCtx, + ), + ), + ) + const rejected = results.find( + (result): result is PromiseRejectedResult => result.status === 'rejected', + ) + if (rejected) throw rejected.reason return { winnerSurface: SEED, cost: { totalCostUsd: 0, accountingComplete: false, incompleteReasons: [] }, @@ -726,6 +881,7 @@ describe('fanOutLoopsGenerator with the gepa seat', () => { }, }) const gen = fanOutLoopsGenerator(baseConfig([seat({ maxMetricCalls: 3 })]), { + ...IMPLEMENTATION_REFS, smokeRunner: async () => smokeVerdict(), smokeInstanceId: 'astropy__astropy-13033', scoreSplit: null, @@ -736,6 +892,7 @@ describe('fanOutLoopsGenerator with the gepa seat', () => { it('refuses to feed a PRIVATE smoke verdict to the bridge', async () => { const gen = fanOutLoopsGenerator(baseConfig([seat()]), { + ...IMPLEMENTATION_REFS, smokeRunner: async () => smokeVerdict({ iid: 'django__django-11532' }), smokeInstanceId: 'astropy__astropy-13033', scoreSplit: { privateInstances: ['django__django-11532'] }, @@ -746,6 +903,7 @@ describe('fanOutLoopsGenerator with the gepa seat', () => { it('declines the slot without a kill when GEPA returns the seed unchanged', async () => { const gen = fanOutLoopsGenerator(baseConfig([seat()]), { + ...IMPLEMENTATION_REFS, smokeRunner: async () => smokeVerdict(), smokeInstanceId: 'astropy__astropy-13033', scoreSplit: null, @@ -837,6 +995,7 @@ describe('integration: real adapter roundtrip', () => { prefilter: { enabled: true, smokeInstance: 'cheapest-of-set', requireResolved: false }, }, { + ...IMPLEMENTATION_REFS, smokeRunner: async ({ scratchPath }) => { const candidate = await readFile(join(scratchPath, SURFACE), 'utf8') return { diff --git a/bench/src/swe-arena/implementation-ref.test.mts b/bench/src/swe-arena/implementation-ref.test.mts new file mode 100644 index 00000000..1a965079 --- /dev/null +++ b/bench/src/swe-arena/implementation-ref.test.mts @@ -0,0 +1,64 @@ +import { mkdir, mkdtemp, rm, symlink, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + fileTreeImplementationRef, + pythonDistributionImplementationRef, +} from './implementation-ref.ts' + +describe('implementation refs', () => { + const roots: string[] = [] + + afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))) + }) + + it('is stable across creation order and changes with file content or symlink targets', async () => { + const first = await mkdtemp(join(tmpdir(), 'implementation-ref-a-')) + const second = await mkdtemp(join(tmpdir(), 'implementation-ref-b-')) + roots.push(first, second) + await mkdir(join(first, 'nested')) + await writeFile(join(first, 'nested', 'b.ts'), 'export const b = 2\n') + await writeFile(join(first, 'a.ts'), 'export const a = 1\n') + await symlink('a.ts', join(first, 'entry.ts')) + await writeFile(join(second, 'a.ts'), 'export const a = 1\n') + await symlink('a.ts', join(second, 'entry.ts')) + await mkdir(join(second, 'nested')) + await writeFile(join(second, 'nested', 'b.ts'), 'export const b = 2\n') + + const initial = await fileTreeImplementationRef(first) + expect(await fileTreeImplementationRef(second)).toBe(initial) + + await writeFile(join(second, 'nested', 'b.ts'), 'export const b = 3\n') + expect(await fileTreeImplementationRef(second)).not.toBe(initial) + await writeFile(join(second, 'nested', 'b.ts'), 'export const b = 2\n') + await rm(join(second, 'entry.ts')) + await symlink('nested/b.ts', join(second, 'entry.ts')) + expect(await fileTreeImplementationRef(second)).not.toBe(initial) + }) + + it('binds a Python distribution to its complete manifest', async () => { + const runPython = async (_script: string, args: string[]): Promise => + `import warning\n${JSON.stringify({ + distribution: args[0], + version: '1.0.0', + python: '3.12.4', + files: [{ path: '__init__.py', sha256: 'a'.repeat(64) }], + })}` + const original = await pythonDistributionImplementationRef('swebench', runPython) + const changed = await pythonDistributionImplementationRef( + 'swebench', + async (_script, args) => + JSON.stringify({ + distribution: args[0], + version: '1.0.1', + python: '3.12.4', + files: [{ path: '__init__.py', sha256: 'b'.repeat(64) }], + }), + ) + + expect(original).toMatch(/^sha256:[a-f0-9]{64}$/) + expect(changed).not.toBe(original) + }) +}) diff --git a/bench/src/swe-arena/implementation-ref.ts b/bench/src/swe-arena/implementation-ref.ts new file mode 100644 index 00000000..84aeb39e --- /dev/null +++ b/bench/src/swe-arena/implementation-ref.ts @@ -0,0 +1,62 @@ +import { createHash } from 'node:crypto' +import { lstat, readFile, readdir, readlink, realpath } from 'node:fs/promises' +import { join, relative } from 'node:path' +import { canonicalCandidateDigest } from '@tangle-network/agent-interface' + +const sha256Content = (value: Uint8Array): string => + `sha256:${createHash('sha256').update(value).digest('hex')}` + +export async function fileTreeImplementationRef(root: string): Promise { + const canonicalRoot = await realpath(root) + const files: Array<{ path: string; sha256?: string; symlink?: string }> = [] + + const walk = async (directory: string): Promise => { + const entries = await readdir(directory, { withFileTypes: true }) + entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)) + for (const entry of entries) { + const path = join(directory, entry.name) + const relativePath = relative(canonicalRoot, path) + if (entry.isDirectory()) { + await walk(path) + } else if (entry.isSymbolicLink()) { + files.push({ path: relativePath, symlink: await readlink(path) }) + } else if (entry.isFile() || (await lstat(path)).isFile()) { + files.push({ path: relativePath, sha256: sha256Content(await readFile(path)) }) + } + } + } + + await walk(canonicalRoot) + return canonicalCandidateDigest(files) +} + +export async function pythonDistributionImplementationRef( + distribution: string, + runPython: (script: string, args: string[]) => Promise, +): Promise { + const output = await runPython( + [ + 'import hashlib, importlib, importlib.metadata, json, pathlib, sys', + 'name = sys.argv[1]', + 'module = importlib.import_module(name)', + 'root = pathlib.Path(module.__file__).resolve().parent', + 'files = []', + 'for path in sorted(p for p in root.rglob("*") if p.is_file()):', + ' if "__pycache__" in path.parts or path.suffix == ".pyc":', + ' continue', + ' files.append({"path": str(path.relative_to(root)), "sha256": hashlib.sha256(path.read_bytes()).hexdigest()})', + 'try:', + ' version = importlib.metadata.version(name)', + 'except importlib.metadata.PackageNotFoundError:', + ' version = None', + 'print(json.dumps({"distribution": name, "version": version, "python": sys.version, "files": files}, sort_keys=True))', + ].join('\n'), + [distribution], + ) + const line = output.trim().split('\n').at(-1) + if (line === undefined) { + throw new Error(`python distribution ${distribution} returned no implementation manifest`) + } + const manifest = JSON.parse(line) as unknown + return canonicalCandidateDigest(manifest) +} diff --git a/bench/src/swe-arena/outer-loop.mts b/bench/src/swe-arena/outer-loop.mts index d2590c44..06113628 100644 --- a/bench/src/swe-arena/outer-loop.mts +++ b/bench/src/swe-arena/outer-loop.mts @@ -72,6 +72,7 @@ import { type Verifier, } from '@tangle-network/agent-runtime' import { runLocalHarness } from '@tangle-network/agent-runtime/mcp' +import { canonicalCandidateDigest } from '@tangle-network/agent-interface' import { makeFinding } from '@tangle-network/agent-eval' import { FsLabeledScenarioStore, @@ -85,7 +86,12 @@ import { type Scenario, } from '@tangle-network/agent-eval/campaign' import type { CostLedgerHandle } from '@tangle-network/agent-eval' +import { runVenvPython } from '../benchmarks/_harness.ts' import { createSweBenchAdapter } from '../benchmarks/swe-bench.ts' +import { + fileTreeImplementationRef, + pythonDistributionImplementationRef, +} from './implementation-ref.ts' import { baselineDriftWarnings, cellsFromCampaign, @@ -1528,11 +1534,24 @@ export async function runRound(config: OuterLoopConfig, signal?: AbortSignal): P const smokeRunner: SmokeRunner | undefined = smokeIid === null ? undefined - : async ({ scratchPath, generation, proposer, costLedger }): Promise => { + : async ({ + scratchPath, + generation, + proposer, + evaluationKey, + costLedger, + }): Promise => { const iid = smokeIid + if (!/^[a-zA-Z0-9_-]+$/.test(evaluationKey)) { + throw new Error(`outer-loop: invalid smoke evaluation key ${JSON.stringify(evaluationKey)}`) + } const requireResolved = config.prefilter?.requireResolved === true const entry = images[iid]! - const armOutDir = join(config.outDir, 'prefilter-smoke', `gen${generation}-${proposer.name}`) + const armOutDir = join( + config.outDir, + 'prefilter-smoke', + `gen${generation}-${proposer.name}-${evaluationKey}`, + ) const nm = join(scratchPath, 'node_modules') let linked = false const t0 = Date.now() @@ -1591,7 +1610,7 @@ export async function runRound(config: OuterLoopConfig, signal?: AbortSignal): P const paid = await costLedger.runPaidCall({ channel: 'agent', phase: 'search.prefilter', - actor: `prefilter-smoke:${iid}:g${generation}:${proposer.name}`, + actor: `prefilter-smoke:${iid}:g${generation}:${proposer.name}:${evaluationKey}`, model: config.arm.workerModel, execute: runWork, receipt: ({ armRes }) => { @@ -1652,6 +1671,79 @@ export async function runRound(config: OuterLoopConfig, signal?: AbortSignal): P } } + let runnerImplementationRef: string | undefined + let judgeImplementationRef: string | undefined + const hasGepaSeat = config.proposers?.some((proposer) => proposer.engine !== undefined) === true + if (hasGepaSeat && smokeIid !== null && smokeRunner !== undefined) { + const runtimeRoot = fileURLToPath(new URL('../../..', import.meta.url)) + const benchRoot = fileURLToPath(new URL('../..', import.meta.url)) + const sourceCommit = ( + await runOk('git', ['-C', config.loopsRepo, 'rev-parse', 'HEAD']) + ).stdout.trim() + const entry = images[smokeIid]! + const verifyScript = await readFile(join(config.verifyDir, `${smokeIid}.sh`), 'utf8') + const implementationArtifacts = { + benchSource: await fileTreeImplementationRef(join(benchRoot, 'src')), + runtimeSource: await fileTreeImplementationRef(join(runtimeRoot, 'src')), + runtimeDist: await fileTreeImplementationRef(join(runtimeRoot, 'dist')), + packageFiles: canonicalCandidateDigest({ + runtimePackage: await readFile(join(runtimeRoot, 'package.json'), 'utf8'), + benchPackage: await readFile(join(benchRoot, 'package.json'), 'utf8'), + lockfile: await readFile(join(runtimeRoot, 'pnpm-lock.yaml'), 'utf8'), + }), + swebench: await pythonDistributionImplementationRef( + 'swebench', + (script, args) => runVenvPython(script, args), + ), + } + const runnerSourceRef = canonicalCandidateDigest({ + benchSource: implementationArtifacts.benchSource, + runtimeSource: implementationArtifacts.runtimeSource, + runtimeDist: implementationArtifacts.runtimeDist, + packageFiles: implementationArtifacts.packageFiles, + }) + const judgeSourceRef = canonicalCandidateDigest({ + benchSource: implementationArtifacts.benchSource, + packageFiles: implementationArtifacts.packageFiles, + swebench: implementationArtifacts.swebench, + }) + runnerImplementationRef = canonicalCandidateDigest({ + implementation: 'swe-arena-smoke-runner', + sourceCommit, + sourceRef: runnerSourceRef, + smoke: { + iid: smokeIid, + image: entry.image, + baseCommit: entry.base_commit, + problemStatement: problemById.get(smokeIid)!, + verifyScript, + requireResolved: config.prefilter?.requireResolved === true, + }, + arm: { + name: config.armName, + workerModel: config.arm.workerModel, + driverModel: config.arm.driverModel, + budget: config.arm.budget, + maxSandboxes: config.arm.maxSandboxes, + maxUsd: config.arm.maxUsd, + maxDepth: config.arm.maxDepth, + timeoutMs: config.arm.timeoutMs, + envKnobs: config.arm.envKnobs ?? null, + }, + dispatchTimeoutMs: config.dispatchTimeoutMs, + capacityModel: config.capacityModel ?? null, + gateWaitCeilingMs: config.gateWaitCeilingMs ?? null, + excludes, + }) + judgeImplementationRef = canonicalCandidateDigest({ + implementation: 'serialized-swebench-judge', + sourceCommit, + sourceRef: judgeSourceRef, + timeoutMs: config.judgeTimeoutMs ?? JUDGE_TIMEOUT_FLOOR_MS, + cacheLevel: 'instance', + }) + } + // ── dispatch: one (surface × scenario) cell ────────────────────────── const agent = async (surface: MutableSurface, scenario: Scenario, ctx: DispatchContext): Promise => { const cs = asCodeSurface(surface) @@ -2032,6 +2124,8 @@ export async function runRound(config: OuterLoopConfig, signal?: AbortSignal): P config.proposers !== undefined ? fanOutLoopsGenerator(config, { ...(smokeRunner ? { smokeRunner } : {}), + ...(runnerImplementationRef ? { runnerImplementationRef } : {}), + ...(judgeImplementationRef ? { judgeImplementationRef } : {}), ...(paretoParents.length > 0 ? { parents: paretoParents } : {}), ...(briefingCtx !== undefined ? { briefing: briefingCtx } : {}), // The GEPA seat's inner evaluator uses the same public-only diff --git a/bench/src/swe-arena/proposer-fanout.mts b/bench/src/swe-arena/proposer-fanout.mts index 80a223cc..40a57f86 100644 --- a/bench/src/swe-arena/proposer-fanout.mts +++ b/bench/src/swe-arena/proposer-fanout.mts @@ -73,6 +73,11 @@ import { } from './gepa-seat.mts' import type { ScoreSplit } from './score-split.mts' import { run, runOk } from './proc.ts' +import { + createDetachedWorktree, + pruneDetachedWorktrees, + removeDetachedWorktree, +} from './scratch-worktree.ts' // --------------------------------------------------------------------------- // Config types. @@ -151,6 +156,7 @@ export type SmokeRunner = (args: { scratchPath: string generation: number proposer: ProposerSpec + evaluationKey: string costLedger?: CostLedgerHandle }) => Promise @@ -442,21 +448,6 @@ export function proposerShotHooks(opts: { // smoke runner link one in temporarily when they need it). // --------------------------------------------------------------------------- -async function addAuthoringWorktree(loopsRepo: string, commit: string, dest: string): Promise { - await run('git', ['-C', loopsRepo, 'worktree', 'remove', '--force', '--', dest]) - await rm(dest, { recursive: true, force: true }) - await run('git', ['-C', loopsRepo, 'worktree', 'prune']) - await runOk('git', ['-C', loopsRepo, 'worktree', 'add', '--detach', dest, commit]) -} - -async function removeAuthoringWorktree(loopsRepo: string, dest: string): Promise { - const res = await run('git', ['-C', loopsRepo, 'worktree', 'remove', '--force', '--', dest]) - if (res.code !== 0) { - await rm(dest, { recursive: true, force: true }) - await run('git', ['-C', loopsRepo, 'worktree', 'prune']) - } -} - const sanitize = (s: string): string => s.replace(/[^a-zA-Z0-9_-]/g, '_') // --------------------------------------------------------------------------- @@ -485,6 +476,10 @@ export type AuthorFn = ( export interface FanOutDeps { author?: AuthorFn smokeRunner?: SmokeRunner + /** Immutable digest of the smoke runner plus every captured execution input. */ + runnerImplementationRef?: string + /** Immutable digest of the official judge implementation and configuration. */ + judgeImplementationRef?: string /** GEN-4: materialized Pareto parents seeded into every author's prompt (and * the merge seat's explicit merge input). Empty/omitted = gen-3 behavior. */ parents?: ParetoParentContext[] @@ -523,6 +518,8 @@ function defaultAuthor(config: OuterLoopConfig, deps: FanOutDeps): AuthorFn { if (isGepaSeat(proposer)) { gepaAuthor ??= gepaSeatAuthor(config, { smokeRunner: deps.smokeRunner!, + runnerImplementationRef: deps.runnerImplementationRef!, + judgeImplementationRef: deps.judgeImplementationRef!, smokeInstanceId: deps.smokeInstanceId!, scoreSplit: deps.scoreSplit ?? null, ...(deps.gepaMethodFactory ? { methodFactory: deps.gepaMethodFactory } : {}), @@ -604,10 +601,19 @@ export function fanOutLoopsGenerator(config: OuterLoopConfig, deps: FanOutDeps = throw new Error(`fanOutLoopsGenerator: proposer ${seat.name} has neither a harness nor an engine`) } } - if (gepaSeats.length > 0 && deps.author === undefined && (deps.smokeRunner === undefined || deps.smokeInstanceId === undefined)) { + if ( + gepaSeats.length > 0 && + deps.author === undefined && + ( + deps.smokeRunner === undefined || + deps.smokeInstanceId === undefined || + deps.runnerImplementationRef === undefined || + deps.judgeImplementationRef === undefined + ) + ) { throw new Error( `fanOutLoopsGenerator: gepa seat(s) ${gepaSeats.map((p) => p.name).join(', ')} need the pre-filter smoke ` + - 'cell as their inner evaluator — enable config.prefilter and provide smokeRunner + smokeInstanceId', + 'cell plus immutable runner and judge references as their inner evaluator', ) } const author = deps.author ?? defaultAuthor(config, deps) @@ -623,11 +629,12 @@ export function fanOutLoopsGenerator(config: OuterLoopConfig, deps: FanOutDeps = const baseCommit = head.stdout.trim() const patchesDir = join(config.outDir, 'proposer-patches') await mkdir(patchesDir, { recursive: true }) + await pruneDetachedWorktrees(config.loopsRepo) log(`fan-out gen ${generation}: ${proposers.length} proposer(s) authoring in parallel from ${baseCommit.slice(0, 10)}`) return Promise.all( proposers.map(async (proposer, index): Promise => { const scratch = join(config.outDir, 'proposer-wt', `gen${generation}-cand${index}-${sanitize(proposer.name)}`) - await addAuthoringWorktree(config.loopsRepo, baseCommit, scratch) + await createDetachedWorktree(config.loopsRepo, baseCommit, scratch) try { const authored = await author(proposer, { ...args, @@ -704,6 +711,7 @@ export function fanOutLoopsGenerator(config: OuterLoopConfig, deps: FanOutDeps = scratchPath: scratch, generation, proposer, + evaluationKey: `candidate-${index}`, ...(args.costLedger ? { costLedger: args.costLedger } : {}), }) if (!smoke.pass) { @@ -726,7 +734,7 @@ export function fanOutLoopsGenerator(config: OuterLoopConfig, deps: FanOutDeps = } return { proposer, applied: true, summary: authored.summary, patch: diff, kill: null } } finally { - await removeAuthoringWorktree(config.loopsRepo, scratch) + await removeDetachedWorktree(config.loopsRepo, scratch) } }), ) diff --git a/bench/src/swe-arena/scratch-worktree.test.mts b/bench/src/swe-arena/scratch-worktree.test.mts new file mode 100644 index 00000000..a57aa464 --- /dev/null +++ b/bench/src/swe-arena/scratch-worktree.test.mts @@ -0,0 +1,55 @@ +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { runOk } from './proc.ts' +import { + createDetachedWorktree, + pruneDetachedWorktrees, + removeDetachedWorktree, +} from './scratch-worktree.ts' + +describe('scratch worktrees', () => { + const roots: string[] = [] + + afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))) + }) + + it('creates and removes a parallel batch without pruning live metadata', async () => { + const repository = await mkdtemp(join(tmpdir(), 'scratch-worktree-repo-')) + const output = await mkdtemp(join(tmpdir(), 'scratch-worktree-out-')) + roots.push(repository, output) + await runOk('git', ['init', '-q', '-b', 'main', repository]) + await runOk('git', ['-C', repository, 'config', 'user.email', 'test@example.com']) + await runOk('git', ['-C', repository, 'config', 'user.name', 'Test']) + await writeFile(join(repository, 'seed.txt'), 'seed\n') + await runOk('git', ['-C', repository, 'add', 'seed.txt']) + await runOk('git', ['-C', repository, 'commit', '-q', '-m', 'seed']) + const commit = ( + await runOk('git', ['-C', repository, 'rev-parse', 'HEAD']) + ).stdout.trim() + for (let batch = 0; batch < 5; batch += 1) { + const worktrees = Array.from( + { length: 8 }, + (_, index) => join(output, `batch-${batch}-candidate-${index}`), + ) + await pruneDetachedWorktrees(repository) + await Promise.all( + worktrees.map((worktree) => createDetachedWorktree(repository, commit, worktree)), + ) + expect( + await Promise.all( + worktrees.map((worktree) => readFile(join(worktree, 'seed.txt'), 'utf8')), + ), + ).toEqual(Array.from({ length: 8 }, () => 'seed\n')) + await Promise.all( + worktrees.map((worktree) => removeDetachedWorktree(repository, worktree)), + ) + } + const listed = ( + await runOk('git', ['-C', repository, 'worktree', 'list', '--porcelain']) + ).stdout + expect(listed.match(/^worktree /gmu)).toHaveLength(1) + }) +}) diff --git a/bench/src/swe-arena/scratch-worktree.ts b/bench/src/swe-arena/scratch-worktree.ts new file mode 100644 index 00000000..cc7bef52 --- /dev/null +++ b/bench/src/swe-arena/scratch-worktree.ts @@ -0,0 +1,34 @@ +import { rm } from 'node:fs/promises' +import { run, runOk } from './proc' + +export async function createDetachedWorktree( + repository: string, + commit: string, + destination: string, +): Promise { + await run('git', ['-C', repository, 'worktree', 'remove', '--force', '--', destination]) + await rm(destination, { recursive: true, force: true }) + await runOk('git', ['-C', repository, 'worktree', 'add', '--detach', destination, commit]) +} + +export async function pruneDetachedWorktrees(repository: string): Promise { + await runOk('git', ['-C', repository, 'worktree', 'prune']) +} + +export async function removeDetachedWorktree( + repository: string, + destination: string, +): Promise { + const result = await run('git', [ + '-C', + repository, + 'worktree', + 'remove', + '--force', + '--', + destination, + ]) + if (result.code !== 0) { + await rm(destination, { recursive: true, force: true }) + } +} From f71696b3c91de5149e3caec145260ff81f1b626d Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 24 Jul 2026 16:36:51 -0600 Subject: [PATCH 13/14] docs(api): lock optimization release version --- docs/api/primitive-catalog.md | 2 +- docs/canonical-api.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index b8c2b249..dfdeb23d 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.104.0` and `@tangle-network/agent-eval@0.126.6` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.105.0` and `@tangle-network/agent-eval@0.126.6` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface diff --git a/docs/canonical-api.md b/docs/canonical-api.md index ec41be6f..4edf0bed 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,7 +4,7 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.104.0.** +> **Version 0.105.0.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. > `agent-eval` must satisfy `>=0.126.6 <0.127.0`. > `sandbox` must satisfy `>=0.12.0 <1.0.0`. From 970c1dd8ce4f5de00a69a61cd997c890700fa06f Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 24 Jul 2026 16:38:24 -0600 Subject: [PATCH 14/14] test(fixtures): regenerate for runtime 0.105.0 --- src/testing/fixtures/agent-improvement-proposal.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index a3f7c8f6..a46e7e40 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:3bfcc7c157327b447ba74cd2cbbaab6e73c893d606dba8a4f70ba2f4752002de", + "digest": "sha256:1f223831184b2b53fe1be2b950a63856acb5abe887e5022a6114b5165a4f1802", "evaluation": { "decision": { "contributingChecks": [ @@ -2472,7 +2472,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.104.0" + "runtimeVersion": "0.105.0" }, "objectives": [ { @@ -2583,8 +2583,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:6de14fd199e2cd140dab91ef10a0f258732e990525a67783735ff2ec75e8358a", - "runId": "agent-runtime-0.104.0-proposal-fixture", + "recordDigest": "sha256:0e732c488490a383c61f092698759f8a32979f739b3e3689bf6466fb8fb10c3b", + "runId": "agent-runtime-0.105.0-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -2610,5 +2610,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.104.0-proposal-fixture" + "runId": "agent-runtime-0.105.0-proposal-fixture" }