From 0b6fce6f9c7d37021b3d7d342dc88f6792b23250 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 24 Jul 2026 19:19:41 -0600 Subject: [PATCH 1/4] feat(bench): check model availability before dispatch --- src/runtime/run-benchmark.test.ts | 245 ++++++++++++++++++++++++++++++ src/runtime/run-benchmark.ts | 42 +++++ 2 files changed, 287 insertions(+) create mode 100644 src/runtime/run-benchmark.test.ts diff --git a/src/runtime/run-benchmark.test.ts b/src/runtime/run-benchmark.test.ts new file mode 100644 index 00000000..8ae5cc54 --- /dev/null +++ b/src/runtime/run-benchmark.test.ts @@ -0,0 +1,245 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { runBenchmark } from './run-benchmark' +import { + type AgenticOptions, + type AgenticSurface, + type AgenticTask, + defineStrategy, +} from './strategy' + +interface ChatRequest { + model?: string + messages?: Array<{ role?: string; content?: string }> + temperature?: number + max_tokens?: number +} + +const task: AgenticTask = { + id: 'task-1', + systemPrompt: 'Use the test surface.', + userPrompt: 'Complete the task.', +} + +const worker: AgenticOptions = { + routerBaseUrl: 'http://router.test/v1', + routerKey: 'test-key', + model: 'worker-model', + maxTokens: 8, +} + +const oneShot = defineStrategy('one-shot', async ({ shot }) => { + const out = await shot() + return { + score: out?.score ?? 0, + resolved: out?.score === 1, + completions: out?.completions ?? 0, + progression: out ? [out.score] : [], + shots: 1, + } +}) + +function surface(events: string[]): AgenticSurface { + let sequence = 0 + return { + name: 'test', + async open() { + events.push('open') + sequence += 1 + return { id: `handle-${sequence}`, surface: 'test' } + }, + async tools() { + return [] + }, + async call() { + return 'ok' + }, + async score() { + return { passes: 1, total: 1, errored: 0 } + }, + async close() {}, + } +} + +function okResponse(): Response { + return Response.json({ + choices: [{ message: { content: 'DONE' } }], + usage: { prompt_tokens: 2, completion_tokens: 1 }, + }) +} + +function stubRouter( + requests: ChatRequest[], + events: string[], + respond: (request: ChatRequest, index: number) => Response = () => okResponse(), +): void { + vi.stubGlobal( + 'fetch', + vi.fn(async (_url: string | URL | Request, init?: RequestInit) => { + const request = JSON.parse(String(init?.body ?? '{}')) as ChatRequest + requests.push(request) + events.push(`request:${request.model}:${request.max_tokens ?? 'default'}`) + return respond(request, requests.length - 1) + }), + ) +} + +function isModelCheck(request: ChatRequest): boolean { + return request.messages?.[0]?.content === 'Reply OK.' +} + +afterEach(() => { + vi.unstubAllGlobals() +}) + +describe('runBenchmark model availability', () => { + it('checks every unique model once before opening any task', async () => { + const requests: ChatRequest[] = [] + const events: string[] = [] + stubRouter(requests, events) + + await runBenchmark({ + environment: surface(events), + tasks: [task, { ...task, id: 'task-2' }], + worker: { ...worker, analystModel: 'analyst-model' }, + strategies: [oneShot], + budget: 1, + concurrency: 2, + }) + + const checks = requests.filter(isModelCheck) + expect(checks.map((request) => request.model).sort()).toEqual(['analyst-model', 'worker-model']) + expect(checks.every((request) => request.max_tokens === 1)).toBe(true) + expect(events.indexOf('open')).toBe(2) + }) + + it('deduplicates identical worker and analyst models', async () => { + const checked: string[] = [] + const events: string[] = [] + + await runBenchmark({ + environment: surface(events), + tasks: [task], + worker: { + ...worker, + analystModel: worker.model, + complete: async () => okResponse().json(), + }, + strategies: [oneShot], + budget: 1, + modelPreflight: async (model) => { + checked.push(model) + }, + }) + + expect(checked).toEqual(['worker-model']) + }) + + it('rejects an unavailable model before task dispatch', async () => { + const requests: ChatRequest[] = [] + const events: string[] = [] + const onTask = vi.fn() + stubRouter(requests, events, () => new Response('model has been deprecated', { status: 404 })) + + await expect( + runBenchmark({ + environment: surface(events), + tasks: [task], + worker, + strategies: [oneShot], + budget: 1, + onTask, + }), + ).rejects.toThrow( + 'Benchmark model "worker-model" preflight failed: router 404: model has been deprecated', + ) + + expect(requests).toHaveLength(1) + expect(events).not.toContain('open') + expect(onTask).not.toHaveBeenCalled() + }) + + it('skips the check for an injected completion transport', async () => { + const requests: ChatRequest[] = [] + const events: string[] = [] + const complete = vi.fn(async (body: Record) => { + requests.push(body as ChatRequest) + events.push('complete') + return okResponse().json() + }) + + await runBenchmark({ + environment: surface(events), + tasks: [task], + worker: { ...worker, complete }, + strategies: [oneShot], + budget: 1, + }) + + expect(complete).toHaveBeenCalledTimes(1) + expect(requests.some(isModelCheck)).toBe(false) + expect(events[0]).toBe('open') + }) + + it('allows the live check to be disabled explicitly', async () => { + const requests: ChatRequest[] = [] + const events: string[] = [] + stubRouter(requests, events) + + await runBenchmark({ + environment: surface(events), + tasks: [task], + worker, + strategies: [oneShot], + budget: 1, + modelPreflight: false, + }) + + expect(requests.some(isModelCheck)).toBe(false) + expect(events[0]).toBe('open') + }) + + it('runs a custom check for injected transports', async () => { + const events: string[] = [] + const checked: string[] = [] + + await runBenchmark({ + environment: surface(events), + tasks: [task], + worker: { + ...worker, + analystModel: 'analyst-model', + complete: async () => okResponse().json(), + }, + strategies: [oneShot], + budget: 1, + modelPreflight: async (model) => { + events.push(`check:${model}`) + checked.push(model) + }, + }) + + expect(checked.sort()).toEqual(['analyst-model', 'worker-model']) + expect(events.indexOf('open')).toBe(2) + }) + + it('retries at temperature one when the provider requires it', async () => { + const requests: ChatRequest[] = [] + const events: string[] = [] + stubRouter(requests, events, (_request, index) => + index === 0 + ? new Response('only temperature 1 is allowed for this model', { status: 400 }) + : okResponse(), + ) + + await runBenchmark({ + environment: surface(events), + tasks: [task], + worker, + strategies: [oneShot], + budget: 1, + }) + + expect(requests.slice(0, 2).map((request) => request.temperature)).toEqual([0.2, 1]) + expect(events.indexOf('open')).toBe(2) + }) +}) diff --git a/src/runtime/run-benchmark.ts b/src/runtime/run-benchmark.ts index ac1c3dfa..ff5dee1f 100644 --- a/src/runtime/run-benchmark.ts +++ b/src/runtime/run-benchmark.ts @@ -15,6 +15,7 @@ import { pairedBootstrap, paretoFrontier } from '@tangle-network/agent-eval' import type { RuntimeHooks } from '../runtime-hooks' +import { routerChatWithUsage } from './router-client' import { type AgenticOptions, type AgenticSurface, @@ -49,6 +50,14 @@ export interface BenchmarkConfig { /** Lifecycle observability — every spawn/settle of every cell's shots/analysts streams * here live (the watchdog/route-auditor seam, passed through to `runAgentic`). */ hooks?: RuntimeHooks + /** + * Model availability check before tasks start. + * + * By default, live router workers send one one-token request per unique worker and analyst + * model. Injected `worker.complete` transports skip the check. Pass `false` to disable it or a + * callback to check each unique model through a custom transport. + */ + modelPreflight?: false | ((model: string, worker: Readonly) => Promise) } export interface BenchmarkLift { @@ -127,6 +136,37 @@ async function pool( return out } +async function preflightModels(cfg: BenchmarkConfig): Promise { + if (cfg.modelPreflight === false) return + if (cfg.worker.complete && !cfg.modelPreflight) return + + const models = [...new Set([cfg.worker.model, cfg.worker.analystModel ?? cfg.worker.model])] + const check = + cfg.modelPreflight ?? + (async (model: string, worker: Readonly) => { + await routerChatWithUsage( + { + routerBaseUrl: worker.routerBaseUrl, + routerKey: worker.routerKey, + model, + }, + [{ role: 'user', content: 'Reply OK.' }], + { maxTokens: 1 }, + ) + }) + + await Promise.all( + models.map(async (model) => { + try { + await check(model, cfg.worker) + } catch (cause) { + const message = cause instanceof Error ? cause.message : String(cause) + throw new Error(`Benchmark model "${model}" preflight failed: ${message}`, { cause }) + } + }), + ) +} + /** Run the requested strategies over the tasks, scored by the Environment's own check. * Resilient: a task whose rollouts fail (transient infra) is excluded from the stats but * reported in `perTask` with the error — never silently dropped. */ @@ -135,6 +175,8 @@ export async function runBenchmark(cfg: BenchmarkConfig): Promise => { const cells: Record = {} From 1ba4b8069ee667b1e9020da8bf15caab03295680 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 24 Jul 2026 19:34:31 -0600 Subject: [PATCH 2/4] docs(api): refresh benchmark model preflight reference --- docs/api/runtime.md | 94 +++++++++++++++++++++++++-------------------- 1 file changed, 53 insertions(+), 41 deletions(-) diff --git a/docs/api/runtime.md b/docs/api/runtime.md index ce47c7ca..6e3b10bb 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -6104,7 +6104,7 @@ The full conversation after the loop (seed + every assistant/tool turn). Lets a ### BenchmarkConfig -Defined in: [src/runtime/run-benchmark.ts:32](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L32) +Defined in: [src/runtime/run-benchmark.ts:33](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L33) #### Properties @@ -6112,7 +6112,7 @@ Defined in: [src/runtime/run-benchmark.ts:32](https://github.com/tangle-network/ > **environment**: [`AgenticSurface`](#agenticsurface) -Defined in: [src/runtime/run-benchmark.ts:34](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L34) +Defined in: [src/runtime/run-benchmark.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L35) The task domain (5 hooks). @@ -6120,7 +6120,7 @@ The task domain (5 hooks). > **tasks**: [`AgenticTask`](#agentictask)[] -Defined in: [src/runtime/run-benchmark.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L36) +Defined in: [src/runtime/run-benchmark.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L37) The tasks to score across. @@ -6128,7 +6128,7 @@ The tasks to score across. > **worker**: [`AgenticOptions`](#agenticoptions) -Defined in: [src/runtime/run-benchmark.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L38) +Defined in: [src/runtime/run-benchmark.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L39) The worker: model + router + (optional) the critic's instruction (the steerer knob). @@ -6136,7 +6136,7 @@ The worker: model + router + (optional) the critic's instruction (the steerer kn > `optional` **strategies?**: [`Strategy`](#strategy-3)\<[`StrategyResult`](#strategyresult-1)\>[] -Defined in: [src/runtime/run-benchmark.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L41) +Defined in: [src/runtime/run-benchmark.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L42) Which strategies to compare. Pass the built-ins (`refine`, `sample`) or your own. Default: [sample, refine]. @@ -6145,7 +6145,7 @@ Which strategies to compare. Pass the built-ins (`refine`, `sample`) or your own > `optional` **budget?**: `number` -Defined in: [src/runtime/run-benchmark.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L43) +Defined in: [src/runtime/run-benchmark.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L44) Shots (refine) / width (sample) — the equal compute budget per strategy. Default 3. @@ -6153,7 +6153,7 @@ Shots (refine) / width (sample) — the equal compute budget per strategy. Defau > `optional` **concurrency?**: `number` -Defined in: [src/runtime/run-benchmark.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L45) +Defined in: [src/runtime/run-benchmark.ts:46](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L46) Tasks scored in parallel. Default 3. @@ -6161,7 +6161,7 @@ Tasks scored in parallel. Default 3. > `optional` **onTask?**: (`row`, `done`, `total`) => `void` -Defined in: [src/runtime/run-benchmark.ts:48](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L48) +Defined in: [src/runtime/run-benchmark.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L49) Progress hook — fires as each task settles (the live-monitoring seam: append to a progress file, render a tree, stream to a dashboard). `done` counts settled tasks. @@ -6188,16 +6188,28 @@ Progress hook — fires as each task settles (the live-monitoring seam: append t > `optional` **hooks?**: [`RuntimeHooks`](index.md#runtimehooks) -Defined in: [src/runtime/run-benchmark.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L51) +Defined in: [src/runtime/run-benchmark.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L52) Lifecycle observability — every spawn/settle of every cell's shots/analysts streams here live (the watchdog/route-auditor seam, passed through to `runAgentic`). +##### modelPreflight? + +> `optional` **modelPreflight?**: `false` \| ((`model`, `worker`) => `Promise`\<`void`\>) + +Defined in: [src/runtime/run-benchmark.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L60) + +Model availability check before tasks start. + +By default, live router workers send one one-token request per unique worker and analyst +model. Injected `worker.complete` transports skip the check. Pass `false` to disable it or a +callback to check each unique model through a custom transport. + *** ### BenchmarkLift -Defined in: [src/runtime/run-benchmark.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L54) +Defined in: [src/runtime/run-benchmark.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L63) #### Properties @@ -6205,7 +6217,7 @@ Defined in: [src/runtime/run-benchmark.ts:54](https://github.com/tangle-network/ > **mean**: `number` -Defined in: [src/runtime/run-benchmark.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L56) +Defined in: [src/runtime/run-benchmark.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L65) Mean of paired deltas (refine − sample). @@ -6213,25 +6225,25 @@ Mean of paired deltas (refine − sample). > **low**: `number` -Defined in: [src/runtime/run-benchmark.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L57) +Defined in: [src/runtime/run-benchmark.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L66) ##### high > **high**: `number` -Defined in: [src/runtime/run-benchmark.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L58) +Defined in: [src/runtime/run-benchmark.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L67) ##### n > **n**: `number` -Defined in: [src/runtime/run-benchmark.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L59) +Defined in: [src/runtime/run-benchmark.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L68) *** ### BenchmarkCell -Defined in: [src/runtime/run-benchmark.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L63) +Defined in: [src/runtime/run-benchmark.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L72) One strategy's outcome on one task — the per-task cell an optimizer consumes. @@ -6241,19 +6253,19 @@ One strategy's outcome on one task — the per-task cell an optimizer consumes. > **score**: `number` -Defined in: [src/runtime/run-benchmark.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L64) +Defined in: [src/runtime/run-benchmark.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L73) ##### resolved > **resolved**: `boolean` -Defined in: [src/runtime/run-benchmark.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L65) +Defined in: [src/runtime/run-benchmark.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L74) ##### progression > **progression**: `number`[] -Defined in: [src/runtime/run-benchmark.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L67) +Defined in: [src/runtime/run-benchmark.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L76) The progress curve (refine: score per shot; sample: best-so-far per rollout). @@ -6261,19 +6273,19 @@ The progress curve (refine: score per shot; sample: best-so-far per rollout). > **usd**: `number` -Defined in: [src/runtime/run-benchmark.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L68) +Defined in: [src/runtime/run-benchmark.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L77) ##### ms > **ms**: `number` -Defined in: [src/runtime/run-benchmark.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L69) +Defined in: [src/runtime/run-benchmark.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L78) ##### tokens > **tokens**: `object` -Defined in: [src/runtime/run-benchmark.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L70) +Defined in: [src/runtime/run-benchmark.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L79) ###### input @@ -6287,7 +6299,7 @@ Defined in: [src/runtime/run-benchmark.ts:70](https://github.com/tangle-network/ ### BenchmarkTaskRow -Defined in: [src/runtime/run-benchmark.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L73) +Defined in: [src/runtime/run-benchmark.ts:82](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L82) #### Properties @@ -6295,13 +6307,13 @@ Defined in: [src/runtime/run-benchmark.ts:73](https://github.com/tangle-network/ > **taskId**: `string` -Defined in: [src/runtime/run-benchmark.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L74) +Defined in: [src/runtime/run-benchmark.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L83) ##### cells? > `optional` **cells?**: `Record`\<`string`, [`BenchmarkCell`](#benchmarkcell)\> -Defined in: [src/runtime/run-benchmark.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L76) +Defined in: [src/runtime/run-benchmark.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L85) Per-strategy cells; absent when the task errored before completing all strategies. @@ -6309,7 +6321,7 @@ Per-strategy cells; absent when the task errored before completing all strategie > `optional` **errors?**: `Record`\<`string`, `string`\> -Defined in: [src/runtime/run-benchmark.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L80) +Defined in: [src/runtime/run-benchmark.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L89) Per-strategy failures on this task: the strategy competed, threw, and scored an honest zero — it loses, it does not poison the row. The message is kept so a later @@ -6319,7 +6331,7 @@ Per-strategy failures on this task: the strategy competed, threw, and scored an > `optional` **error?**: `string` -Defined in: [src/runtime/run-benchmark.ts:82](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L82) +Defined in: [src/runtime/run-benchmark.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L91) Why the task was excluded (infra/setup failure) — never silently dropped. @@ -6327,7 +6339,7 @@ Why the task was excluded (infra/setup failure) — never silently dropped. ### BenchmarkStrategySummary -Defined in: [src/runtime/run-benchmark.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L85) +Defined in: [src/runtime/run-benchmark.ts:94](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L94) #### Properties @@ -6335,7 +6347,7 @@ Defined in: [src/runtime/run-benchmark.ts:85](https://github.com/tangle-network/ > **score**: `number` -Defined in: [src/runtime/run-benchmark.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L87) +Defined in: [src/runtime/run-benchmark.ts:96](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L96) Mean verifier score (0..1). @@ -6343,7 +6355,7 @@ Mean verifier score (0..1). > **resolved**: `number` -Defined in: [src/runtime/run-benchmark.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L89) +Defined in: [src/runtime/run-benchmark.ts:98](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L98) Fraction of tasks fully resolved. @@ -6351,7 +6363,7 @@ Fraction of tasks fully resolved. > **usd**: `number` -Defined in: [src/runtime/run-benchmark.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L91) +Defined in: [src/runtime/run-benchmark.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L100) Mean cost vector per task. @@ -6359,13 +6371,13 @@ Mean cost vector per task. > **ms**: `number` -Defined in: [src/runtime/run-benchmark.ts:92](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L92) +Defined in: [src/runtime/run-benchmark.ts:101](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L101) *** ### BenchmarkReport -Defined in: [src/runtime/run-benchmark.ts:96](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L96) +Defined in: [src/runtime/run-benchmark.ts:105](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L105) Benchmark output: per-strategy means plus the full per-task × per-strategy losses table an optimizer mines. @@ -6375,19 +6387,19 @@ Benchmark output: per-strategy means plus the full per-task × per-strategy loss > **n**: `number` -Defined in: [src/runtime/run-benchmark.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L97) +Defined in: [src/runtime/run-benchmark.ts:106](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L106) ##### excluded > **excluded**: `number` -Defined in: [src/runtime/run-benchmark.ts:98](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L98) +Defined in: [src/runtime/run-benchmark.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L107) ##### perStrategy > **perStrategy**: `Record`\<`string`, [`BenchmarkStrategySummary`](#benchmarkstrategysummary)\> -Defined in: [src/runtime/run-benchmark.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L100) +Defined in: [src/runtime/run-benchmark.ts:109](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L109) Per-strategy means (keyed by strategy.name). @@ -6395,7 +6407,7 @@ Per-strategy means (keyed by strategy.name). > **perTask**: [`BenchmarkTaskRow`](#benchmarktaskrow)[] -Defined in: [src/runtime/run-benchmark.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L103) +Defined in: [src/runtime/run-benchmark.ts:112](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L112) The full per-task × per-strategy table — the LOSSES an optimizer (GEPA, a strategy-author, an operator) consumes. Includes errored tasks with the reason. @@ -6404,7 +6416,7 @@ The full per-task × per-strategy table — the LOSSES an optimizer (GEPA, a > **pareto**: `string`[] -Defined in: [src/runtime/run-benchmark.ts:106](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L106) +Defined in: [src/runtime/run-benchmark.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L115) The non-dominated strategies on (score ↑, $/task ↓) — collapse-last, per the canon: a strategy that ties on score at half the cost WINS and a scalar would hide it. @@ -6413,7 +6425,7 @@ The non-dominated strategies on (score ↑, $/task ↓) — collapse-last, per t > `optional` **refineVsSample?**: [`BenchmarkLift`](#benchmarklift) -Defined in: [src/runtime/run-benchmark.ts:108](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L108) +Defined in: [src/runtime/run-benchmark.ts:117](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L117) The headline when both `refine` and `sample` ran: paired-bootstrap lift of refine over sample. @@ -18124,7 +18136,7 @@ Defined in: [src/runtime/personify/wave-types.ts:619](https://github.com/tangle- > **Environment** = [`AgenticSurface`](#agenticsurface) -Defined in: [src/runtime/run-benchmark.ts:30](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L30) +Defined in: [src/runtime/run-benchmark.ts:31](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L31) A checkable task domain — implement these 5 hooks and the suite does the rest. The same seam as `AgenticSurface`; `Environment` is the RL/gym-standard name for it. @@ -21168,7 +21180,7 @@ The turnkey production brain — tests script a mock `ToolLoopChat`; production > **runBenchmark**(`cfg`): `Promise`\<[`BenchmarkReport`](#benchmarkreport)\> -Defined in: [src/runtime/run-benchmark.ts:133](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L133) +Defined in: [src/runtime/run-benchmark.ts:173](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L173) Run the requested strategies over the tasks, scored by the Environment's own check. Resilient: a task whose rollouts fail (transient infra) is excluded from the stats but @@ -21190,7 +21202,7 @@ Run the requested strategies over the tasks, scored by the Environment's own che > **printBenchmarkReport**(`report`): `void` -Defined in: [src/runtime/run-benchmark.ts:232](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L232) +Defined in: [src/runtime/run-benchmark.ts:274](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L274) Pretty-print a report — the "free optimization" verdict, with the cost vector. From c6737d49004369df4d359b6309a7d719b1b50d83 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 24 Jul 2026 19:37:03 -0600 Subject: [PATCH 3/4] fix(bench): reuse model checks across evolution phases --- docs/api/runtime.md | 163 +++++++++++++------------ src/runtime/run-benchmark.test.ts | 2 + src/runtime/run-benchmark.ts | 2 +- src/runtime/strategy-evolution.ts | 14 ++- tests/loops/strategy-evolution.test.ts | 5 + 5 files changed, 108 insertions(+), 78 deletions(-) diff --git a/docs/api/runtime.md b/docs/api/runtime.md index 6e3b10bb..bb295299 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -7849,7 +7849,7 @@ Defined in: [src/runtime/strategy-author.ts:141](https://github.com/tangle-netwo ### EvolutionAuthor -Defined in: [src/runtime/strategy-evolution.ts:46](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L46) +Defined in: [src/runtime/strategy-evolution.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L47) #### Properties @@ -7857,7 +7857,7 @@ Defined in: [src/runtime/strategy-evolution.ts:46](https://github.com/tangle-net > **chat**: `ChatClient` -Defined in: [src/runtime/strategy-evolution.ts:48](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L48) +Defined in: [src/runtime/strategy-evolution.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L49) The model-call seam (agent-eval `createChatClient`). @@ -7865,31 +7865,31 @@ The model-call seam (agent-eval `createChatClient`). > `optional` **model?**: `string` -Defined in: [src/runtime/strategy-evolution.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L49) +Defined in: [src/runtime/strategy-evolution.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L50) ##### fallbackModel? > `optional` **fallbackModel?**: `string` -Defined in: [src/runtime/strategy-evolution.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L50) +Defined in: [src/runtime/strategy-evolution.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L51) ##### temperature? > `optional` **temperature?**: `number` -Defined in: [src/runtime/strategy-evolution.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L51) +Defined in: [src/runtime/strategy-evolution.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L52) ##### maxTokens? > `optional` **maxTokens?**: `number` -Defined in: [src/runtime/strategy-evolution.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L52) +Defined in: [src/runtime/strategy-evolution.ts:53](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L53) *** ### StrategyEvolutionConfig -Defined in: [src/runtime/strategy-evolution.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L57) +Defined in: [src/runtime/strategy-evolution.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L58) #### Properties @@ -7897,13 +7897,13 @@ Defined in: [src/runtime/strategy-evolution.ts:57](https://github.com/tangle-net > **environment**: [`AgenticSurface`](#agenticsurface) -Defined in: [src/runtime/strategy-evolution.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L58) +Defined in: [src/runtime/strategy-evolution.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L59) ##### tasks > **tasks**: (`offset`, `n`) => `Promise`\<[`AgenticTask`](#agentictask)[]\> -Defined in: [src/runtime/strategy-evolution.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L62) +Defined in: [src/runtime/strategy-evolution.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L63) Task supply by DISJOINT slice: `(offset, n)` must return n tasks unique to that offset range. Train draws [0, trainN); the holdout draws [trainN + holdoutOffset, @@ -7927,19 +7927,19 @@ Task supply by DISJOINT slice: `(offset, n)` must return n tasks unique to that > **trainN**: `number` -Defined in: [src/runtime/strategy-evolution.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L63) +Defined in: [src/runtime/strategy-evolution.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L64) ##### holdoutN > **holdoutN**: `number` -Defined in: [src/runtime/strategy-evolution.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L64) +Defined in: [src/runtime/strategy-evolution.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L65) ##### holdoutOffset? > `optional` **holdoutOffset?**: `number` -Defined in: [src/runtime/strategy-evolution.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L66) +Defined in: [src/runtime/strategy-evolution.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L67) Extra offset past the train slice for the holdout draw (rotate across runs). @@ -7947,19 +7947,30 @@ Extra offset past the train slice for the holdout draw (rotate across runs). > **worker**: [`AgenticOptions`](#agenticoptions) -Defined in: [src/runtime/strategy-evolution.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L67) +Defined in: [src/runtime/strategy-evolution.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L68) + +##### modelPreflight? + +> `optional` **modelPreflight?**: `false` \| ((`model`, `worker`) => `Promise`\<`void`\>) + +Defined in: [src/runtime/strategy-evolution.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L75) + +Model availability check before the first benchmark phase. + +A successful check is reused for the remaining phases in this evolution run. +See `BenchmarkConfig.modelPreflight`. ##### author > **author**: [`EvolutionAuthor`](#evolutionauthor) -Defined in: [src/runtime/strategy-evolution.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L68) +Defined in: [src/runtime/strategy-evolution.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L76) ##### budget? > `optional` **budget?**: `number` -Defined in: [src/runtime/strategy-evolution.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L70) +Defined in: [src/runtime/strategy-evolution.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L78) Rollouts (sample) / shots (refine) per strategy per task. Default 3. @@ -7967,13 +7978,13 @@ Rollouts (sample) / shots (refine) per strategy per task. Default 3. > `optional` **concurrency?**: `number` -Defined in: [src/runtime/strategy-evolution.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L71) +Defined in: [src/runtime/strategy-evolution.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L79) ##### generations? > `optional` **generations?**: `number` -Defined in: [src/runtime/strategy-evolution.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L73) +Defined in: [src/runtime/strategy-evolution.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L81) Author→tournament rounds after gen0. Default 2. @@ -7981,7 +7992,7 @@ Author→tournament rounds after gen0. Default 2. > `optional` **populationSize?**: `number` -Defined in: [src/runtime/strategy-evolution.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L75) +Defined in: [src/runtime/strategy-evolution.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L83) Authored candidates per generation. Default 2. @@ -7989,7 +8000,7 @@ Authored candidates per generation. Default 2. > `optional` **baselines?**: [`Strategy`](#strategy-3)\<[`StrategyResult`](#strategyresult-1)\>[] -Defined in: [src/runtime/strategy-evolution.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L77) +Defined in: [src/runtime/strategy-evolution.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L85) The gen0 field. Default [sample, refine, sampleThenRefine]. @@ -7997,7 +8008,7 @@ The gen0 field. Default [sample, refine, sampleThenRefine]. > `optional` **objective?**: `"score"` \| `"cost"` -Defined in: [src/runtime/strategy-evolution.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L83) +Defined in: [src/runtime/strategy-evolution.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L91) What "better" means for PROMOTION. 'score' (default): the candidate must beat the incumbent's score (superiority gate). 'cost': the candidate must prove score @@ -8009,7 +8020,7 @@ What "better" means for PROMOTION. 'score' (default): the candidate must beat th > `optional` **scoreTolerance?**: `number` -Defined in: [src/runtime/strategy-evolution.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L85) +Defined in: [src/runtime/strategy-evolution.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L93) Cost objective: the score CI lower bound must clear −scoreTolerance. Default 0.05. @@ -8017,7 +8028,7 @@ Cost objective: the score CI lower bound must clear −scoreTolerance. Default 0 > `optional` **champion?**: [`ChampionPolicy`](#championpolicy) -Defined in: [src/runtime/strategy-evolution.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L87) +Defined in: [src/runtime/strategy-evolution.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L95) Search-side champion selection. Default 'costAware'. @@ -8025,7 +8036,7 @@ Search-side champion selection. Default 'costAware'. > `optional` **championEpsilon?**: `number` -Defined in: [src/runtime/strategy-evolution.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L89) +Defined in: [src/runtime/strategy-evolution.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L97) Score band treated as a tie under 'costAware'. Default 0.01. @@ -8033,7 +8044,7 @@ Score band treated as a tie under 'costAware'. Default 0.01. > **outDir**: `string` -Defined in: [src/runtime/strategy-evolution.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L91) +Defined in: [src/runtime/strategy-evolution.ts:99](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L99) Where authored modules are written. @@ -8041,7 +8052,7 @@ Where authored modules are written. > `optional` **minPairedTasks?**: `number` -Defined in: [src/runtime/strategy-evolution.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L93) +Defined in: [src/runtime/strategy-evolution.ts:101](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L101) Promotion-gate evidence floor (paired holdout tasks). @@ -8049,7 +8060,7 @@ Promotion-gate evidence floor (paired holdout tasks). > `optional` **band?**: `object` -Defined in: [src/runtime/strategy-evolution.ts:102](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L102) +Defined in: [src/runtime/strategy-evolution.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L110) BAND-AWARE scoring — concentrate the measurement where lift is possible. Holdout: draw `holdoutPoolN` candidate tasks and run `baselines[0]` once at the run @@ -8075,7 +8086,7 @@ Keep holdout tasks where the reference scores ≤ this. Default 0.99 — drop on > `optional` **lossesDetail?**: `"exact"` \| `"binary"` -Defined in: [src/runtime/strategy-evolution.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L111) +Defined in: [src/runtime/strategy-evolution.ts:119](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L119) What the author learns from a tournament. 'exact' (default) = scores + progressions per task; 'binary' = pass/fail only — the leakage-bounded channel (one bit per cell @@ -8085,7 +8096,7 @@ What the author learns from a tournament. 'exact' (default) = scores + progressi > `optional` **reproducerCheck?**: `object` -Defined in: [src/runtime/strategy-evolution.ts:118](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L118) +Defined in: [src/runtime/strategy-evolution.ts:126](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L126) Reproducer certification (arXiv:2606.11045): when the final champion is AUTHORED, compress it to a short natural-language summary, have a fresh author re-implement @@ -8111,7 +8122,7 @@ Reproduction counts as faithful when reproducedScore ≥ championScore − toler > `optional` **checkpoint?**: `object` -Defined in: [src/runtime/strategy-evolution.ts:128](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L128) +Defined in: [src/runtime/strategy-evolution.ts:136](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L136) Endurance: write the run state after every completed phase; with `resume`, a restart skips completed phases (authored modules re-imported from their files). @@ -8129,7 +8140,7 @@ Endurance: write the run state after every completed phase; with `resume`, a > `optional` **onPhase?**: (`phase`) => `Promise`\<`void`\> -Defined in: [src/runtime/strategy-evolution.ts:135](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L135) +Defined in: [src/runtime/strategy-evolution.ts:143](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L143) Called before each benchmark phase (gen0, gen1…, band-screen, holdout, reproduce). The seam for environment recycling — no artifacts span phases, so a runner may @@ -8149,7 +8160,7 @@ Called before each benchmark phase (gen0, gen1…, band-screen, holdout, reprodu > `optional` **onTask?**: (`phase`, `row`, `done`, `total`) => `void` -Defined in: [src/runtime/strategy-evolution.ts:136](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L136) +Defined in: [src/runtime/strategy-evolution.ts:144](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L144) ###### Parameters @@ -8177,13 +8188,13 @@ Defined in: [src/runtime/strategy-evolution.ts:136](https://github.com/tangle-ne > `optional` **hooks?**: [`RuntimeHooks`](index.md#runtimehooks) -Defined in: [src/runtime/strategy-evolution.ts:137](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L137) +Defined in: [src/runtime/strategy-evolution.ts:145](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L145) *** ### ChampionPick -Defined in: [src/runtime/strategy-evolution.ts:152](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L152) +Defined in: [src/runtime/strategy-evolution.ts:160](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L160) #### Properties @@ -8191,25 +8202,25 @@ Defined in: [src/runtime/strategy-evolution.ts:152](https://github.com/tangle-ne > **name**: `string` -Defined in: [src/runtime/strategy-evolution.ts:153](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L153) +Defined in: [src/runtime/strategy-evolution.ts:161](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L161) ##### score > **score**: `number` -Defined in: [src/runtime/strategy-evolution.ts:154](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L154) +Defined in: [src/runtime/strategy-evolution.ts:162](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L162) ##### usd > **usd**: `number` -Defined in: [src/runtime/strategy-evolution.ts:155](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L155) +Defined in: [src/runtime/strategy-evolution.ts:163](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L163) *** ### EvolutionCandidate -Defined in: [src/runtime/strategy-evolution.ts:158](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L158) +Defined in: [src/runtime/strategy-evolution.ts:166](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L166) #### Properties @@ -8217,31 +8228,31 @@ Defined in: [src/runtime/strategy-evolution.ts:158](https://github.com/tangle-ne > **name**: `string` -Defined in: [src/runtime/strategy-evolution.ts:159](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L159) +Defined in: [src/runtime/strategy-evolution.ts:167](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L167) ##### file? > `optional` **file?**: `string` -Defined in: [src/runtime/strategy-evolution.ts:160](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L160) +Defined in: [src/runtime/strategy-evolution.ts:168](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L168) ##### gzipBits? > `optional` **gzipBits?**: `number` -Defined in: [src/runtime/strategy-evolution.ts:161](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L161) +Defined in: [src/runtime/strategy-evolution.ts:169](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L169) ##### codeChars? > `optional` **codeChars?**: `number` -Defined in: [src/runtime/strategy-evolution.ts:162](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L162) +Defined in: [src/runtime/strategy-evolution.ts:170](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L170) ##### error? > `optional` **error?**: `string` -Defined in: [src/runtime/strategy-evolution.ts:164](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L164) +Defined in: [src/runtime/strategy-evolution.ts:172](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L172) Present when this author attempt failed (recorded, never silent). @@ -8249,7 +8260,7 @@ Present when this author attempt failed (recorded, never silent). ### EvolutionGeneration -Defined in: [src/runtime/strategy-evolution.ts:167](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L167) +Defined in: [src/runtime/strategy-evolution.ts:175](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L175) #### Properties @@ -8257,31 +8268,31 @@ Defined in: [src/runtime/strategy-evolution.ts:167](https://github.com/tangle-ne > **generation**: `number` -Defined in: [src/runtime/strategy-evolution.ts:168](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L168) +Defined in: [src/runtime/strategy-evolution.ts:176](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L176) ##### candidates > **candidates**: [`EvolutionCandidate`](#evolutioncandidate)[] -Defined in: [src/runtime/strategy-evolution.ts:169](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L169) +Defined in: [src/runtime/strategy-evolution.ts:177](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L177) ##### report > **report**: [`BenchmarkReport`](#benchmarkreport) -Defined in: [src/runtime/strategy-evolution.ts:170](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L170) +Defined in: [src/runtime/strategy-evolution.ts:178](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L178) ##### champion > **champion**: [`ChampionPick`](#championpick) -Defined in: [src/runtime/strategy-evolution.ts:171](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L171) +Defined in: [src/runtime/strategy-evolution.ts:179](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L179) *** ### EvolutionArchiveNode -Defined in: [src/runtime/strategy-evolution.ts:174](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L174) +Defined in: [src/runtime/strategy-evolution.ts:182](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L182) #### Properties @@ -8289,25 +8300,25 @@ Defined in: [src/runtime/strategy-evolution.ts:174](https://github.com/tangle-ne > **name**: `string` -Defined in: [src/runtime/strategy-evolution.ts:175](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L175) +Defined in: [src/runtime/strategy-evolution.ts:183](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L183) ##### source > **source**: `"baseline"` \| `"authored"` -Defined in: [src/runtime/strategy-evolution.ts:176](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L176) +Defined in: [src/runtime/strategy-evolution.ts:184](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L184) ##### generation > **generation**: `number` -Defined in: [src/runtime/strategy-evolution.ts:177](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L177) +Defined in: [src/runtime/strategy-evolution.ts:185](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L185) ##### parent? > `optional` **parent?**: `string` -Defined in: [src/runtime/strategy-evolution.ts:179](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L179) +Defined in: [src/runtime/strategy-evolution.ts:187](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L187) The champion whose tournament losses this candidate was authored from. @@ -8315,19 +8326,19 @@ The champion whose tournament losses this candidate was authored from. > `optional` **gzipBits?**: `number` -Defined in: [src/runtime/strategy-evolution.ts:180](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L180) +Defined in: [src/runtime/strategy-evolution.ts:188](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L188) ##### file? > `optional` **file?**: `string` -Defined in: [src/runtime/strategy-evolution.ts:181](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L181) +Defined in: [src/runtime/strategy-evolution.ts:189](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L189) ##### score > **score**: `number` -Defined in: [src/runtime/strategy-evolution.ts:184](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L184) +Defined in: [src/runtime/strategy-evolution.ts:192](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L192) Latest measured tournament result — 0 until the node's first tournament settles (an authored node is created before its generation's benchmark runs). @@ -8336,13 +8347,13 @@ Latest measured tournament result — 0 until the node's first tournament settle > **usd**: `number` -Defined in: [src/runtime/strategy-evolution.ts:185](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L185) +Defined in: [src/runtime/strategy-evolution.ts:193](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L193) *** ### EvolutionBandInfo -Defined in: [src/runtime/strategy-evolution.ts:204](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L204) +Defined in: [src/runtime/strategy-evolution.ts:212](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L212) #### Properties @@ -8350,7 +8361,7 @@ Defined in: [src/runtime/strategy-evolution.ts:204](https://github.com/tangle-ne > **screened**: `number` -Defined in: [src/runtime/strategy-evolution.ts:206](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L206) +Defined in: [src/runtime/strategy-evolution.ts:214](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L214) Tasks screened by the reference on the holdout pool. @@ -8358,7 +8369,7 @@ Tasks screened by the reference on the holdout pool. > **inBand**: `number` -Defined in: [src/runtime/strategy-evolution.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L208) +Defined in: [src/runtime/strategy-evolution.ts:216](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L216) Tasks kept (reference score ≤ maxRefScore) before truncating to holdoutN. @@ -8366,7 +8377,7 @@ Tasks kept (reference score ≤ maxRefScore) before truncating to holdoutN. > **refScores**: `object`[] -Defined in: [src/runtime/strategy-evolution.ts:210](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L210) +Defined in: [src/runtime/strategy-evolution.ts:218](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L218) Reference scores per screened task (the screening record). @@ -8382,7 +8393,7 @@ Reference scores per screened task (the screening record). ### EvolutionReport -Defined in: [src/runtime/strategy-evolution.ts:213](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L213) +Defined in: [src/runtime/strategy-evolution.ts:221](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L221) #### Properties @@ -8390,49 +8401,49 @@ Defined in: [src/runtime/strategy-evolution.ts:213](https://github.com/tangle-ne > **gen0**: [`BenchmarkReport`](#benchmarkreport) -Defined in: [src/runtime/strategy-evolution.ts:214](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L214) +Defined in: [src/runtime/strategy-evolution.ts:222](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L222) ##### gen0Champion > **gen0Champion**: [`ChampionPick`](#championpick) -Defined in: [src/runtime/strategy-evolution.ts:215](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L215) +Defined in: [src/runtime/strategy-evolution.ts:223](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L223) ##### generations > **generations**: [`EvolutionGeneration`](#evolutiongeneration)[] -Defined in: [src/runtime/strategy-evolution.ts:216](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L216) +Defined in: [src/runtime/strategy-evolution.ts:224](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L224) ##### archive > **archive**: [`EvolutionArchiveNode`](#evolutionarchivenode)[] -Defined in: [src/runtime/strategy-evolution.ts:217](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L217) +Defined in: [src/runtime/strategy-evolution.ts:225](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L225) ##### finalChampion > **finalChampion**: [`ChampionPick`](#championpick) -Defined in: [src/runtime/strategy-evolution.ts:218](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L218) +Defined in: [src/runtime/strategy-evolution.ts:226](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L226) ##### holdout > **holdout**: [`BenchmarkReport`](#benchmarkreport) -Defined in: [src/runtime/strategy-evolution.ts:219](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L219) +Defined in: [src/runtime/strategy-evolution.ts:227](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L227) ##### verdict > **verdict**: [`PromotionVerdict`](#promotionverdict) -Defined in: [src/runtime/strategy-evolution.ts:220](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L220) +Defined in: [src/runtime/strategy-evolution.ts:228](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L228) ##### band? > `optional` **band?**: [`EvolutionBandInfo`](#evolutionbandinfo) -Defined in: [src/runtime/strategy-evolution.ts:223](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L223) +Defined in: [src/runtime/strategy-evolution.ts:231](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L231) Present when band screening ran — the verdict's estimand is then "paired lift on headroom tasks" (band membership fixed by the reference screen, pre-registered). @@ -8441,7 +8452,7 @@ Present when band screening ran — the verdict's estimand is then "paired lift > `optional` **reproduction?**: `ReproductionCheck` -Defined in: [src/runtime/strategy-evolution.ts:225](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L225) +Defined in: [src/runtime/strategy-evolution.ts:233](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L233) Present when reproducerCheck ran (final champion was authored). @@ -8449,7 +8460,7 @@ Present when reproducerCheck ran (final champion was authored). > **trajectory**: `object`[] -Defined in: [src/runtime/strategy-evolution.ts:230](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L230) +Defined in: [src/runtime/strategy-evolution.ts:238](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L238) SEARCH TELEMETRY, not evidence: each entry is that generation's own train-slice re-measurement, so cross-generation deltas mix true drift with run-to-run variance @@ -18260,7 +18271,7 @@ fixtures, feature names) and replace only the instruction. > **ChampionPolicy** = `"score"` \| `"costAware"` -Defined in: [src/runtime/strategy-evolution.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L55) +Defined in: [src/runtime/strategy-evolution.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L56) *** @@ -21801,7 +21812,7 @@ Author + load a strategy from losses. Throws when the author emits no loadable m > **discriminatingMeans**(`report`, `fieldOrder`): `Record`\<`string`, \{ `score`: `number`; `usd`: `number`; \}\> \| `null` -Defined in: [src/runtime/strategy-evolution.ts:237](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L237) +Defined in: [src/runtime/strategy-evolution.ts:245](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L245) Strategy means recomputed over the DISCRIMINATING tasks only — tasks where the field strategies did not all score identically. Zero-spread tasks (everyone 1.0, everyone @@ -21828,7 +21839,7 @@ Strategy means recomputed over the DISCRIMINATING tasks only — tasks where the > **pickChampion**(`means`, `fieldOrder`, `policy`, `epsilon`): [`ChampionPick`](#championpick) -Defined in: [src/runtime/strategy-evolution.ts:262](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L262) +Defined in: [src/runtime/strategy-evolution.ts:270](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L270) The champion pick over a means table. 'score' takes the best mean score (ties → field order). 'costAware' treats scores within `epsilon` of the best as tied and @@ -21862,7 +21873,7 @@ The champion pick over a means table. 'score' takes the best mean score (ties > **selectChampion**(`report`, `fieldOrder`, `policy`, `epsilon`): [`ChampionPick`](#championpick) -Defined in: [src/runtime/strategy-evolution.ts:285](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L285) +Defined in: [src/runtime/strategy-evolution.ts:293](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L293) Search-side champion selection over a tournament report. @@ -21894,7 +21905,7 @@ Search-side champion selection over a tournament report. > **runStrategyEvolution**(`cfg`): `Promise`\<[`EvolutionReport`](#evolutionreport)\> -Defined in: [src/runtime/strategy-evolution.ts:365](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L365) +Defined in: [src/runtime/strategy-evolution.ts:373](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L373) Multi-generation strategy search: author candidates from tournament losses, play them against the incumbent at equal budget, promote via `promotionGate` on an untouched holdout slice. diff --git a/src/runtime/run-benchmark.test.ts b/src/runtime/run-benchmark.test.ts index 8ae5cc54..34244764 100644 --- a/src/runtime/run-benchmark.test.ts +++ b/src/runtime/run-benchmark.test.ts @@ -12,6 +12,7 @@ interface ChatRequest { messages?: Array<{ role?: string; content?: string }> temperature?: number max_tokens?: number + reasoning_effort?: string } const task: AgenticTask = { @@ -109,6 +110,7 @@ describe('runBenchmark model availability', () => { const checks = requests.filter(isModelCheck) expect(checks.map((request) => request.model).sort()).toEqual(['analyst-model', 'worker-model']) expect(checks.every((request) => request.max_tokens === 1)).toBe(true) + expect(checks.every((request) => request.reasoning_effort === 'none')).toBe(true) expect(events.indexOf('open')).toBe(2) }) diff --git a/src/runtime/run-benchmark.ts b/src/runtime/run-benchmark.ts index ff5dee1f..d3fbe853 100644 --- a/src/runtime/run-benchmark.ts +++ b/src/runtime/run-benchmark.ts @@ -151,7 +151,7 @@ async function preflightModels(cfg: BenchmarkConfig): Promise { model, }, [{ role: 'user', content: 'Reply OK.' }], - { maxTokens: 1 }, + { maxTokens: 1, reasoningEffort: 'none' }, ) }) diff --git a/src/runtime/strategy-evolution.ts b/src/runtime/strategy-evolution.ts index ae000929..b535181e 100644 --- a/src/runtime/strategy-evolution.ts +++ b/src/runtime/strategy-evolution.ts @@ -28,6 +28,7 @@ import type { ChatClient } from '@tangle-network/agent-eval' import type { RuntimeHooks } from '../runtime-hooks' import { type PromotionVerdict, promotionGate } from './promotion-gate' import { + type BenchmarkConfig, type BenchmarkReport, type BenchmarkTaskRow, type Environment, @@ -65,6 +66,13 @@ export interface StrategyEvolutionConfig { /** Extra offset past the train slice for the holdout draw (rotate across runs). */ holdoutOffset?: number worker: AgenticOptions + /** + * Model availability check before the first benchmark phase. + * + * A successful check is reused for the remaining phases in this evolution run. + * See `BenchmarkConfig.modelPreflight`. + */ + modelPreflight?: BenchmarkConfig['modelPreflight'] author: EvolutionAuthor /** Rollouts (sample) / shots (refine) per strategy per task. Default 3. */ budget?: number @@ -404,20 +412,24 @@ export async function runStrategyEvolution(cfg: StrategyEvolutionConfig): Promis writeFileSync(cfg.checkpoint.path, JSON.stringify({ ...state, fingerprint }, null, 1)) } + let modelsPreflighted = false const bench = async (phase: string, tasks: AgenticTask[], strategies: Strategy[]) => { await cfg.onPhase?.(phase) - return runBenchmark({ + const report = await runBenchmark({ environment: cfg.environment, tasks, worker: cfg.worker, strategies, budget, concurrency, + modelPreflight: modelsPreflighted ? false : cfg.modelPreflight, ...(cfg.onTask ? { onTask: (row, done, total) => cfg.onTask?.(phase, row, done, total) } : {}), ...(cfg.hooks ? { hooks: cfg.hooks } : {}), }) + modelsPreflighted = true + return report } const train = await cfg.tasks(0, cfg.trainN) diff --git a/tests/loops/strategy-evolution.test.ts b/tests/loops/strategy-evolution.test.ts index 806c4d36..93487104 100644 --- a/tests/loops/strategy-evolution.test.ts +++ b/tests/loops/strategy-evolution.test.ts @@ -162,12 +162,16 @@ describe('runStrategyEvolution', () => { stubWorkerRouter() const { chat } = scriptedChat([fenced(twoShotDepthModule)]) const sliceCalls: Array<{ offset: number; n: number }> = [] + const checkedModels: string[] = [] const report = await runStrategyEvolution({ environment: shotCountingSurface(), tasks: sliceTasks(sliceCalls), trainN: 8, holdoutN: 8, worker, + modelPreflight: async (model) => { + checkedModels.push(model) + }, author: { chat, model: 'author-model' }, budget: 3, concurrency: 2, @@ -178,6 +182,7 @@ describe('runStrategyEvolution', () => { }) expect(report.gen0Champion.name).toBe('sample') + expect(checkedModels).toEqual(['test-model']) expect(report.finalChampion.name).toBe('two-shot-depth') expect(report.verdict.promoted).toBe(true) expect(report.verdict.reason).toBe('significant') From d73908f73801ab0e79f121295288ee00f99972bb Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 24 Jul 2026 19:41:32 -0600 Subject: [PATCH 4/4] fix(bench): bound and aggregate model checks --- docs/api/runtime.md | 206 ++++++++++++++++-------------- src/runtime/run-benchmark.test.ts | 43 +++++++ src/runtime/run-benchmark.ts | 49 ++++++- src/runtime/strategy-evolution.ts | 3 + 4 files changed, 199 insertions(+), 102 deletions(-) diff --git a/docs/api/runtime.md b/docs/api/runtime.md index bb295299..f5a9415d 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -6195,7 +6195,7 @@ Lifecycle observability — every spawn/settle of every cell's shots/analysts st ##### modelPreflight? -> `optional` **modelPreflight?**: `false` \| ((`model`, `worker`) => `Promise`\<`void`\>) +> `optional` **modelPreflight?**: `false` \| ((`model`, `worker`, `signal`) => `Promise`\<`void`\>) Defined in: [src/runtime/run-benchmark.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L60) @@ -6205,11 +6205,19 @@ By default, live router workers send one one-token request per unique worker and model. Injected `worker.complete` transports skip the check. Pass `false` to disable it or a callback to check each unique model through a custom transport. +##### modelPreflightTimeoutMs? + +> `optional` **modelPreflightTimeoutMs?**: `number` + +Defined in: [src/runtime/run-benchmark.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L64) + +Maximum time for each model availability check. Default 30 seconds. + *** ### BenchmarkLift -Defined in: [src/runtime/run-benchmark.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L63) +Defined in: [src/runtime/run-benchmark.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L67) #### Properties @@ -6217,7 +6225,7 @@ Defined in: [src/runtime/run-benchmark.ts:63](https://github.com/tangle-network/ > **mean**: `number` -Defined in: [src/runtime/run-benchmark.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L65) +Defined in: [src/runtime/run-benchmark.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L69) Mean of paired deltas (refine − sample). @@ -6225,25 +6233,25 @@ Mean of paired deltas (refine − sample). > **low**: `number` -Defined in: [src/runtime/run-benchmark.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L66) +Defined in: [src/runtime/run-benchmark.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L70) ##### high > **high**: `number` -Defined in: [src/runtime/run-benchmark.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L67) +Defined in: [src/runtime/run-benchmark.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L71) ##### n > **n**: `number` -Defined in: [src/runtime/run-benchmark.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L68) +Defined in: [src/runtime/run-benchmark.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L72) *** ### BenchmarkCell -Defined in: [src/runtime/run-benchmark.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L72) +Defined in: [src/runtime/run-benchmark.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L76) One strategy's outcome on one task — the per-task cell an optimizer consumes. @@ -6253,19 +6261,19 @@ One strategy's outcome on one task — the per-task cell an optimizer consumes. > **score**: `number` -Defined in: [src/runtime/run-benchmark.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L73) +Defined in: [src/runtime/run-benchmark.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L77) ##### resolved > **resolved**: `boolean` -Defined in: [src/runtime/run-benchmark.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L74) +Defined in: [src/runtime/run-benchmark.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L78) ##### progression > **progression**: `number`[] -Defined in: [src/runtime/run-benchmark.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L76) +Defined in: [src/runtime/run-benchmark.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L80) The progress curve (refine: score per shot; sample: best-so-far per rollout). @@ -6273,19 +6281,19 @@ The progress curve (refine: score per shot; sample: best-so-far per rollout). > **usd**: `number` -Defined in: [src/runtime/run-benchmark.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L77) +Defined in: [src/runtime/run-benchmark.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L81) ##### ms > **ms**: `number` -Defined in: [src/runtime/run-benchmark.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L78) +Defined in: [src/runtime/run-benchmark.ts:82](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L82) ##### tokens > **tokens**: `object` -Defined in: [src/runtime/run-benchmark.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L79) +Defined in: [src/runtime/run-benchmark.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L83) ###### input @@ -6299,7 +6307,7 @@ Defined in: [src/runtime/run-benchmark.ts:79](https://github.com/tangle-network/ ### BenchmarkTaskRow -Defined in: [src/runtime/run-benchmark.ts:82](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L82) +Defined in: [src/runtime/run-benchmark.ts:86](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L86) #### Properties @@ -6307,13 +6315,13 @@ Defined in: [src/runtime/run-benchmark.ts:82](https://github.com/tangle-network/ > **taskId**: `string` -Defined in: [src/runtime/run-benchmark.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L83) +Defined in: [src/runtime/run-benchmark.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L87) ##### cells? > `optional` **cells?**: `Record`\<`string`, [`BenchmarkCell`](#benchmarkcell)\> -Defined in: [src/runtime/run-benchmark.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L85) +Defined in: [src/runtime/run-benchmark.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L89) Per-strategy cells; absent when the task errored before completing all strategies. @@ -6321,7 +6329,7 @@ Per-strategy cells; absent when the task errored before completing all strategie > `optional` **errors?**: `Record`\<`string`, `string`\> -Defined in: [src/runtime/run-benchmark.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L89) +Defined in: [src/runtime/run-benchmark.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L93) Per-strategy failures on this task: the strategy competed, threw, and scored an honest zero — it loses, it does not poison the row. The message is kept so a later @@ -6331,7 +6339,7 @@ Per-strategy failures on this task: the strategy competed, threw, and scored an > `optional` **error?**: `string` -Defined in: [src/runtime/run-benchmark.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L91) +Defined in: [src/runtime/run-benchmark.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L95) Why the task was excluded (infra/setup failure) — never silently dropped. @@ -6339,7 +6347,7 @@ Why the task was excluded (infra/setup failure) — never silently dropped. ### BenchmarkStrategySummary -Defined in: [src/runtime/run-benchmark.ts:94](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L94) +Defined in: [src/runtime/run-benchmark.ts:98](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L98) #### Properties @@ -6347,7 +6355,7 @@ Defined in: [src/runtime/run-benchmark.ts:94](https://github.com/tangle-network/ > **score**: `number` -Defined in: [src/runtime/run-benchmark.ts:96](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L96) +Defined in: [src/runtime/run-benchmark.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L100) Mean verifier score (0..1). @@ -6355,7 +6363,7 @@ Mean verifier score (0..1). > **resolved**: `number` -Defined in: [src/runtime/run-benchmark.ts:98](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L98) +Defined in: [src/runtime/run-benchmark.ts:102](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L102) Fraction of tasks fully resolved. @@ -6363,7 +6371,7 @@ Fraction of tasks fully resolved. > **usd**: `number` -Defined in: [src/runtime/run-benchmark.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L100) +Defined in: [src/runtime/run-benchmark.ts:104](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L104) Mean cost vector per task. @@ -6371,13 +6379,13 @@ Mean cost vector per task. > **ms**: `number` -Defined in: [src/runtime/run-benchmark.ts:101](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L101) +Defined in: [src/runtime/run-benchmark.ts:105](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L105) *** ### BenchmarkReport -Defined in: [src/runtime/run-benchmark.ts:105](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L105) +Defined in: [src/runtime/run-benchmark.ts:109](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L109) Benchmark output: per-strategy means plus the full per-task × per-strategy losses table an optimizer mines. @@ -6387,19 +6395,19 @@ Benchmark output: per-strategy means plus the full per-task × per-strategy loss > **n**: `number` -Defined in: [src/runtime/run-benchmark.ts:106](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L106) +Defined in: [src/runtime/run-benchmark.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L110) ##### excluded > **excluded**: `number` -Defined in: [src/runtime/run-benchmark.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L107) +Defined in: [src/runtime/run-benchmark.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L111) ##### perStrategy > **perStrategy**: `Record`\<`string`, [`BenchmarkStrategySummary`](#benchmarkstrategysummary)\> -Defined in: [src/runtime/run-benchmark.ts:109](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L109) +Defined in: [src/runtime/run-benchmark.ts:113](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L113) Per-strategy means (keyed by strategy.name). @@ -6407,7 +6415,7 @@ Per-strategy means (keyed by strategy.name). > **perTask**: [`BenchmarkTaskRow`](#benchmarktaskrow)[] -Defined in: [src/runtime/run-benchmark.ts:112](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L112) +Defined in: [src/runtime/run-benchmark.ts:116](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L116) The full per-task × per-strategy table — the LOSSES an optimizer (GEPA, a strategy-author, an operator) consumes. Includes errored tasks with the reason. @@ -6416,7 +6424,7 @@ The full per-task × per-strategy table — the LOSSES an optimizer (GEPA, a > **pareto**: `string`[] -Defined in: [src/runtime/run-benchmark.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L115) +Defined in: [src/runtime/run-benchmark.ts:119](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L119) The non-dominated strategies on (score ↑, $/task ↓) — collapse-last, per the canon: a strategy that ties on score at half the cost WINS and a scalar would hide it. @@ -6425,7 +6433,7 @@ The non-dominated strategies on (score ↑, $/task ↓) — collapse-last, per t > `optional` **refineVsSample?**: [`BenchmarkLift`](#benchmarklift) -Defined in: [src/runtime/run-benchmark.ts:117](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L117) +Defined in: [src/runtime/run-benchmark.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L121) The headline when both `refine` and `sample` ran: paired-bootstrap lift of refine over sample. @@ -7951,7 +7959,7 @@ Defined in: [src/runtime/strategy-evolution.ts:68](https://github.com/tangle-net ##### modelPreflight? -> `optional` **modelPreflight?**: `false` \| ((`model`, `worker`) => `Promise`\<`void`\>) +> `optional` **modelPreflight?**: `false` \| ((`model`, `worker`, `signal`) => `Promise`\<`void`\>) Defined in: [src/runtime/strategy-evolution.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L75) @@ -7960,17 +7968,25 @@ Model availability check before the first benchmark phase. A successful check is reused for the remaining phases in this evolution run. See `BenchmarkConfig.modelPreflight`. +##### modelPreflightTimeoutMs? + +> `optional` **modelPreflightTimeoutMs?**: `number` + +Defined in: [src/runtime/strategy-evolution.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L77) + +Maximum time for each model availability check. Default 30 seconds. + ##### author > **author**: [`EvolutionAuthor`](#evolutionauthor) -Defined in: [src/runtime/strategy-evolution.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L76) +Defined in: [src/runtime/strategy-evolution.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L78) ##### budget? > `optional` **budget?**: `number` -Defined in: [src/runtime/strategy-evolution.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L78) +Defined in: [src/runtime/strategy-evolution.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L80) Rollouts (sample) / shots (refine) per strategy per task. Default 3. @@ -7978,13 +7994,13 @@ Rollouts (sample) / shots (refine) per strategy per task. Default 3. > `optional` **concurrency?**: `number` -Defined in: [src/runtime/strategy-evolution.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L79) +Defined in: [src/runtime/strategy-evolution.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L81) ##### generations? > `optional` **generations?**: `number` -Defined in: [src/runtime/strategy-evolution.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L81) +Defined in: [src/runtime/strategy-evolution.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L83) Author→tournament rounds after gen0. Default 2. @@ -7992,7 +8008,7 @@ Author→tournament rounds after gen0. Default 2. > `optional` **populationSize?**: `number` -Defined in: [src/runtime/strategy-evolution.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L83) +Defined in: [src/runtime/strategy-evolution.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L85) Authored candidates per generation. Default 2. @@ -8000,7 +8016,7 @@ Authored candidates per generation. Default 2. > `optional` **baselines?**: [`Strategy`](#strategy-3)\<[`StrategyResult`](#strategyresult-1)\>[] -Defined in: [src/runtime/strategy-evolution.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L85) +Defined in: [src/runtime/strategy-evolution.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L87) The gen0 field. Default [sample, refine, sampleThenRefine]. @@ -8008,7 +8024,7 @@ The gen0 field. Default [sample, refine, sampleThenRefine]. > `optional` **objective?**: `"score"` \| `"cost"` -Defined in: [src/runtime/strategy-evolution.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L91) +Defined in: [src/runtime/strategy-evolution.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L93) What "better" means for PROMOTION. 'score' (default): the candidate must beat the incumbent's score (superiority gate). 'cost': the candidate must prove score @@ -8020,7 +8036,7 @@ What "better" means for PROMOTION. 'score' (default): the candidate must beat th > `optional` **scoreTolerance?**: `number` -Defined in: [src/runtime/strategy-evolution.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L93) +Defined in: [src/runtime/strategy-evolution.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L95) Cost objective: the score CI lower bound must clear −scoreTolerance. Default 0.05. @@ -8028,7 +8044,7 @@ Cost objective: the score CI lower bound must clear −scoreTolerance. Default 0 > `optional` **champion?**: [`ChampionPolicy`](#championpolicy) -Defined in: [src/runtime/strategy-evolution.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L95) +Defined in: [src/runtime/strategy-evolution.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L97) Search-side champion selection. Default 'costAware'. @@ -8036,7 +8052,7 @@ Search-side champion selection. Default 'costAware'. > `optional` **championEpsilon?**: `number` -Defined in: [src/runtime/strategy-evolution.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L97) +Defined in: [src/runtime/strategy-evolution.ts:99](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L99) Score band treated as a tie under 'costAware'. Default 0.01. @@ -8044,7 +8060,7 @@ Score band treated as a tie under 'costAware'. Default 0.01. > **outDir**: `string` -Defined in: [src/runtime/strategy-evolution.ts:99](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L99) +Defined in: [src/runtime/strategy-evolution.ts:101](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L101) Where authored modules are written. @@ -8052,7 +8068,7 @@ Where authored modules are written. > `optional` **minPairedTasks?**: `number` -Defined in: [src/runtime/strategy-evolution.ts:101](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L101) +Defined in: [src/runtime/strategy-evolution.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L103) Promotion-gate evidence floor (paired holdout tasks). @@ -8060,7 +8076,7 @@ Promotion-gate evidence floor (paired holdout tasks). > `optional` **band?**: `object` -Defined in: [src/runtime/strategy-evolution.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L110) +Defined in: [src/runtime/strategy-evolution.ts:112](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L112) BAND-AWARE scoring — concentrate the measurement where lift is possible. Holdout: draw `holdoutPoolN` candidate tasks and run `baselines[0]` once at the run @@ -8086,7 +8102,7 @@ Keep holdout tasks where the reference scores ≤ this. Default 0.99 — drop on > `optional` **lossesDetail?**: `"exact"` \| `"binary"` -Defined in: [src/runtime/strategy-evolution.ts:119](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L119) +Defined in: [src/runtime/strategy-evolution.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L121) What the author learns from a tournament. 'exact' (default) = scores + progressions per task; 'binary' = pass/fail only — the leakage-bounded channel (one bit per cell @@ -8096,7 +8112,7 @@ What the author learns from a tournament. 'exact' (default) = scores + progressi > `optional` **reproducerCheck?**: `object` -Defined in: [src/runtime/strategy-evolution.ts:126](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L126) +Defined in: [src/runtime/strategy-evolution.ts:128](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L128) Reproducer certification (arXiv:2606.11045): when the final champion is AUTHORED, compress it to a short natural-language summary, have a fresh author re-implement @@ -8122,7 +8138,7 @@ Reproduction counts as faithful when reproducedScore ≥ championScore − toler > `optional` **checkpoint?**: `object` -Defined in: [src/runtime/strategy-evolution.ts:136](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L136) +Defined in: [src/runtime/strategy-evolution.ts:138](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L138) Endurance: write the run state after every completed phase; with `resume`, a restart skips completed phases (authored modules re-imported from their files). @@ -8140,7 +8156,7 @@ Endurance: write the run state after every completed phase; with `resume`, a > `optional` **onPhase?**: (`phase`) => `Promise`\<`void`\> -Defined in: [src/runtime/strategy-evolution.ts:143](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L143) +Defined in: [src/runtime/strategy-evolution.ts:145](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L145) Called before each benchmark phase (gen0, gen1…, band-screen, holdout, reproduce). The seam for environment recycling — no artifacts span phases, so a runner may @@ -8160,7 +8176,7 @@ Called before each benchmark phase (gen0, gen1…, band-screen, holdout, reprodu > `optional` **onTask?**: (`phase`, `row`, `done`, `total`) => `void` -Defined in: [src/runtime/strategy-evolution.ts:144](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L144) +Defined in: [src/runtime/strategy-evolution.ts:146](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L146) ###### Parameters @@ -8188,13 +8204,13 @@ Defined in: [src/runtime/strategy-evolution.ts:144](https://github.com/tangle-ne > `optional` **hooks?**: [`RuntimeHooks`](index.md#runtimehooks) -Defined in: [src/runtime/strategy-evolution.ts:145](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L145) +Defined in: [src/runtime/strategy-evolution.ts:147](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L147) *** ### ChampionPick -Defined in: [src/runtime/strategy-evolution.ts:160](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L160) +Defined in: [src/runtime/strategy-evolution.ts:162](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L162) #### Properties @@ -8202,25 +8218,25 @@ Defined in: [src/runtime/strategy-evolution.ts:160](https://github.com/tangle-ne > **name**: `string` -Defined in: [src/runtime/strategy-evolution.ts:161](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L161) +Defined in: [src/runtime/strategy-evolution.ts:163](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L163) ##### score > **score**: `number` -Defined in: [src/runtime/strategy-evolution.ts:162](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L162) +Defined in: [src/runtime/strategy-evolution.ts:164](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L164) ##### usd > **usd**: `number` -Defined in: [src/runtime/strategy-evolution.ts:163](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L163) +Defined in: [src/runtime/strategy-evolution.ts:165](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L165) *** ### EvolutionCandidate -Defined in: [src/runtime/strategy-evolution.ts:166](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L166) +Defined in: [src/runtime/strategy-evolution.ts:168](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L168) #### Properties @@ -8228,31 +8244,31 @@ Defined in: [src/runtime/strategy-evolution.ts:166](https://github.com/tangle-ne > **name**: `string` -Defined in: [src/runtime/strategy-evolution.ts:167](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L167) +Defined in: [src/runtime/strategy-evolution.ts:169](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L169) ##### file? > `optional` **file?**: `string` -Defined in: [src/runtime/strategy-evolution.ts:168](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L168) +Defined in: [src/runtime/strategy-evolution.ts:170](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L170) ##### gzipBits? > `optional` **gzipBits?**: `number` -Defined in: [src/runtime/strategy-evolution.ts:169](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L169) +Defined in: [src/runtime/strategy-evolution.ts:171](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L171) ##### codeChars? > `optional` **codeChars?**: `number` -Defined in: [src/runtime/strategy-evolution.ts:170](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L170) +Defined in: [src/runtime/strategy-evolution.ts:172](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L172) ##### error? > `optional` **error?**: `string` -Defined in: [src/runtime/strategy-evolution.ts:172](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L172) +Defined in: [src/runtime/strategy-evolution.ts:174](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L174) Present when this author attempt failed (recorded, never silent). @@ -8260,7 +8276,7 @@ Present when this author attempt failed (recorded, never silent). ### EvolutionGeneration -Defined in: [src/runtime/strategy-evolution.ts:175](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L175) +Defined in: [src/runtime/strategy-evolution.ts:177](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L177) #### Properties @@ -8268,31 +8284,31 @@ Defined in: [src/runtime/strategy-evolution.ts:175](https://github.com/tangle-ne > **generation**: `number` -Defined in: [src/runtime/strategy-evolution.ts:176](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L176) +Defined in: [src/runtime/strategy-evolution.ts:178](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L178) ##### candidates > **candidates**: [`EvolutionCandidate`](#evolutioncandidate)[] -Defined in: [src/runtime/strategy-evolution.ts:177](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L177) +Defined in: [src/runtime/strategy-evolution.ts:179](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L179) ##### report > **report**: [`BenchmarkReport`](#benchmarkreport) -Defined in: [src/runtime/strategy-evolution.ts:178](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L178) +Defined in: [src/runtime/strategy-evolution.ts:180](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L180) ##### champion > **champion**: [`ChampionPick`](#championpick) -Defined in: [src/runtime/strategy-evolution.ts:179](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L179) +Defined in: [src/runtime/strategy-evolution.ts:181](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L181) *** ### EvolutionArchiveNode -Defined in: [src/runtime/strategy-evolution.ts:182](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L182) +Defined in: [src/runtime/strategy-evolution.ts:184](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L184) #### Properties @@ -8300,25 +8316,25 @@ Defined in: [src/runtime/strategy-evolution.ts:182](https://github.com/tangle-ne > **name**: `string` -Defined in: [src/runtime/strategy-evolution.ts:183](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L183) +Defined in: [src/runtime/strategy-evolution.ts:185](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L185) ##### source > **source**: `"baseline"` \| `"authored"` -Defined in: [src/runtime/strategy-evolution.ts:184](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L184) +Defined in: [src/runtime/strategy-evolution.ts:186](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L186) ##### generation > **generation**: `number` -Defined in: [src/runtime/strategy-evolution.ts:185](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L185) +Defined in: [src/runtime/strategy-evolution.ts:187](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L187) ##### parent? > `optional` **parent?**: `string` -Defined in: [src/runtime/strategy-evolution.ts:187](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L187) +Defined in: [src/runtime/strategy-evolution.ts:189](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L189) The champion whose tournament losses this candidate was authored from. @@ -8326,19 +8342,19 @@ The champion whose tournament losses this candidate was authored from. > `optional` **gzipBits?**: `number` -Defined in: [src/runtime/strategy-evolution.ts:188](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L188) +Defined in: [src/runtime/strategy-evolution.ts:190](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L190) ##### file? > `optional` **file?**: `string` -Defined in: [src/runtime/strategy-evolution.ts:189](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L189) +Defined in: [src/runtime/strategy-evolution.ts:191](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L191) ##### score > **score**: `number` -Defined in: [src/runtime/strategy-evolution.ts:192](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L192) +Defined in: [src/runtime/strategy-evolution.ts:194](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L194) Latest measured tournament result — 0 until the node's first tournament settles (an authored node is created before its generation's benchmark runs). @@ -8347,13 +8363,13 @@ Latest measured tournament result — 0 until the node's first tournament settle > **usd**: `number` -Defined in: [src/runtime/strategy-evolution.ts:193](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L193) +Defined in: [src/runtime/strategy-evolution.ts:195](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L195) *** ### EvolutionBandInfo -Defined in: [src/runtime/strategy-evolution.ts:212](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L212) +Defined in: [src/runtime/strategy-evolution.ts:214](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L214) #### Properties @@ -8361,7 +8377,7 @@ Defined in: [src/runtime/strategy-evolution.ts:212](https://github.com/tangle-ne > **screened**: `number` -Defined in: [src/runtime/strategy-evolution.ts:214](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L214) +Defined in: [src/runtime/strategy-evolution.ts:216](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L216) Tasks screened by the reference on the holdout pool. @@ -8369,7 +8385,7 @@ Tasks screened by the reference on the holdout pool. > **inBand**: `number` -Defined in: [src/runtime/strategy-evolution.ts:216](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L216) +Defined in: [src/runtime/strategy-evolution.ts:218](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L218) Tasks kept (reference score ≤ maxRefScore) before truncating to holdoutN. @@ -8377,7 +8393,7 @@ Tasks kept (reference score ≤ maxRefScore) before truncating to holdoutN. > **refScores**: `object`[] -Defined in: [src/runtime/strategy-evolution.ts:218](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L218) +Defined in: [src/runtime/strategy-evolution.ts:220](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L220) Reference scores per screened task (the screening record). @@ -8393,7 +8409,7 @@ Reference scores per screened task (the screening record). ### EvolutionReport -Defined in: [src/runtime/strategy-evolution.ts:221](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L221) +Defined in: [src/runtime/strategy-evolution.ts:223](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L223) #### Properties @@ -8401,49 +8417,49 @@ Defined in: [src/runtime/strategy-evolution.ts:221](https://github.com/tangle-ne > **gen0**: [`BenchmarkReport`](#benchmarkreport) -Defined in: [src/runtime/strategy-evolution.ts:222](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L222) +Defined in: [src/runtime/strategy-evolution.ts:224](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L224) ##### gen0Champion > **gen0Champion**: [`ChampionPick`](#championpick) -Defined in: [src/runtime/strategy-evolution.ts:223](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L223) +Defined in: [src/runtime/strategy-evolution.ts:225](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L225) ##### generations > **generations**: [`EvolutionGeneration`](#evolutiongeneration)[] -Defined in: [src/runtime/strategy-evolution.ts:224](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L224) +Defined in: [src/runtime/strategy-evolution.ts:226](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L226) ##### archive > **archive**: [`EvolutionArchiveNode`](#evolutionarchivenode)[] -Defined in: [src/runtime/strategy-evolution.ts:225](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L225) +Defined in: [src/runtime/strategy-evolution.ts:227](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L227) ##### finalChampion > **finalChampion**: [`ChampionPick`](#championpick) -Defined in: [src/runtime/strategy-evolution.ts:226](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L226) +Defined in: [src/runtime/strategy-evolution.ts:228](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L228) ##### holdout > **holdout**: [`BenchmarkReport`](#benchmarkreport) -Defined in: [src/runtime/strategy-evolution.ts:227](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L227) +Defined in: [src/runtime/strategy-evolution.ts:229](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L229) ##### verdict > **verdict**: [`PromotionVerdict`](#promotionverdict) -Defined in: [src/runtime/strategy-evolution.ts:228](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L228) +Defined in: [src/runtime/strategy-evolution.ts:230](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L230) ##### band? > `optional` **band?**: [`EvolutionBandInfo`](#evolutionbandinfo) -Defined in: [src/runtime/strategy-evolution.ts:231](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L231) +Defined in: [src/runtime/strategy-evolution.ts:233](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L233) Present when band screening ran — the verdict's estimand is then "paired lift on headroom tasks" (band membership fixed by the reference screen, pre-registered). @@ -8452,7 +8468,7 @@ Present when band screening ran — the verdict's estimand is then "paired lift > `optional` **reproduction?**: `ReproductionCheck` -Defined in: [src/runtime/strategy-evolution.ts:233](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L233) +Defined in: [src/runtime/strategy-evolution.ts:235](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L235) Present when reproducerCheck ran (final champion was authored). @@ -8460,7 +8476,7 @@ Present when reproducerCheck ran (final champion was authored). > **trajectory**: `object`[] -Defined in: [src/runtime/strategy-evolution.ts:238](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L238) +Defined in: [src/runtime/strategy-evolution.ts:240](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L240) SEARCH TELEMETRY, not evidence: each entry is that generation's own train-slice re-measurement, so cross-generation deltas mix true drift with run-to-run variance @@ -21191,7 +21207,7 @@ The turnkey production brain — tests script a mock `ToolLoopChat`; production > **runBenchmark**(`cfg`): `Promise`\<[`BenchmarkReport`](#benchmarkreport)\> -Defined in: [src/runtime/run-benchmark.ts:173](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L173) +Defined in: [src/runtime/run-benchmark.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L208) Run the requested strategies over the tasks, scored by the Environment's own check. Resilient: a task whose rollouts fail (transient infra) is excluded from the stats but @@ -21213,7 +21229,7 @@ Run the requested strategies over the tasks, scored by the Environment's own che > **printBenchmarkReport**(`report`): `void` -Defined in: [src/runtime/run-benchmark.ts:274](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L274) +Defined in: [src/runtime/run-benchmark.ts:309](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/run-benchmark.ts#L309) Pretty-print a report — the "free optimization" verdict, with the cost vector. @@ -21812,7 +21828,7 @@ Author + load a strategy from losses. Throws when the author emits no loadable m > **discriminatingMeans**(`report`, `fieldOrder`): `Record`\<`string`, \{ `score`: `number`; `usd`: `number`; \}\> \| `null` -Defined in: [src/runtime/strategy-evolution.ts:245](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L245) +Defined in: [src/runtime/strategy-evolution.ts:247](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L247) Strategy means recomputed over the DISCRIMINATING tasks only — tasks where the field strategies did not all score identically. Zero-spread tasks (everyone 1.0, everyone @@ -21839,7 +21855,7 @@ Strategy means recomputed over the DISCRIMINATING tasks only — tasks where the > **pickChampion**(`means`, `fieldOrder`, `policy`, `epsilon`): [`ChampionPick`](#championpick) -Defined in: [src/runtime/strategy-evolution.ts:270](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L270) +Defined in: [src/runtime/strategy-evolution.ts:272](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L272) The champion pick over a means table. 'score' takes the best mean score (ties → field order). 'costAware' treats scores within `epsilon` of the best as tied and @@ -21873,7 +21889,7 @@ The champion pick over a means table. 'score' takes the best mean score (ties > **selectChampion**(`report`, `fieldOrder`, `policy`, `epsilon`): [`ChampionPick`](#championpick) -Defined in: [src/runtime/strategy-evolution.ts:293](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L293) +Defined in: [src/runtime/strategy-evolution.ts:295](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L295) Search-side champion selection over a tournament report. @@ -21905,7 +21921,7 @@ Search-side champion selection over a tournament report. > **runStrategyEvolution**(`cfg`): `Promise`\<[`EvolutionReport`](#evolutionreport)\> -Defined in: [src/runtime/strategy-evolution.ts:373](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L373) +Defined in: [src/runtime/strategy-evolution.ts:375](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy-evolution.ts#L375) Multi-generation strategy search: author candidates from tournament losses, play them against the incumbent at equal budget, promote via `promotionGate` on an untouched holdout slice. diff --git a/src/runtime/run-benchmark.test.ts b/src/runtime/run-benchmark.test.ts index 34244764..a03cc9e2 100644 --- a/src/runtime/run-benchmark.test.ts +++ b/src/runtime/run-benchmark.test.ts @@ -160,6 +160,49 @@ describe('runBenchmark model availability', () => { expect(onTask).not.toHaveBeenCalled() }) + it('reports every unavailable model in one failure', async () => { + const events: string[] = [] + + await expect( + runBenchmark({ + environment: surface(events), + tasks: [task], + worker: { ...worker, analystModel: 'analyst-model' }, + strategies: [oneShot], + budget: 1, + modelPreflight: async (model) => { + throw new Error(`${model} is unavailable`) + }, + }), + ).rejects.toThrow( + 'Benchmark model "worker-model" preflight failed: worker-model is unavailable; ' + + 'Benchmark model "analyst-model" preflight failed: analyst-model is unavailable', + ) + + expect(events).not.toContain('open') + }) + + it('bounds a model check that never settles', async () => { + const events: string[] = [] + + await expect( + runBenchmark({ + environment: surface(events), + tasks: [task], + worker, + strategies: [oneShot], + budget: 1, + modelPreflightTimeoutMs: 5, + modelPreflight: async (_model, _worker, signal) => + new Promise((_, reject) => { + signal.addEventListener('abort', () => reject(signal.reason), { once: true }) + }), + }), + ).rejects.toThrow('Benchmark model "worker-model" preflight failed: timed out after 5 ms') + + expect(events).not.toContain('open') + }) + it('skips the check for an injected completion transport', async () => { const requests: ChatRequest[] = [] const events: string[] = [] diff --git a/src/runtime/run-benchmark.ts b/src/runtime/run-benchmark.ts index d3fbe853..ea008d82 100644 --- a/src/runtime/run-benchmark.ts +++ b/src/runtime/run-benchmark.ts @@ -57,7 +57,11 @@ export interface BenchmarkConfig { * model. Injected `worker.complete` transports skip the check. Pass `false` to disable it or a * callback to check each unique model through a custom transport. */ - modelPreflight?: false | ((model: string, worker: Readonly) => Promise) + modelPreflight?: + | false + | ((model: string, worker: Readonly, signal: AbortSignal) => Promise) + /** Maximum time for each model availability check. Default 30 seconds. */ + modelPreflightTimeoutMs?: number } export interface BenchmarkLift { @@ -141,9 +145,12 @@ async function preflightModels(cfg: BenchmarkConfig): Promise { if (cfg.worker.complete && !cfg.modelPreflight) return const models = [...new Set([cfg.worker.model, cfg.worker.analystModel ?? cfg.worker.model])] + const timeoutMs = cfg.modelPreflightTimeoutMs ?? 30_000 + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) + throw new Error('modelPreflightTimeoutMs must be a positive finite number') const check = cfg.modelPreflight ?? - (async (model: string, worker: Readonly) => { + (async (model: string, worker: Readonly, signal: AbortSignal) => { await routerChatWithUsage( { routerBaseUrl: worker.routerBaseUrl, @@ -151,20 +158,48 @@ async function preflightModels(cfg: BenchmarkConfig): Promise { model, }, [{ role: 'user', content: 'Reply OK.' }], - { maxTokens: 1, reasoningEffort: 'none' }, + { maxTokens: 1, reasoningEffort: 'none', signal }, ) }) - await Promise.all( + const results = await Promise.allSettled( models.map(async (model) => { + const controller = new AbortController() + let timeoutError: Error | undefined + let timer: ReturnType | undefined try { - await check(model, cfg.worker) + await Promise.race([ + check(model, cfg.worker, controller.signal), + new Promise((_, reject) => { + timer = setTimeout(() => { + timeoutError = new Error(`timed out after ${timeoutMs} ms`) + controller.abort(timeoutError) + reject(timeoutError) + }, timeoutMs) + }), + ]) } catch (cause) { - const message = cause instanceof Error ? cause.message : String(cause) - throw new Error(`Benchmark model "${model}" preflight failed: ${message}`, { cause }) + const reported = timeoutError ?? cause + const message = reported instanceof Error ? reported.message : String(reported) + throw new Error(`Benchmark model "${model}" preflight failed: ${message}`, { + cause: reported, + }) + } finally { + if (timer) clearTimeout(timer) } }), ) + + const failures = results.flatMap((result) => + result.status === 'rejected' ? [result.reason] : [], + ) + if (failures.length === 1) throw failures[0] + if (failures.length > 1) { + const message = failures + .map((failure) => (failure instanceof Error ? failure.message : String(failure))) + .join('; ') + throw new AggregateError(failures, message) + } } /** Run the requested strategies over the tasks, scored by the Environment's own check. diff --git a/src/runtime/strategy-evolution.ts b/src/runtime/strategy-evolution.ts index b535181e..fcb16c88 100644 --- a/src/runtime/strategy-evolution.ts +++ b/src/runtime/strategy-evolution.ts @@ -73,6 +73,8 @@ export interface StrategyEvolutionConfig { * See `BenchmarkConfig.modelPreflight`. */ modelPreflight?: BenchmarkConfig['modelPreflight'] + /** Maximum time for each model availability check. Default 30 seconds. */ + modelPreflightTimeoutMs?: BenchmarkConfig['modelPreflightTimeoutMs'] author: EvolutionAuthor /** Rollouts (sample) / shots (refine) per strategy per task. Default 3. */ budget?: number @@ -423,6 +425,7 @@ export async function runStrategyEvolution(cfg: StrategyEvolutionConfig): Promis budget, concurrency, modelPreflight: modelsPreflighted ? false : cfg.modelPreflight, + modelPreflightTimeoutMs: cfg.modelPreflightTimeoutMs, ...(cfg.onTask ? { onTask: (row, done, total) => cfg.onTask?.(phase, row, done, total) } : {}),