From 99571606577c81c62408f595610dbddf64c5319c Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:13:24 +0200 Subject: [PATCH 01/21] feat: add engine-neutral wake limit env names with Codex aliases --- src/pi/cliSession.ts | 34 +++---------------- src/pi/engineWakeLimits.test.ts | 20 ++++++++++++ src/pi/engineWakeLimits.ts | 58 +++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 30 deletions(-) create mode 100644 src/pi/engineWakeLimits.test.ts create mode 100644 src/pi/engineWakeLimits.ts diff --git a/src/pi/cliSession.ts b/src/pi/cliSession.ts index 07b964c..49f8af8 100644 --- a/src/pi/cliSession.ts +++ b/src/pi/cliSession.ts @@ -59,36 +59,10 @@ export type CliEngineKind = "agy" | "codex" | "grok"; */ export const AGY_MAX_TOOL_TURNS = 16; -/** - * Codex's per-wake bounds, and the one place they are decided. - * - * `maxToolTurns` mediates only daimon-MCP tool calls; Codex's own shell - * (`exec_command`) is never routed through that gate, so a single Codex turn - * previously had no ceiling at all — one production wake ran 23:32→23:42 - * (unbounded wall clock) making 51 shell calls. Codex's `--json` stream - * reports token usage exactly once, on `turn.completed` — there is no - * incremental total to watch mid-turn (verified against a live multi-tool-call - * turn: `item.completed` fires once per tool call, but usage is reported only - * on the single terminal `turn.completed`) — so the token ceiling is the best - * bound obtainable from that wire shape: it converts an over-budget turn into - * an explicit, killed, named failure instead of a silent success, and the - * wall-clock timeout is what actually interrupts a runaway turn in progress. - */ -export const DEFAULT_CODEX_WAKE_TIMEOUT_MS = 240_000; -export const DEFAULT_CODEX_WAKE_TOKEN_CEILING = 300_000; -export const DAIMON_CODEX_WAKE_TIMEOUT_MS_ENV = "DAIMON_CODEX_WAKE_TIMEOUT_MS"; -export const DAIMON_CODEX_WAKE_TOKEN_CEILING_ENV = "DAIMON_CODEX_WAKE_TOKEN_CEILING"; - -const positiveInteger = (value: string | undefined, fallback: number, name: string): number => { - if (value === undefined) return fallback; - const parsed = Number(value); - if (!Number.isSafeInteger(parsed) || parsed <= 0) throw new Error(`${name} must be a positive integer`); - return parsed; -}; -export const resolveCodexWakeTimeoutMs = (environment: NodeJS.ProcessEnv = process.env): number => - positiveInteger(environment[DAIMON_CODEX_WAKE_TIMEOUT_MS_ENV], DEFAULT_CODEX_WAKE_TIMEOUT_MS, DAIMON_CODEX_WAKE_TIMEOUT_MS_ENV); -export const resolveCodexWakeTokenCeiling = (environment: NodeJS.ProcessEnv = process.env): number => - positiveInteger(environment[DAIMON_CODEX_WAKE_TOKEN_CEILING_ENV], DEFAULT_CODEX_WAKE_TOKEN_CEILING, DAIMON_CODEX_WAKE_TOKEN_CEILING_ENV); +export { + DAIMON_CODEX_WAKE_TIMEOUT_MS_ENV, DAIMON_CODEX_WAKE_TOKEN_CEILING_ENV, DAIMON_ENGINE_WAKE_TIMEOUT_MS_ENV, DAIMON_ENGINE_WAKE_TOKEN_CEILING_ENV, + DEFAULT_CODEX_WAKE_TIMEOUT_MS, DEFAULT_CODEX_WAKE_TOKEN_CEILING, resolveCodexWakeTimeoutMs, resolveCodexWakeTokenCeiling, resolveEngineWakeLimitOverrides +} from "./engineWakeLimits.js"; export type CliEngineOptions = { readonly commandArgs?: readonly string[]; diff --git a/src/pi/engineWakeLimits.test.ts b/src/pi/engineWakeLimits.test.ts new file mode 100644 index 0000000..e514af1 --- /dev/null +++ b/src/pi/engineWakeLimits.test.ts @@ -0,0 +1,20 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { resolveCodexWakeTimeoutMs, resolveCodexWakeTokenCeiling, resolveEngineWakeLimitOverrides } from "./engineWakeLimits.js"; + +test("engine-neutral wake bounds drive Codex, with the Codex names kept as aliases", () => { + assert.equal(resolveCodexWakeTimeoutMs({}), 240_000); + assert.equal(resolveCodexWakeTokenCeiling({}), 300_000); + assert.equal(resolveCodexWakeTimeoutMs({ DAIMON_ENGINE_WAKE_TIMEOUT_MS: "5000" }), 5_000); + assert.equal(resolveCodexWakeTokenCeiling({ DAIMON_CODEX_WAKE_TOKEN_CEILING: "7000" }), 7_000); + assert.equal(resolveCodexWakeTokenCeiling({ DAIMON_ENGINE_WAKE_TOKEN_CEILING: "7000", DAIMON_CODEX_WAKE_TOKEN_CEILING: "7000" }), 7_000); + assert.throws(() => resolveCodexWakeTokenCeiling({ DAIMON_ENGINE_WAKE_TOKEN_CEILING: "7000", DAIMON_CODEX_WAKE_TOKEN_CEILING: "8000" }), /disagree/u); + assert.throws(() => resolveCodexWakeTimeoutMs({ DAIMON_ENGINE_WAKE_TIMEOUT_MS: "0" }), /positive integer/u); +}); + +test("the broker receives only the bounds an operator actually set, as lowering limits", () => { + assert.equal(resolveEngineWakeLimitOverrides({}), undefined); + assert.deepEqual(resolveEngineWakeLimitOverrides({ DAIMON_ENGINE_WAKE_TOKEN_CEILING: "400000" }), { maxTokens: 400_000 }); + assert.deepEqual(resolveEngineWakeLimitOverrides({ DAIMON_CODEX_WAKE_TIMEOUT_MS: "480000", DAIMON_ENGINE_WAKE_TOKEN_CEILING: "1" }), { timeoutMs: 480_000, maxTokens: 1 }); +}); diff --git a/src/pi/engineWakeLimits.ts b/src/pi/engineWakeLimits.ts new file mode 100644 index 0000000..a4a0299 --- /dev/null +++ b/src/pi/engineWakeLimits.ts @@ -0,0 +1,58 @@ +/** + * Per-wake engine bounds, and the one place they are decided. + * + * `maxToolTurns` mediates only daimon-MCP tool calls; Codex's own shell + * (`exec_command`) is never routed through that gate, so a single Codex turn + * previously had no ceiling at all — one production wake ran 23:32→23:42 + * (unbounded wall clock) making 51 shell calls. Codex's `--json` stream + * reports token usage exactly once, on `turn.completed` — there is no + * incremental total to watch mid-turn (verified against a live multi-tool-call + * turn: `item.completed` fires once per tool call, but usage is reported only + * on the single terminal `turn.completed`) — so the token ceiling is the best + * bound obtainable from that wire shape: it converts an over-budget turn into + * an explicit, killed, named failure instead of a silent success, and the + * wall-clock timeout is what actually interrupts a runaway turn in progress. + * + * The names are engine-neutral: `DAIMON_ENGINE_WAKE_TIMEOUT_MS` and + * `DAIMON_ENGINE_WAKE_TOKEN_CEILING` bound Codex locally and are passed to the + * Grok broker as the wake's *lowering* limits (the broker refuses a value above + * its registration). The `DAIMON_CODEX_*` names remain aliases; setting both + * names of one bound to different values is refused rather than guessed. + */ +export const DEFAULT_CODEX_WAKE_TIMEOUT_MS = 240_000; +export const DEFAULT_CODEX_WAKE_TOKEN_CEILING = 300_000; +export const DAIMON_ENGINE_WAKE_TIMEOUT_MS_ENV = "DAIMON_ENGINE_WAKE_TIMEOUT_MS"; +export const DAIMON_ENGINE_WAKE_TOKEN_CEILING_ENV = "DAIMON_ENGINE_WAKE_TOKEN_CEILING"; +export const DAIMON_CODEX_WAKE_TIMEOUT_MS_ENV = "DAIMON_CODEX_WAKE_TIMEOUT_MS"; +export const DAIMON_CODEX_WAKE_TOKEN_CEILING_ENV = "DAIMON_CODEX_WAKE_TOKEN_CEILING"; + +const positiveInteger = (value: string, name: string): number => { + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed <= 0) throw new Error(`${name} must be a positive integer`); + return parsed; +}; + +const declared = (environment: NodeJS.ProcessEnv, neutral: string, alias: string): number | undefined => { + const primary = environment[neutral], legacy = environment[alias]; + const value = primary === undefined ? undefined : positiveInteger(primary, neutral); + const aliased = legacy === undefined ? undefined : positiveInteger(legacy, alias); + if (value !== undefined && aliased !== undefined && value !== aliased) throw new Error(`${neutral} and ${alias} disagree; set one`); + return value ?? aliased; +}; + +export const resolveCodexWakeTimeoutMs = (environment: NodeJS.ProcessEnv = process.env): number => + declared(environment, DAIMON_ENGINE_WAKE_TIMEOUT_MS_ENV, DAIMON_CODEX_WAKE_TIMEOUT_MS_ENV) ?? DEFAULT_CODEX_WAKE_TIMEOUT_MS; +export const resolveCodexWakeTokenCeiling = (environment: NodeJS.ProcessEnv = process.env): number => + declared(environment, DAIMON_ENGINE_WAKE_TOKEN_CEILING_ENV, DAIMON_CODEX_WAKE_TOKEN_CEILING_ENV) ?? DEFAULT_CODEX_WAKE_TOKEN_CEILING; + +/** + * The limits a wake asks a broker to lower to: only the bounds the operator + * actually set, never the Codex defaults (a broker registration's declared + * limits already are the defaults there). + */ +export const resolveEngineWakeLimitOverrides = (environment: NodeJS.ProcessEnv = process.env): Readonly<{ timeoutMs?: number; maxTokens?: number }> | undefined => { + const timeoutMs = declared(environment, DAIMON_ENGINE_WAKE_TIMEOUT_MS_ENV, DAIMON_CODEX_WAKE_TIMEOUT_MS_ENV); + const maxTokens = declared(environment, DAIMON_ENGINE_WAKE_TOKEN_CEILING_ENV, DAIMON_CODEX_WAKE_TOKEN_CEILING_ENV); + if (timeoutMs === undefined && maxTokens === undefined) return undefined; + return { ...(timeoutMs === undefined ? {} : { timeoutMs }), ...(maxTokens === undefined ? {} : { maxTokens }) }; +}; From 7e285428475d680329ea6ccd7c5db4939bcb4442 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:13:24 +0200 Subject: [PATCH 02/21] feat: record per-request timestamps, Grok stream usage and turn-keyed usage rows --- src/pi/codexRolloutUsage.test.ts | 8 ++ src/pi/codexRolloutUsage.ts | 34 ++++- src/pi/fixtures/README.md | 20 +++ .../grok-1.0.34-streaming-two-requests.jsonl | 5 + src/pi/grokStreamUsage.test.ts | 40 ++++++ src/pi/grokStreamUsage.ts | 52 ++++++++ src/runtime/engineBrokerTurnAccounting.ts | 125 ++++++++++++++++++ src/runtime/turnRequestLedger.test.ts | 14 ++ src/runtime/turnRequestLedger.ts | 52 +++++++- src/runtime/turnUsageLedger.test.ts | 13 +- src/runtime/turnUsageLedger.ts | 34 ++++- src/runtime/wakeFuse.test.ts | 12 ++ src/runtime/wakeFuse.ts | 5 +- 13 files changed, 408 insertions(+), 6 deletions(-) create mode 100644 src/pi/fixtures/grok-1.0.34-streaming-two-requests.jsonl create mode 100644 src/pi/grokStreamUsage.test.ts create mode 100644 src/pi/grokStreamUsage.ts create mode 100644 src/runtime/engineBrokerTurnAccounting.ts diff --git a/src/pi/codexRolloutUsage.test.ts b/src/pi/codexRolloutUsage.test.ts index e7fc317..ea41272 100644 --- a/src/pi/codexRolloutUsage.test.ts +++ b/src/pi/codexRolloutUsage.test.ts @@ -46,6 +46,14 @@ test("a real multi-request rollout yields one row per model request, cached and [288, 228, 50_292] ]); assert.deepEqual(requests.map((request) => request.cacheWrite), [0, 0, 0, 0]); + // End is the usage frame; start is the first non-usage frame after the previous + // request's usage frame, else the previous request's end. + assert.deepEqual(requests.map((request) => [request.startedAt, request.endedAt]), [ + ["2026-09-05T01:32:00.678Z", "2026-09-05T01:32:19.183Z"], + ["2026-09-05T01:34:00.679Z", "2026-09-05T01:52:22.056Z"], + ["2026-09-05T01:52:22.056Z", "2026-09-05T01:52:37.604Z"], + ["2026-09-05T01:52:37.604Z", "2026-09-05T01:52:51.112Z"] + ]); }); test("reasoning tokens, which the per-wake ledger drops entirely, survive per request", async () => { diff --git a/src/pi/codexRolloutUsage.ts b/src/pi/codexRolloutUsage.ts index b803f5e..0c872bf 100644 --- a/src/pi/codexRolloutUsage.ts +++ b/src/pi/codexRolloutUsage.ts @@ -44,6 +44,17 @@ export type CodexRequestUsage = Readonly<{ output: number; reasoning: number; total: number; + /** + * When the request began and ended, from the rollout's own frame + * `timestamp`s. `endedAt` is the usage frame's timestamp (Codex appends it at + * `response.completed`); `startedAt` is the first non-usage frame after the + * previous request's usage frame — the tool output or turn context that + * triggers the next request — falling back to the previous request's end. + * Either is absent when the frames carry no valid timestamp: a wake-end stamp + * substituted here would be indistinguishable from a measured one. + */ + startedAt?: string; + endedAt?: string; }>; /** @@ -177,17 +188,20 @@ export const parseCodexRolloutRequests = (text: string, threadId: string): reado const requests: CodexRequestUsage[] = []; const fallback: CodexRequestUsage[] = []; let previousFallback = ""; + const clocks = { record: requestClock(), fallback: requestClock() }; for (const line of text.split("\n")) { if (line.trim().length === 0) continue; let frame: unknown; try { frame = JSON.parse(line); } catch { continue; } if (!isRecord(frame)) continue; const block = usageBlock(frame, threadId); + const usageFrame = frame.type === "token_usage_record" || (frame.type === "event_msg" && isRecord(frame.payload) && frame.payload.type === "token_count"); + if (!usageFrame) { if (frame.type !== "session_meta") { clocks.record.observe(frame.timestamp); clocks.fallback.observe(frame.timestamp); } continue; } if (block === undefined) continue; if (frame.type === "token_usage_record") { const decoded = decodeRequestUsage(block.usage, requests.length); if (decoded === undefined) return []; - requests.push(decoded); + requests.push({ ...decoded, ...clocks.record.close(frame.timestamp) }); continue; } // `token_count` is NOT one frame per request: the captured fixture carries @@ -201,7 +215,7 @@ export const parseCodexRolloutRequests = (text: string, threadId: string): reado previousFallback = serialized; const decoded = decodeRequestUsage(block.usage, fallback.length); if (decoded === undefined) return []; - fallback.push(decoded); + fallback.push({ ...decoded, ...clocks.fallback.close(frame.timestamp) }); } // A Codex version that emits both shapes emits `token_usage_record` once per // request, so the richer one wins outright rather than being merged into a @@ -209,6 +223,22 @@ export const parseCodexRolloutRequests = (text: string, threadId: string): reado return requests.length > 0 ? requests : fallback; }; +const timestampOf = (value: unknown): string | undefined => + typeof value === "string" && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,9})?Z$/u.test(value) && !Number.isNaN(Date.parse(value)) ? value : undefined; + +/** Tracks one request stream's start/end stamps; see {@link CodexRequestUsage.startedAt}. */ +const requestClock = () => { + let start: string | undefined, previousEnd: string | undefined; + return { + observe(value: unknown): void { start ??= timestampOf(value); }, + close(value: unknown): { startedAt?: string; endedAt?: string } { + const endedAt = timestampOf(value), startedAt = start ?? previousEnd; + start = undefined; previousEnd = endedAt; + return { ...(startedAt === undefined ? {} : { startedAt }), ...(endedAt === undefined ? {} : { endedAt }) }; + } + }; +}; + /** * Read one turn's per-request usage. Never throws. * diff --git a/src/pi/fixtures/README.md b/src/pi/fixtures/README.md index f158a6d..0cf75ed 100644 --- a/src/pi/fixtures/README.md +++ b/src/pi/fixtures/README.md @@ -117,3 +117,23 @@ the previous block and why `token_usage_record` wins outright when both exist. The four requests also show exactly the shape the study predicted, in one wake: fresh input 15,742 → 248 → 4,276 → 10,068 against a context that only grows from 34,686 to 50,004 — most of every request after the first is cache-read replay. + + +# Grok 1.0.34 per-request stream fixture + +`grok-1.0.34-streaming-two-requests.jsonl` is a real two-request turn captured on +2026-09-17 from `grok 1.0.34` (macOS arm64) with the lean worker flags and +`--output-format streaming-messages-json` (P0 host matrix cell c14: one MCP +`use_tool` call, then the answer). Sanitization before commit: the capturing +scratchpad `cwd` was replaced with `/workspace`; every frame is otherwise +verbatim. + +It pins what `grokStreamUsage.ts` reads and the broker meters per request: + + assistant.message.id one request per distinct id + assistant.message.usage that request's own four buckets + result.modelUsage keys "grok-4.6-build" for grok-4.6 + +The two per-request totals (2,775 + 2,810) sum exactly to the terminal +`result.usage` (5,585), which is why a failed turn's frames are trusted as its +partial usage. diff --git a/src/pi/fixtures/grok-1.0.34-streaming-two-requests.jsonl b/src/pi/fixtures/grok-1.0.34-streaming-two-requests.jsonl new file mode 100644 index 0000000..3eaabd0 --- /dev/null +++ b/src/pi/fixtures/grok-1.0.34-streaming-two-requests.jsonl @@ -0,0 +1,5 @@ +{"type":"system","subtype":"init","session_id":"01a0ad21-a90f-7f71-8054-93fdb4334d6a","apiKeySource":"oauth","model":"grok-4.6","cwd":"/workspace","permissionMode":"bypassPermissions","tools":["run_terminal_command","read_file","list_dir","grep","search_tool","use_tool"],"slash_commands":["compact","always-approve","context","session-info","feedback"],"mcp_servers":[{"name":"probe","status":"connected"}],"skills":[],"uuid":"880b6dc2-0488-4678-9ec8-04b2c2fe9efd"} +{"type":"assistant","message":{"id":"msg_0","type":"message","role":"assistant","model":"grok-4.6","content":[{"type":"thinking","thinking":"The user wants me to use moltnet_read on room \"conference\" to get a code word. The instructions say I can call probe__moltnet_read directly with use_tool.","signature":"tl43Ieb3gy7AEz+YdNJWSx3rN7HTKi9cJKhytQwzBfudDDWXQ6UEWS4JxMYQZJRrKQeBXNtXluNa1UHRr5C3OMGlpHodcziT9fqhMn5tgr0xjUjCRq1/KL0fdIGj+tU+8sHeAKtHBCMC2bFFkbtdm4BrWR0kLcHzQKbla9tof1RpzzgFtwyIx3fsVCmp/AYHHNrm0BNZvQpfbb9M1yyN4AD0d7E0b1VZru3uMx9N47RtFoH7uWQ"},{"type":"text","text":"I'll read the conference room now."},{"type":"tool_use","id":"call-9582489e-555b-4f65-81ba-8a46439283ce-0","name":"use_tool","input":{"tool_name":"probe__moltnet_read","tool_input":{"room":"conference"}}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":2568,"output_tokens":79,"cache_read_input_tokens":128,"cache_creation_input_tokens":0}},"parent_tool_use_id":null,"session_id":"01a0ad21-a90f-7f71-8054-93fdb4334d6a","uuid":"6c8748c9-2ce6-437b-a0d2-24a01ca23b3b"} +{"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"call-9582489e-555b-4f65-81ba-8a46439283ce-0","content":"{\"type\":\"MCP\",\"tool_name\":\"moltnet_read\",\"server_name\":\"probe\",\"output\":{\"OkayOutput\":\"Brass: the code word is TANGERINE-7.\"}}","is_error":false}]},"parent_tool_use_id":null,"session_id":"01a0ad21-a90f-7f71-8054-93fdb4334d6a","uuid":"fcd581d3-b43f-4aa0-a9d7-9c25862b4bfc"} +{"type":"assistant","message":{"id":"msg_1","type":"message","role":"assistant","model":"grok-4.6","content":[{"type":"thinking","thinking":"Got the code word. Answer briefly.","signature":"KcaYjJIHN4fzMKF4FdR0695QoQUKP7Byln+CLEgdhkDCFs3y0wBDWWYCWDkmvXSZnjMYqtfHn79Tgkif73U"},{"type":"text","text":"TANGERINE-7"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":109,"output_tokens":13,"cache_read_input_tokens":2688,"cache_creation_input_tokens":0}},"parent_tool_use_id":null,"session_id":"01a0ad21-a90f-7f71-8054-93fdb4334d6a","uuid":"b1ddbf87-b115-4ee9-b773-0d81f65a8ef7"} +{"type":"result","subtype":"success","is_error":false,"duration_ms":3300,"duration_api_ms":3118,"num_turns":2,"result":"TANGERINE-7","stop_reason":"end_turn","total_cost_usd":0.00248676,"usage":{"input_tokens":2677,"output_tokens":92,"cache_read_input_tokens":2816,"cache_creation_input_tokens":0,"server_tool_use":{"web_search_requests":0}},"modelUsage":{"grok-4.6-build":{"inputTokens":2677,"outputTokens":92,"cacheReadInputTokens":2816,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.00248676}},"session_id":"01a0ad21-a90f-7f71-8054-93fdb4334d6a","uuid":"0dbd02b1-4721-4c52-8416-25bf2455bb8a"} diff --git a/src/pi/grokStreamUsage.test.ts b/src/pi/grokStreamUsage.test.ts new file mode 100644 index 0000000..37942fb --- /dev/null +++ b/src/pi/grokStreamUsage.test.ts @@ -0,0 +1,40 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +import { decodeGrokHeadlessTurn } from "./grokHeadlessResult.js"; +import { decodeGrokStreamUsage } from "./grokStreamUsage.js"; + +const fixture = (): Promise => readFile(fileURLToPath(new URL("./fixtures/grok-1.0.34-streaming-two-requests.jsonl", import.meta.url)), "utf8"); + +test("a real 1.0.34 two-request stream yields one row per request, summing exactly to the terminal result", async () => { + const output = await fixture(); + const stream = decodeGrokStreamUsage(output); + assert.deepEqual(stream.requests, [ + { index: 0, input: 2_568, cacheRead: 128, cacheWrite: 0, output: 79, total: 2_775 }, + { index: 1, input: 109, cacheRead: 2_688, cacheWrite: 0, output: 13, total: 2_810 } + ]); + assert.equal(stream.sessionId, "01a0ad21-a90f-7f71-8054-93fdb4334d6a"); + assert.deepEqual(stream.reportedModels, ["grok-4.6-build"]); + const terminal = decodeGrokHeadlessTurn(output).usage!; + assert.equal(stream.requests.reduce((sum, request) => sum + request.total, 0), terminal.total); +}); + +test("a malformed per-request usage block discards every request instead of reporting part of the turn", async () => { + const lines = (await fixture()).split("\n"); + const index = lines.findIndex((line) => line.includes('"msg_1"')); + lines[index] = lines[index]!.replace('"input_tokens":109', '"input_tokens":"109"'); + assert.deepEqual(decodeGrokStreamUsage(lines.join("\n")).requests, []); +}); + +test("frames repeating one message id are one request, and a torn line is skipped", async () => { + const lines = (await fixture()).split("\n").filter((line) => line.length > 0); + const repeated = lines.find((line) => line.includes('"msg_0"'))!; + const stream = decodeGrokStreamUsage([...lines.slice(0, 2), repeated, ...lines.slice(2), '{"type":"assist'].join("\n")); + assert.equal(stream.requests.length, 2); +}); + +test("the captured fixture carries no capturing machine's environment", async () => { + assert.doesNotMatch(await fixture(), /\/Users\/|\/private\/|scratchpad|\/home\//u); +}); diff --git a/src/pi/grokStreamUsage.ts b/src/pi/grokStreamUsage.ts new file mode 100644 index 0000000..79b4aef --- /dev/null +++ b/src/pi/grokStreamUsage.ts @@ -0,0 +1,52 @@ +/** + * Per-request token accounting read off a Grok `streaming-messages-json` + * stream. + * + * Grok 1.0.34 puts the usage of each model request on that request's + * top-level `assistant` frame (`message.usage`, the same four disjoint + * Messages API buckets as the terminal `result.usage`), before any tool result + * of that request, and the per-request frames sum exactly to the terminal + * result (P0 host matrix). That makes the stream usable in two places the + * terminal frame is not: a turn that failed before its `result` frame, and a + * per-request ledger row. + * + * Never throws, and never invents a number: a usage block that is present but + * does not decode discards *all* requests, because one fabricated zero is + * byte-identical to a measured one. + */ +export type GrokRequestUsage = Readonly<{ index: number; input: number; cacheRead: number; cacheWrite: number; output: number; total: number }>; +export type GrokStreamUsage = Readonly<{ requests: readonly GrokRequestUsage[]; sessionId?: string; reportedModels: readonly string[] }>; + +type JsonRecord = Readonly>; +const isRecord = (value: unknown): value is JsonRecord => typeof value === "object" && value !== null && !Array.isArray(value); +const tokenCount = (value: unknown): number | undefined => typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : undefined; +const SESSION_ID = /^[A-Za-z0-9][A-Za-z0-9-]{0,63}$/u; +const MODEL_KEY = /^[a-z0-9][a-z0-9.-]{0,63}$/u; + +const decodeUsage = (usage: unknown): Omit | undefined => { + if (!isRecord(usage)) return undefined; + const input = tokenCount(usage.input_tokens), output = tokenCount(usage.output_tokens), cacheRead = tokenCount(usage.cache_read_input_tokens), cacheWrite = tokenCount(usage.cache_creation_input_tokens); + if (input === undefined || output === undefined || cacheRead === undefined || cacheWrite === undefined) return undefined; + return { input, cacheRead, cacheWrite, output, total: input + cacheRead + cacheWrite + output }; +}; + +export const decodeGrokStreamUsage = (output: string): GrokStreamUsage => { + const byMessage = new Map>(); + const reportedModels = new Set(); + let sessionId: string | undefined, corrupt = false; + for (const line of output.split(/\r?\n/u)) { + if (line.trim().length === 0) continue; + let event: unknown; + try { event = JSON.parse(line); } catch { continue; } + if (!isRecord(event)) continue; + if (typeof event.session_id === "string" && SESSION_ID.test(event.session_id)) sessionId ??= event.session_id; + if (event.type === "result" && isRecord(event.modelUsage)) for (const key of Object.keys(event.modelUsage)) reportedModels.add(MODEL_KEY.test(key) ? key : "invalid"); + if (event.type !== "assistant" || event.parent_tool_use_id !== null || !isRecord(event.message) || event.message.usage === undefined) continue; + const decoded = decodeUsage(event.message.usage); + if (decoded === undefined || typeof event.message.id !== "string") { corrupt = true; continue; } + // One request can surface as more than one frame of the same message; it is one request. + byMessage.set(event.message.id, decoded); + } + const requests = corrupt ? [] : [...byMessage.values()].map((usage, index) => Object.freeze({ index, ...usage })); + return { requests, ...(sessionId === undefined ? {} : { sessionId }), reportedModels: [...reportedModels].sort() }; +}; diff --git a/src/runtime/engineBrokerTurnAccounting.ts b/src/runtime/engineBrokerTurnAccounting.ts new file mode 100644 index 0000000..951c4f4 --- /dev/null +++ b/src/runtime/engineBrokerTurnAccounting.ts @@ -0,0 +1,125 @@ +import { GROK_BROKER_MODELS, GROK_WORKER_MAX_TURNS } from "../contracts/grokWorkerContract.js"; +import type { GrokBrokerModel } from "./grokBrokerModelPolicy.js"; + +/** + * Numeric-only accounting the Grok engine broker seals beside every terminal + * turn response, and the per-turn limits it enforces. + * + * The broker is the single writer of this data (turn registry record, control + * response, usage ledger row). Nothing engine-controlled and non-numeric is + * persisted: `model` is a member of the closed declared list, never the + * provider's own string, and `limitReason` is a closed vocabulary. + * + * Buckets are disjoint and `total = input + cacheRead + cacheWrite + output`. + * `reasoning` is reported only when the source separates it; it is already + * inside `output` and never added to `total`. + */ +export type EngineBrokerTurnUsage = Readonly<{ input: number; cacheRead: number; cacheWrite: number; output: number; total: number; reasoning?: number }>; +export const ENGINE_BROKER_LIMIT_REASONS = ["tokens", "requests", "timeout", "none"] as const; +export type EngineBrokerLimitReason = (typeof ENGINE_BROKER_LIMIT_REASONS)[number]; +export type EngineBrokerTurnLimits = Readonly<{ maxRequests: number; maxTokens: number; timeoutMs: number }>; +export type EngineBrokerTurnLimitOverrides = Readonly>; +export type EngineBrokerTurnAccounting = Readonly<{ + outcome: "completed" | "failed"; + usage: EngineBrokerTurnUsage | null; + model: GrokBrokerModel; + requests: number; + limitReason: EngineBrokerLimitReason; +}>; + +/** + * Bounds every declared limit must sit inside. `maxRequests` stays at or below + * the launcher's compiled `--max-turns` backstop, so the broker ceiling is the + * one that fires first. + */ +export const ENGINE_BROKER_LIMIT_BOUNDS = Object.freeze({ + maxRequests: [1, GROK_WORKER_MAX_TURNS], + maxTokens: [1, 10_000_000], + timeoutMs: [1_000, 3_600_000] +} as const); + +/** What a v1 `service.json` registration gets; equal to the Codex per-wake defaults. */ +export const DEFAULT_GROK_BROKER_TURN_LIMITS: EngineBrokerTurnLimits = Object.freeze({ maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }); + +const LIMIT_KEYS = ["maxRequests", "maxTokens", "timeoutMs"] as const; +type JsonRecord = Record; +const plain = (value: unknown): value is JsonRecord => + value !== null && typeof value === "object" && !Array.isArray(value) && (Object.getPrototypeOf(value) === Object.prototype || Object.getPrototypeOf(value) === null); +const count = (value: unknown): value is number => typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +const invalid = (label: string): TypeError => new TypeError(`invalid ${label}`); + +const limitValue = (key: (typeof LIMIT_KEYS)[number], value: unknown, label: string): number => { + const [minimum, maximum] = ENGINE_BROKER_LIMIT_BOUNDS[key]; + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < minimum || value > maximum) throw invalid(label); + return value; +}; + +/** Exactly `{maxRequests, maxTokens, timeoutMs}`, each inside its bound. */ +export function parseEngineBrokerTurnLimits(value: unknown, label = "engine broker turn limits"): EngineBrokerTurnLimits { + if (!plain(value) || Object.keys(value).length !== LIMIT_KEYS.length || LIMIT_KEYS.some((key) => !Object.hasOwn(value, key))) throw invalid(label); + return Object.freeze({ maxRequests: limitValue("maxRequests", value.maxRequests, label), maxTokens: limitValue("maxTokens", value.maxTokens, label), timeoutMs: limitValue("timeoutMs", value.timeoutMs, label) }); +} + +/** A non-empty subset of the limit keys, each inside its bound. */ +export function parseEngineBrokerTurnLimitOverrides(value: unknown, label = "engine broker turn limits"): EngineBrokerTurnLimitOverrides { + if (!plain(value) || Object.keys(value).length === 0 || Object.keys(value).some((key) => !(LIMIT_KEYS as readonly string[]).includes(key))) throw invalid(label); + const result: Partial> = {}; + for (const key of LIMIT_KEYS) if (Object.hasOwn(value, key)) result[key] = limitValue(key, value[key], label); + return Object.freeze(result); +} + +/** + * The limits one turn runs under: the registration's, lowered by the wake. + * A wake can never raise a declared limit; asking to is refused rather than + * clamped, so a misconfigured caller learns it instead of silently getting less. + */ +export function lowerEngineBrokerTurnLimits(declared: EngineBrokerTurnLimits, overrides: EngineBrokerTurnLimitOverrides | undefined): EngineBrokerTurnLimits { + if (overrides === undefined) return declared; + for (const key of LIMIT_KEYS) { + const requested = overrides[key]; + if (requested !== undefined && requested > declared[key]) throw new RangeError(`engine broker turn limit ${key} may only be lowered`); + } + return Object.freeze({ maxRequests: overrides.maxRequests ?? declared.maxRequests, maxTokens: overrides.maxTokens ?? declared.maxTokens, timeoutMs: overrides.timeoutMs ?? declared.timeoutMs }); +} + +/** Four disjoint buckets plus an optional reasoning split; the total invariant is re-checked. */ +export function parseEngineBrokerTurnUsage(value: unknown, label = "engine broker turn usage"): EngineBrokerTurnUsage { + const required = ["input", "cacheRead", "cacheWrite", "output", "total"]; + if (!plain(value)) throw invalid(label); + const keys = Object.keys(value); + if (required.some((key) => !Object.hasOwn(value, key)) || keys.some((key) => !required.includes(key) && key !== "reasoning")) throw invalid(label); + if (![value.input, value.cacheRead, value.cacheWrite, value.output, value.total].every(count)) throw invalid(label); + const usage = value as { input: number; cacheRead: number; cacheWrite: number; output: number; total: number; reasoning?: unknown }; + if (usage.total !== usage.input + usage.cacheRead + usage.cacheWrite + usage.output) throw invalid(label); + if (usage.reasoning !== undefined && (!count(usage.reasoning) || usage.reasoning > usage.output)) throw invalid(label); + return Object.freeze({ input: usage.input, cacheRead: usage.cacheRead, cacheWrite: usage.cacheWrite, output: usage.output, total: usage.total, ...(usage.reasoning === undefined ? {} : { reasoning: usage.reasoning as number }) }); +} + +export const sumEngineBrokerTurnUsage = (items: readonly EngineBrokerTurnUsage[]): EngineBrokerTurnUsage | null => { + if (items.length === 0) return null; + const sum = (pick: (usage: EngineBrokerTurnUsage) => number): number => items.reduce((total, usage) => total + pick(usage), 0); + const reasoning = items.every((usage) => usage.reasoning !== undefined) ? { reasoning: sum((usage) => usage.reasoning ?? 0) } : {}; + return Object.freeze({ input: sum((usage) => usage.input), cacheRead: sum((usage) => usage.cacheRead), cacheWrite: sum((usage) => usage.cacheWrite), output: sum((usage) => usage.output), total: sum((usage) => usage.total), ...reasoning }); +}; + +/** Validates the accounting members of a v2 terminal response against its kind. */ +export function parseEngineBrokerTurnAccounting(value: JsonRecord, kind: "completed" | "failed"): EngineBrokerTurnAccounting { + if (value.outcome !== kind) throw invalid("broker frame"); + if (!(GROK_BROKER_MODELS as readonly unknown[]).includes(value.model)) throw invalid("broker frame"); + if (!count(value.requests) || value.requests > 1_024) throw invalid("broker frame"); + if (!(ENGINE_BROKER_LIMIT_REASONS as readonly unknown[]).includes(value.limitReason)) throw invalid("broker frame"); + if (kind === "completed" && value.limitReason !== "none") throw invalid("broker frame"); + let usage: EngineBrokerTurnUsage | null = null; + if (value.usage !== null) { try { usage = parseEngineBrokerTurnUsage(value.usage); } catch { throw invalid("broker frame"); } } + return { outcome: kind, usage, model: value.model as GrokBrokerModel, requests: value.requests, limitReason: value.limitReason as EngineBrokerLimitReason }; +} + +/** + * Maps a provider-reported model key onto the declared closed-list model. + * + * Grok 1.0.34 reports `grok-4.6` usage under `grok-4.6-build` (P0). The exact + * declared id and its `-build` alias are accepted; anything else is a + * different model and yields `undefined`. + */ +export const mapGrokReportedModel = (reported: string, declared: GrokBrokerModel): GrokBrokerModel | undefined => + reported === declared || reported === `${declared}-build` ? declared : undefined; diff --git a/src/runtime/turnRequestLedger.test.ts b/src/runtime/turnRequestLedger.test.ts index 954c069..5944103 100644 --- a/src/runtime/turnRequestLedger.test.ts +++ b/src/runtime/turnRequestLedger.test.ts @@ -96,9 +96,23 @@ test("a real rollout reaches the stream end to end, one line per request", async assert.equal(rows.length, 4); assert.equal(rows.every((row) => row.thread === FIXTURE_THREAD && row.wake === "wake-1" && row.requests === 4), true); assert.deepEqual(rows.map((row) => row.fresh_input), [15_742, 248, 4_276, 10_068]); + // Each request carries its own rollout-frame interval, not the wake's append time. + // Mutation guard: stamping every row with `at` collapses these to one value. + assert.deepEqual(rows.map((row) => [row.started_at, row.ended_at]), [ + ["2026-09-05T01:32:00.678Z", "2026-09-05T01:32:19.183Z"], + ["2026-09-05T01:34:00.679Z", "2026-09-05T01:52:22.056Z"], + ["2026-09-05T01:52:22.056Z", "2026-09-05T01:52:37.604Z"], + ["2026-09-05T01:52:37.604Z", "2026-09-05T01:52:51.112Z"] + ]); }); }); +test("a request without measured timestamps carries none rather than the wake end", () => { + const [row] = renderTurnRequestLines({ agent: "a", wake: "w", thread: FIXTURE_THREAD, at: "2026-09-05T02:00:00.000Z", requests: [request({ startedAt: "not-a-time" })] }).split("\n").filter((line) => line.length > 0).map((line) => JSON.parse(line)); + assert.equal("started_at" in row, false); + assert.equal("ended_at" in row, false); +}); + test("a missing or malformed rollout writes nothing and still resolves", async () => { await withDirectory(async (directory) => { const home = path.join(directory, "codex-home"); diff --git a/src/runtime/turnRequestLedger.ts b/src/runtime/turnRequestLedger.ts index 21d5fbb..a13bfdf 100644 --- a/src/runtime/turnRequestLedger.ts +++ b/src/runtime/turnRequestLedger.ts @@ -2,6 +2,7 @@ import { constants } from "node:fs"; import { open, rename, stat } from "node:fs/promises"; import { readCodexRolloutRequests, type CodexRequestUsage } from "../pi/codexRolloutUsage.js"; +import type { GrokBrokerModel } from "./grokBrokerModelPolicy.js"; import { TURN_USAGE_LEDGER, TURN_USAGE_MAX_IDENTIFIER_CHARS, TURN_USAGE_ROTATE_BYTES } from "./turnUsageLedger.js"; /** @@ -91,10 +92,59 @@ export const renderTurnRequestLines = (entry: TurnRequestEntry): string => { cache_write: request.cacheWrite, output: request.output, reasoning: request.reasoning, - total: request.total + total: request.total, + ...requestClockFields(request) })}\n`).join(""); }; +const TIMESTAMP = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,9})?Z$/u; +/** Per-request `started_at`/`ended_at`, each only when it was measured; `at` stays the append time. */ +const requestClockFields = (request: Readonly<{ startedAt?: string; endedAt?: string }>): Record => ({ + ...(request.startedAt !== undefined && TIMESTAMP.test(request.startedAt) ? { started_at: request.startedAt } : {}), + ...(request.endedAt !== undefined && TIMESTAMP.test(request.endedAt) ? { ended_at: request.endedAt } : {}) +}); + +/** One Grok broker model request: usage from the worker stream, timing from the proxy. */ +export type GrokTurnRequest = Readonly<{ index: number; input: number; cacheRead: number; cacheWrite: number; output: number; total: number; startedAt?: string; endedAt?: string }>; +export type GrokTurnRequestEntry = Readonly<{ agent: string; wake: string; turn: string; session?: string; model: GrokBrokerModel; requests: readonly GrokTurnRequest[]; at?: string }>; + +/** + * Grok rows share the Codex row's field meaning: `input` is the whole prompt + * side the request replayed (`input_tokens + cache_read`), `cached_input` the + * cache read, `fresh_input` the uncached remainder. Grok does not separate + * reasoning tokens, so `reasoning` is absent rather than zero. `turn` is the + * broker idempotency key and `thread` the Grok session id when the stream + * named one. + */ +export const renderGrokTurnRequestLines = (entry: GrokTurnRequestEntry): string => { + const at = entry.at ?? new Date().toISOString(); + return entry.requests.map((request) => `${JSON.stringify({ + v: TURN_REQUEST_LEDGER_VERSION, + agent: bounded(entry.agent), + wake: bounded(entry.wake), + engine: "grok", + at, + turn: entry.turn, + ...(entry.session === undefined ? {} : { thread: bounded(entry.session) }), + model: entry.model, + request: request.index, + requests: entry.requests.length, + input: request.input + request.cacheRead, + cached_input: request.cacheRead, + fresh_input: request.input, + cache_write: request.cacheWrite, + output: request.output, + total: request.total, + ...requestClockFields(request) + })}\n`).join(""); +}; + +/** Advisory and never rejects, like {@link recordTurnRequests}; an empty turn writes nothing. */ +export const recordGrokTurnRequests = async (file: string, entry: GrokTurnRequestEntry): Promise => { + if (entry.requests.length === 0) return false; + try { await rotate(file); await appendLines(file, renderGrokTurnRequestLines(entry)); return true; } catch { return false; } +}; + const rotate = async (file: string): Promise => { let size: number; try { size = (await stat(file)).size; } catch { return; } diff --git a/src/runtime/turnUsageLedger.test.ts b/src/runtime/turnUsageLedger.test.ts index e6be905..7ddefad 100644 --- a/src/runtime/turnUsageLedger.test.ts +++ b/src/runtime/turnUsageLedger.test.ts @@ -15,6 +15,7 @@ import { TURN_USAGE_LEDGER_VERSION, TURN_USAGE_MAX_IDENTIFIER_CHARS, TURN_USAGE_ROTATE_BYTES, + dedupeTurnUsageRows, type TurnUsageEntry } from "./turnUsageLedger.js"; @@ -148,7 +149,7 @@ test("a failed wake's row carries the outcome and a reason from the closed vocab assert.equal(failed.reason, "token_ceiling"); assert.equal(failed.total, measurement.total, "the numbers are the ones the engine reported, not the outcome's"); - assert.deepEqual([...TURN_USAGE_FAILURE_REASONS], ["token_ceiling", "wake_timeout", "output_limit", "engine_exit", "turn_rejected", "unknown"]); + assert.deepEqual([...TURN_USAGE_FAILURE_REASONS], ["token_ceiling", "request_ceiling", "wake_timeout", "output_limit", "engine_exit", "turn_rejected", "unknown"]); for (const reason of TURN_USAGE_FAILURE_REASONS) { assert.equal(JSON.parse(renderTurnUsageLine(entry({ outcome: { status: "failed", reason } }))).reason, reason); } @@ -183,3 +184,13 @@ test("the outcome survives an append and a read back of the ledger file", async for (const record of written) assert.equal(record.v, TURN_USAGE_LEDGER_VERSION, "an added field, not a version bump"); }); }); + +test("broker rows carry the turn key, closed limit reason and closed model, and readers dedupe on the turn", () => { + const turn = "c".repeat(64); + const row = JSON.parse(renderTurnUsageLine(entry({ turn, limitReason: "requests", model: "grok-4.6", outcome: { status: "failed", reason: "request_ceiling" } }))); + assert.deepEqual([row.turn, row.limit_reason, row.model, row.reason], [turn, "requests", "grok-4.6", "request_ceiling"]); + assert.equal(row.total, row.input + row.cache_read + row.cache_write + row.output); + const forged = JSON.parse(renderTurnUsageLine(entry({ turn: "not-a-turn", limitReason: "budget" as never, model: "gpt-5" as never }))); + assert.deepEqual([forged.turn, forged.limit_reason, forged.model], [undefined, undefined, undefined]); + assert.deepEqual(dedupeTurnUsageRows([{ turn, total: 1 }, { total: 2 }, { turn, total: 3 }, { total: 4 }]), [{ turn, total: 1 }, { total: 2 }, { total: 4 }]); +}); diff --git a/src/runtime/turnUsageLedger.ts b/src/runtime/turnUsageLedger.ts index 5ca205a..44e7621 100644 --- a/src/runtime/turnUsageLedger.ts +++ b/src/runtime/turnUsageLedger.ts @@ -1,6 +1,10 @@ import { constants } from "node:fs"; import { open, rename, stat } from "node:fs/promises"; +import { GROK_BROKER_MODELS } from "../contracts/grokWorkerContract.js"; +import { ENGINE_BROKER_LIMIT_REASONS, type EngineBrokerLimitReason } from "./engineBrokerTurnAccounting.js"; +import type { GrokBrokerModel } from "./grokBrokerModelPolicy.js"; + /** * Append-only per-turn token accounting for one container. * @@ -81,6 +85,7 @@ export const TURN_USAGE_OUTCOMES = ["completed", "failed"] as const; /** * - `token_ceiling` — the turn's own reported usage crossed the per-wake ceiling. + * - `request_ceiling` — the broker refused a model request past the turn's request limit. * - `wake_timeout` — the wall-clock bound fired before the child finished. * - `output_limit` — the retained-output bound was exceeded. * - `engine_exit` — the child exited non-zero (or died) after reporting usage. @@ -89,6 +94,7 @@ export const TURN_USAGE_OUTCOMES = ["completed", "failed"] as const; */ export const TURN_USAGE_FAILURE_REASONS = [ "token_ceiling", + "request_ceiling", "wake_timeout", "output_limit", "engine_exit", @@ -110,6 +116,15 @@ export type TurnUsageEntry = Readonly<{ usage: TurnUsageMeasurement; at?: string; outcome?: TurnUsageOutcome; + /** + * Broker rows only. `turn` is the idempotency key readers dedupe on (the + * broker turn id, a sha256 hex); `limitReason` and `model` come from the + * closed broker vocabularies and are dropped rather than written through + * when they are not members. + */ + turn?: string; + limitReason?: EngineBrokerLimitReason; + model?: GrokBrokerModel; }>; /** @@ -166,9 +181,26 @@ export const renderTurnUsageLine = (entry: TurnUsageEntry): string => `${JSON.st calls: entry.usage.calls, notional_usd: entry.usage.notionalUsd, complete: entry.usage.complete, - ...outcomeFields(entry.outcome) + ...outcomeFields(entry.outcome), + ...brokerFields(entry) })}\n`; +const brokerFields = (entry: TurnUsageEntry): Record => ({ + ...(entry.turn !== undefined && /^[a-f0-9]{64}$/u.test(entry.turn) ? { turn: entry.turn } : {}), + ...(entry.limitReason !== undefined && ENGINE_BROKER_LIMIT_REASONS.includes(entry.limitReason) ? { limit_reason: entry.limitReason } : {}), + ...(entry.model !== undefined && (GROK_BROKER_MODELS as readonly string[]).includes(entry.model) ? { model: entry.model } : {}) +}); + +/** + * Collapse rows that share a `turn` key to the first one, keeping rows without + * a key untouched. Every reader that sums the ledger applies this, because a + * turn key exists precisely so a re-appended turn can never be counted twice. + */ +export const dedupeTurnUsageRows = >(rows: readonly T[]): T[] => { + const seen = new Set(); + return rows.filter((row) => { if (typeof row.turn !== "string") return true; if (seen.has(row.turn)) return false; seen.add(row.turn); return true; }); +}; + const rotate = async (file: string): Promise => { let size: number; try { size = (await stat(file)).size; } catch { return; } diff --git a/src/runtime/wakeFuse.test.ts b/src/runtime/wakeFuse.test.ts index 9c6bf50..a526063 100644 --- a/src/runtime/wakeFuse.test.ts +++ b/src/runtime/wakeFuse.test.ts @@ -229,3 +229,15 @@ test("DAIMON_WAKE_FUSE=off never touches the usage ledger, missing or not", asyn const fuse = await WakeFuse.open({ organizationKey: "org", environment: environment(directory, { DAIMON_WAKE_FUSE: "off" }) }); assert.deepEqual(await fuse.admit("alpha", "one"), { state: "admitted" }); })); + +test("usage rows sharing a broker turn key count once toward the token ceiling", async () => await withDirectory(async (directory) => { + const turn = "b".repeat(64); + // 600 + 600 would trip a 1000-token ceiling; the duplicate turn row must not. + await writeFile(path.join(directory, "usage.jsonl"), [ + JSON.stringify({ at: "2026-08-30T00:00:00.000Z", total: 600, turn }), + JSON.stringify({ at: "2026-08-30T00:00:00.001Z", total: 600, turn }) + ].join("\n") + "\n"); + const now = () => new Date("2026-08-30T00:00:00.000Z"); + const fuse = await WakeFuse.open({ organizationKey: "org", environment: environment(directory), now }); + assert.deepEqual(await fuse.admit("alpha", "one"), { state: "admitted" }); +})); diff --git a/src/runtime/wakeFuse.ts b/src/runtime/wakeFuse.ts index 9786dff..d5d00f4 100644 --- a/src/runtime/wakeFuse.ts +++ b/src/runtime/wakeFuse.ts @@ -201,10 +201,13 @@ async function readFuseRecords(directory: string): Promise { let total = 0; + // Rows carrying a `turn` idempotency key count once per turn, whichever file holds them. + const turns = new Set(); for (const file of [`${ledgerPath}.1`, ledgerPath]) { for (const line of await lines(file)) { try { - const value = JSON.parse(line) as { at?: unknown; total?: unknown; agent?: unknown }; + const value = JSON.parse(line) as { at?: unknown; total?: unknown; agent?: unknown; turn?: unknown }; + if (typeof value.turn === "string") { if (turns.has(value.turn)) continue; turns.add(value.turn); } if ((agentId === undefined || value.agent === agentId) && typeof value.at === "string" && !Number.isNaN(Date.parse(value.at)) && value.at >= since && typeof value.total === "number" && Number.isFinite(value.total) && value.total >= 0) total += value.total; } catch { /* usage accounting is advisory input; malformed lines are skipped */ } } From 2bd8dedbcb795ff53b9152c71bfbcd9952c57096 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:13:24 +0200 Subject: [PATCH 03/21] feat: seal Grok broker usage in turn record v2, control protocol v2 and service config v2 with enforced turn limits --- src/runtime/engineBrokerControlClient.ts | 21 +- src/runtime/engineBrokerProtocol.test.ts | 45 +++- src/runtime/engineBrokerProtocol.ts | 74 ++++-- src/runtime/engineBrokerService.test.ts | 20 +- src/runtime/engineBrokerService.ts | 18 +- src/runtime/engineBrokerServiceCli.test.ts | 57 ++++- src/runtime/engineBrokerServiceCli.ts | 14 +- src/runtime/engineBrokerServiceConfig.ts | 69 ++++++ src/runtime/engineBrokerTurnRegistry.test.ts | 49 +++- src/runtime/engineBrokerTurnRegistry.ts | 63 ++++- src/runtime/grokBrokerProxy.test.ts | 22 +- src/runtime/grokBrokerProxy.ts | 34 ++- src/runtime/grokBrokerTurnMeter.test.ts | 103 +++++++++ src/runtime/grokBrokerTurnMeter.ts | 118 ++++++++++ src/runtime/grokEngineBroker.ts | 69 +++--- src/runtime/grokEngineBrokerMetering.ts | 43 ++++ src/runtime/grokEngineBrokerTurn.ts | 124 ++++++++++ src/runtime/grokEngineBrokerUsage.test.ts | 229 +++++++++++-------- 18 files changed, 952 insertions(+), 220 deletions(-) create mode 100644 src/runtime/engineBrokerServiceConfig.ts create mode 100644 src/runtime/grokBrokerTurnMeter.test.ts create mode 100644 src/runtime/grokBrokerTurnMeter.ts create mode 100644 src/runtime/grokEngineBrokerMetering.ts create mode 100644 src/runtime/grokEngineBrokerTurn.ts diff --git a/src/runtime/engineBrokerControlClient.ts b/src/runtime/engineBrokerControlClient.ts index 7136132..5054030 100644 --- a/src/runtime/engineBrokerControlClient.ts +++ b/src/runtime/engineBrokerControlClient.ts @@ -1,13 +1,22 @@ import { createHash, randomUUID } from "node:crypto"; import { createConnection } from "node:net"; -import { encodeEngineBrokerFrame,EngineBrokerFrameDecoder,parseEngineBrokerResponse } from "./engineBrokerProtocol.js"; +import { ENGINE_BROKER_VERSION, encodeEngineBrokerFrame,EngineBrokerFrameDecoder,parseEngineBrokerResponse } from "./engineBrokerProtocol.js"; +import type { EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; -export interface EngineBrokerTurnClient { turn(agentId:string,wakeId:string,prompt:string,mcpEndpoint:string,signal?:AbortSignal):Promise; } +/** + * What the organization runtime asks of a brokered turn beyond the prompt: + * `limits` may only lower the registration's declared limits, and `model`, + * when the agent declared one, must equal the model the broker sealed. + */ +export type EngineBrokerTurnOptions = Readonly<{ limits?: EngineBrokerTurnLimitOverrides; model?: string }>; +export interface EngineBrokerTurnClient { turn(agentId:string,wakeId:string,prompt:string,mcpEndpoint:string,signal?:AbortSignal,options?:EngineBrokerTurnOptions):Promise; } export class EngineBrokerControlClient implements EngineBrokerTurnClient { constructor(private readonly socketPath="/run/daimon-engine-broker/control.sock"){} - async ready():Promise{const requestId=randomUUID(),socket=createConnection({path:this.socketPath}),decoder=new EngineBrokerFrameDecoder();await new Promise((resolve,reject)=>{let settled=false;const fail=()=>{if(settled)return;settled=true;socket.destroy();reject(new Error("engine broker unavailable"));};socket.once("error",fail);socket.once("close",fail);socket.once("connect",()=>socket.write(encodeEngineBrokerFrame({version:"noopolis.daimon.engine-broker.v1",kind:"health",requestId})));socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const response=parseEngineBrokerResponse(value);if(response.kind!=="ready"||response.requestId!==requestId||settled)throw new Error();settled=true;socket.destroy();resolve();}}catch{fail();}});});} - async turn(agentId:string,wakeId:string,prompt:string,mcpEndpoint:string,signal?:AbortSignal):Promise{ - const turnId=createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"),requestId=randomUUID();const request={version:"noopolis.daimon.engine-broker.v1",kind:"start_turn",requestId,turnId,agentId,wakeId,prompt,mcpEndpoint} as const;const socket=createConnection({path:this.socketPath});const decoder=new EngineBrokerFrameDecoder(); - return new Promise((resolve,reject)=>{let accepted=false,settled=false;const fail=()=>{if(settled)return;settled=true;cleanup();reject(new Error("engine broker unavailable"));};const cleanup=()=>{signal?.removeEventListener("abort",abort);socket.destroy();};const abort=()=>fail();signal?.addEventListener("abort",abort,{once:true});if(signal?.aborted)return abort();socket.once("connect",()=>socket.write(encodeEngineBrokerFrame(request)));socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const response=parseEngineBrokerResponse(value);if(response.kind==="ready"||response.requestId!==requestId||response.turnId!==turnId)throw new Error();if(response.kind==="accepted"){if(accepted)throw new Error();accepted=true;continue;}if(!accepted||settled)throw new Error();settled=true;cleanup();if(response.kind==="completed")resolve(response.text);else reject(new Error(response.diagnostic ? `engine broker turn failed (${response.diagnostic.stage}/${response.diagnostic.failureClass}; exit=${response.diagnostic.exitCode}; signal=${response.diagnostic.termSignal})` : "engine broker turn failed"));}}catch{fail();}});socket.once("error",fail);socket.once("close",()=>{if(!settled)fail();});}); + async ready():Promise{const requestId=randomUUID(),socket=createConnection({path:this.socketPath}),decoder=new EngineBrokerFrameDecoder();await new Promise((resolve,reject)=>{let settled=false;const fail=()=>{if(settled)return;settled=true;socket.destroy();reject(new Error("engine broker unavailable"));};socket.once("error",fail);socket.once("close",fail);socket.once("connect",()=>socket.write(encodeEngineBrokerFrame({version:ENGINE_BROKER_VERSION,kind:"health",requestId})));socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const response=parseEngineBrokerResponse(value);if(response.kind!=="ready"||response.requestId!==requestId||settled)throw new Error();settled=true;socket.destroy();resolve();}}catch{fail();}});});} + async turn(agentId:string,wakeId:string,prompt:string,mcpEndpoint:string,signal?:AbortSignal,options:EngineBrokerTurnOptions={}):Promise{ + const turnId=createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"),requestId=randomUUID();const request={version:ENGINE_BROKER_VERSION,kind:"start_turn",requestId,turnId,agentId,wakeId,prompt,mcpEndpoint,...(options.limits===undefined?{}:{limits:options.limits})} as const;const socket=createConnection({path:this.socketPath});const decoder=new EngineBrokerFrameDecoder(); + return new Promise((resolve,reject)=>{let accepted=false,settled=false;const fail=()=>{if(settled)return;settled=true;cleanup();reject(new Error("engine broker unavailable"));};const cleanup=()=>{signal?.removeEventListener("abort",abort);socket.destroy();};const abort=()=>fail();signal?.addEventListener("abort",abort,{once:true});if(signal?.aborted)return abort();socket.once("connect",()=>socket.write(encodeEngineBrokerFrame(request)));socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const response=parseEngineBrokerResponse(value);if(response.kind==="ready"||response.requestId!==requestId||response.turnId!==turnId)throw new Error();if(response.kind==="accepted"){if(accepted)throw new Error();accepted=true;continue;}if(!accepted||settled)throw new Error();settled=true;cleanup(); + if(options.model!==undefined&&response.model!==options.model){reject(new Error(`engine broker turn used model ${response.model}, not the declared ${options.model}`));return;} + if(response.kind==="completed")resolve(response.text);else reject(new Error(`engine broker turn failed (${response.code}${response.limitReason==="none"?"":`; limit=${response.limitReason}`}${response.diagnostic ? `; ${response.diagnostic.stage}/${response.diagnostic.failureClass}; exit=${response.diagnostic.exitCode}; signal=${response.diagnostic.termSignal}` : ""})`));}}catch{fail();}});socket.once("error",fail);socket.once("close",()=>{if(!settled)fail();});}); } } diff --git a/src/runtime/engineBrokerProtocol.test.ts b/src/runtime/engineBrokerProtocol.test.ts index 1be3d53..ae762bc 100644 --- a/src/runtime/engineBrokerProtocol.test.ts +++ b/src/runtime/engineBrokerProtocol.test.ts @@ -1,8 +1,9 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { encodeEngineBrokerFrame, EngineBrokerFrameDecoder, parseEngineBrokerRequest, parseEngineBrokerResponse } from "./engineBrokerProtocol.js"; +import { encodeEngineBrokerFrame, EngineBrokerFrameDecoder, parseEngineBrokerRequest, parseEngineBrokerResponse, parseEngineBrokerV1TerminalResponse } from "./engineBrokerProtocol.js"; -const start = { version: "noopolis.daimon.engine-broker.v1", kind: "start_turn", requestId: "request-1", turnId: "turn-1", agentId: "agent-1", wakeId: "wake-1", prompt: "work",mcpEndpoint:"http://127.0.0.1:4567/mcp" } as const; +const accounting = { outcome: "completed", usage: { input: 20, cacheRead: 0, cacheWrite: 0, output: 10, total: 30 }, model: "grok-4.6", requests: 2, limitReason: "none" } as const; +const start = { version: "noopolis.daimon.engine-broker.v2", kind: "start_turn", requestId: "request-1", turnId: "turn-1", agentId: "agent-1", wakeId: "wake-1", prompt: "work",mcpEndpoint:"http://127.0.0.1:4567/mcp" } as const; test("broker frames survive arbitrary chunking and validate closed requests", () => { const encoded = encodeEngineBrokerFrame(start); const decoder = new EngineBrokerFrameDecoder(); const values: unknown[] = []; @@ -13,15 +14,51 @@ test("broker frames survive arbitrary chunking and validate closed requests", () }); test("broker response attestation is mandatory and bounded", () => { - const value = { version: start.version, kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 12, workerUid: 2200, workerStartTime: "12345" } as const; + const value = { version: start.version, kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 12, workerUid: 2200, workerStartTime: "12345", ...accounting } as const; assert.deepEqual(parseEngineBrokerResponse(value), value); assert.throws(() => parseEngineBrokerResponse({ ...value, workerUid: 0 }), /invalid broker frame/); const decoder = new EngineBrokerFrameDecoder(); assert.throws(() => decoder.push(Uint8Array.from([0, 16, 0, 1])), /invalid broker frame/); }); test("broker failure diagnostics are closed and contain no raw worker output",()=>{ - const value={version:start.version,kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",diagnostic:{status:"prelaunch_failed",stage:"executable",failureClass:"executable",profileApplied:false,exitCode:-1,termSignal:0,workerPid:0,workerUid:0,startTicks:"0"}} as const; + const value={version:start.version,kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",diagnostic:{status:"prelaunch_failed",stage:"executable",failureClass:"executable",profileApplied:false,exitCode:-1,termSignal:0,workerPid:0,workerUid:0,startTicks:"0"},outcome:"failed",usage:null,model:"grok-4.6",requests:0,limitReason:"none"} as const; assert.deepEqual(parseEngineBrokerResponse(value),value); assert.throws(()=>parseEngineBrokerResponse({...value,diagnostic:{...value.diagnostic,rawOutput:"secret"}}),/invalid broker frame/u); assert.throws(()=>parseEngineBrokerResponse({...value,diagnostic:{...value.diagnostic,failureClass:"secret"}}),/invalid broker frame/u); }); + +test("v2 terminal frames carry closed numeric accounting and refuse anything else", () => { + const completed = { version: start.version, kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 12, workerUid: 2200, workerStartTime: "12345", ...accounting } as const; + assert.deepEqual(parseEngineBrokerResponse(completed), completed); + for (const bad of [ + { ...completed, usage: { ...accounting.usage, total: 31 } }, + { ...completed, usage: { ...accounting.usage, note: "text" } }, + { ...completed, usage: { ...accounting.usage, input: "20" } }, + { ...completed, model: "grok-4.6-build" }, + { ...completed, outcome: "failed" }, + { ...completed, limitReason: "tokens" }, + { ...completed, limitReason: "budget" }, + { ...completed, requests: -1 }, + { ...completed, extra: 1 }, + (({ limitReason: _omit, ...rest }) => rest)(completed) + ]) assert.throws(() => parseEngineBrokerResponse(bad), /invalid broker frame/u); + const limit = { version: start.version, kind: "failed", requestId: "request-1", turnId: "turn-1", code: "limit_exceeded", outcome: "failed", usage: { input: 5, cacheRead: 5, cacheWrite: 0, output: 1, total: 11 }, model: "grok-4.6", requests: 3, limitReason: "requests" } as const; + assert.deepEqual(parseEngineBrokerResponse(limit), limit); + assert.throws(() => parseEngineBrokerResponse({ ...limit, limitReason: "none" }), /invalid broker frame/u); + assert.throws(() => parseEngineBrokerResponse({ ...limit, code: "engine_failed" }), /invalid broker frame/u); +}); + +test("v1 frames are refused on the wire but a v1 terminal record still parses", () => { + const v1 = { version: "noopolis.daimon.engine-broker.v1", kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 12, workerUid: 2200, workerStartTime: "12345" } as const; + assert.throws(() => parseEngineBrokerResponse(v1), /invalid broker frame/u); + assert.throws(() => parseEngineBrokerRequest({ ...start, version: "noopolis.daimon.engine-broker.v1" }), /invalid broker frame/u); + assert.deepEqual(parseEngineBrokerV1TerminalResponse(v1), v1); + assert.throws(() => parseEngineBrokerV1TerminalResponse({ ...v1, ...accounting }), /invalid broker frame/u); +}); + +test("start_turn limits are an optional closed subset inside their bounds", () => { + assert.deepEqual(parseEngineBrokerRequest({ ...start, limits: { maxTokens: 1_000 } }), { ...start, limits: { maxTokens: 1_000 } }); + for (const limits of [{}, { maxTokens: 0 }, { maxRequests: 49 }, { timeoutMs: 999 }, { maxTokens: 1, raise: true }, { maxTokens: 1.5 }]) { + assert.throws(() => parseEngineBrokerRequest({ ...start, limits }), /invalid broker frame/u); + } +}); diff --git a/src/runtime/engineBrokerProtocol.ts b/src/runtime/engineBrokerProtocol.ts index f8c79aa..d5de2f0 100644 --- a/src/runtime/engineBrokerProtocol.ts +++ b/src/runtime/engineBrokerProtocol.ts @@ -1,10 +1,21 @@ -const VERSION = "noopolis.daimon.engine-broker.v1" as const; +import { parseEngineBrokerTurnAccounting, parseEngineBrokerTurnLimitOverrides, type EngineBrokerTurnAccounting, type EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; + +/** + * Control protocol v2. Both ends ship in the same Daimon package and image + * (organization runtime client, broker service), so the wire moved to v2 in + * one step: v1 frames are refused. v1 survives only as a *durable record* + * shape, which {@link parseEngineBrokerV1TerminalResponse} still reads so + * turns sealed before the upgrade keep replaying. + */ +const VERSION = "noopolis.daimon.engine-broker.v2" as const; +export const ENGINE_BROKER_VERSION = VERSION; +const V1 = "noopolis.daimon.engine-broker.v1" as const; export const ENGINE_BROKER_MAX_FRAME_BYTES = 1_048_576; const ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; export type EngineBrokerRequest = | Readonly<{ version: typeof VERSION; kind: "health"; requestId: string }> - | Readonly<{ version: typeof VERSION; kind: "start_turn"; requestId: string; turnId: string; agentId: string; wakeId: string; prompt: string; mcpEndpoint: string }> + | Readonly<{ version: typeof VERSION; kind: "start_turn"; requestId: string; turnId: string; agentId: string; wakeId: string; prompt: string; mcpEndpoint: string; limits?: EngineBrokerTurnLimitOverrides }> | Readonly<{ version: typeof VERSION; kind: "cancel_turn"; requestId: string; turnId: string }>; export interface EngineBrokerFailureDiagnostic { status:string;stage:string;failureClass:string;profileApplied:boolean;exitCode:number;termSignal:number;workerPid:number;workerUid:number;startTicks:string } @@ -12,8 +23,15 @@ export interface EngineBrokerFailureDiagnostic { status:string;stage:string;fail export type EngineBrokerResponse = | Readonly<{ version: typeof VERSION; kind: "ready"; requestId: string; brokerUid: 2100; providerProxyPort: 43123; mcpFacadePort: 43124; registrations: number; credentialStale: false; realmLease: true; workerIsolation: true }> | Readonly<{ version: typeof VERSION; kind: "accepted"; requestId: string; turnId: string }> - | Readonly<{ version: typeof VERSION; kind: "completed"; requestId: string; turnId: string; text: string; workerPid: number; workerUid: number; workerStartTime: string }> - | Readonly<{ version: typeof VERSION; kind: "failed"; requestId: string; turnId: string; code: "auth_stale" | "cancelled" | "engine_failed" | "invalid_request" | "turn_conflict" | "unavailable"; diagnostic?: EngineBrokerFailureDiagnostic }>; + | (Readonly<{ version: typeof VERSION; kind: "completed"; requestId: string; turnId: string; text: string; workerPid: number; workerUid: number; workerStartTime: string }> & EngineBrokerTurnAccounting) + | (Readonly<{ version: typeof VERSION; kind: "failed"; requestId: string; turnId: string; code: EngineBrokerFailureCode; diagnostic?: EngineBrokerFailureDiagnostic }> & EngineBrokerTurnAccounting); +export const ENGINE_BROKER_FAILURE_CODES = ["auth_stale", "cancelled", "engine_failed", "invalid_request", "limit_exceeded", "turn_conflict", "unavailable"] as const; +export type EngineBrokerFailureCode = (typeof ENGINE_BROKER_FAILURE_CODES)[number]; +export type EngineBrokerTerminalResponse = Extract; +type V1Completed = Readonly<{ version: typeof V1; kind: "completed"; requestId: string; turnId: string; text: string; workerPid: number; workerUid: number; workerStartTime: string }>; +type V1Failed = Readonly<{ version: typeof V1; kind: "failed"; requestId: string; turnId: string; code: Exclude; diagnostic?: EngineBrokerFailureDiagnostic }>; +export type EngineBrokerV1TerminalResponse = V1Completed | V1Failed; +const ACCOUNTING = ["outcome", "usage", "model", "requests", "limitReason"] as const; type JsonRecord = Record; const record = (value: unknown): JsonRecord => { @@ -34,9 +52,12 @@ export function parseEngineBrokerRequest(value: unknown): EngineBrokerRequest { const input = record(value); version(input.version); if(input.kind==="health"){exact(input,["version","kind","requestId"]);return {version:VERSION,kind:"health",requestId:id(input.requestId)};} if (input.kind === "start_turn") { - exact(input, ["version", "kind", "requestId", "turnId", "agentId", "wakeId", "prompt", "mcpEndpoint"]); + const fields = ["version", "kind", "requestId", "turnId", "agentId", "wakeId", "prompt", "mcpEndpoint"]; + exact(input, input.limits === undefined ? fields : [...fields, "limits"]); const mcpEndpoint=text(input.mcpEndpoint,2048);const url=new URL(mcpEndpoint);if(url.protocol!=="http:"||url.hostname!=="127.0.0.1"||url.pathname!=="/mcp")throw new TypeError("invalid broker frame"); - return { version: VERSION, kind: "start_turn", requestId: id(input.requestId), turnId: id(input.turnId), agentId: id(input.agentId), wakeId: id(input.wakeId), prompt: text(input.prompt, 65_536),mcpEndpoint }; + let limits: EngineBrokerTurnLimitOverrides | undefined; + if (input.limits !== undefined) { try { limits = parseEngineBrokerTurnLimitOverrides(input.limits); } catch { throw new TypeError("invalid broker frame"); } } + return { version: VERSION, kind: "start_turn", requestId: id(input.requestId), turnId: id(input.turnId), agentId: id(input.agentId), wakeId: id(input.wakeId), prompt: text(input.prompt, 65_536),mcpEndpoint,...(limits === undefined ? {} : { limits }) }; } if (input.kind === "cancel_turn") { exact(input, ["version", "kind", "requestId", "turnId"]); @@ -52,20 +73,39 @@ export function parseEngineBrokerResponse(value: unknown): EngineBrokerResponse exact(input, ["version", "kind", "requestId", "turnId"]); return { version: VERSION, kind: "accepted", requestId: id(input.requestId), turnId: id(input.turnId) }; } + if (input.kind === "completed" || input.kind === "failed") return parseTerminal(input, VERSION) as EngineBrokerTerminalResponse; + throw new TypeError("invalid broker frame"); +} + +/** + * A terminal response persisted by a pre-v2 broker: the v1 field sets exactly, + * with no accounting. Accepted only from the durable turn registry, never from + * the wire. + */ +export function parseEngineBrokerV1TerminalResponse(value: unknown): EngineBrokerV1TerminalResponse { + const input = record(value); if (input.version !== V1 || (input.kind !== "completed" && input.kind !== "failed")) throw new TypeError("invalid broker frame"); + return parseTerminal(input, V1) as EngineBrokerV1TerminalResponse; +} + +function parseTerminal(input: JsonRecord, expected: typeof VERSION | typeof V1): EngineBrokerTerminalResponse | EngineBrokerV1TerminalResponse { + const accounting = expected === VERSION ? ACCOUNTING : []; if (input.kind === "completed") { - exact(input, ["version", "kind", "requestId", "turnId", "text", "workerPid", "workerUid", "workerStartTime"]); + exact(input, ["version", "kind", "requestId", "turnId", "text", "workerPid", "workerUid", "workerStartTime", ...accounting]); if (!Number.isSafeInteger(input.workerPid) || (input.workerPid as number) < 1 || !Number.isSafeInteger(input.workerUid) || (input.workerUid as number) < 1) throw new TypeError("invalid broker frame"); - return { version: VERSION, kind: "completed", requestId: id(input.requestId), turnId: id(input.turnId), text: text(input.text, 262_144), workerPid: input.workerPid as number, workerUid: input.workerUid as number, workerStartTime: id(input.workerStartTime) }; + const base = { kind: "completed", requestId: id(input.requestId), turnId: id(input.turnId), text: text(input.text, 262_144), workerPid: input.workerPid as number, workerUid: input.workerUid as number, workerStartTime: id(input.workerStartTime) } as const; + return expected === VERSION ? { version: VERSION, ...base, ...parseEngineBrokerTurnAccounting(input, "completed") } : { version: V1, ...base }; } - if (input.kind === "failed") { - exact(input, input.diagnostic === undefined ? ["version", "kind", "requestId", "turnId", "code"] : ["version", "kind", "requestId", "turnId", "code", "diagnostic"]); - const codes = ["auth_stale", "cancelled", "engine_failed", "invalid_request", "turn_conflict", "unavailable"] as const; - if (!codes.includes(input.code as typeof codes[number])) throw new TypeError("invalid broker frame"); - let diagnostic:EngineBrokerFailureDiagnostic|undefined; - if(input.diagnostic!==undefined){const value=record(input.diagnostic);exact(value,["status","stage","failureClass","profileApplied","exitCode","termSignal","workerPid","workerUid","startTicks"]);const status=["prelaunch_failed","worker_failed","output_failed","cancelled"],stage=["peer","request","registration","executable","exec","wait","output","attestation"],failureClass=["peer","protocol","registration","executable","exec","wait","output_limit","cancelled","profile_missing","profile_invalid"];if(!status.includes(value.status as string)||!stage.includes(value.stage as string)||!failureClass.includes(value.failureClass as string)||typeof value.profileApplied!=="boolean"||![value.exitCode,value.termSignal,value.workerPid,value.workerUid].every(Number.isSafeInteger)||typeof value.startTicks!=="string"||!/^(0|[1-9][0-9]*)$/u.test(value.startTicks)||!closedDiagnostic(value))throw new TypeError("invalid broker frame");diagnostic=value as unknown as EngineBrokerFailureDiagnostic;} - return { version: VERSION, kind: "failed", requestId: id(input.requestId), turnId: id(input.turnId), code: input.code as typeof codes[number],...(diagnostic?{diagnostic}:{}) }; - } - throw new TypeError("invalid broker frame"); + const fields = ["version", "kind", "requestId", "turnId", "code", ...accounting]; + exact(input, input.diagnostic === undefined ? fields : [...fields, "diagnostic"]); + const codes: readonly string[] = expected === VERSION ? ENGINE_BROKER_FAILURE_CODES : ENGINE_BROKER_FAILURE_CODES.filter((code) => code !== "limit_exceeded"); + if (!codes.includes(input.code as string)) throw new TypeError("invalid broker frame"); + let diagnostic:EngineBrokerFailureDiagnostic|undefined; + if(input.diagnostic!==undefined){const value=record(input.diagnostic);exact(value,["status","stage","failureClass","profileApplied","exitCode","termSignal","workerPid","workerUid","startTicks"]);const status=["prelaunch_failed","worker_failed","output_failed","cancelled"],stage=["peer","request","registration","executable","exec","wait","output","attestation"],failureClass=["peer","protocol","registration","executable","exec","wait","output_limit","cancelled","profile_missing","profile_invalid"];if(!status.includes(value.status as string)||!stage.includes(value.stage as string)||!failureClass.includes(value.failureClass as string)||typeof value.profileApplied!=="boolean"||![value.exitCode,value.termSignal,value.workerPid,value.workerUid].every(Number.isSafeInteger)||typeof value.startTicks!=="string"||!/^(0|[1-9][0-9]*)$/u.test(value.startTicks)||!closedDiagnostic(value))throw new TypeError("invalid broker frame");diagnostic=value as unknown as EngineBrokerFailureDiagnostic;} + const base = { kind: "failed", requestId: id(input.requestId), turnId: id(input.turnId), code: input.code as EngineBrokerFailureCode, ...(diagnostic ? { diagnostic } : {}) } as const; + if (expected === V1) return { version: V1, ...base } as V1Failed; + const accountingValue = parseEngineBrokerTurnAccounting(input, "failed"); + if ((input.code === "limit_exceeded") !== (accountingValue.limitReason !== "none")) throw new TypeError("invalid broker frame"); + return { version: VERSION, ...base, ...accountingValue }; } function closedDiagnostic(value:JsonRecord):boolean{ diff --git a/src/runtime/engineBrokerService.test.ts b/src/runtime/engineBrokerService.test.ts index e4fe882..d33f2bf 100644 --- a/src/runtime/engineBrokerService.test.ts +++ b/src/runtime/engineBrokerService.test.ts @@ -5,6 +5,9 @@ import path from "node:path"; import test from "node:test"; import { EngineBrokerControlClient } from "./engineBrokerControlClient.js"; import { startEngineBrokerServiceWithIdentity, type EngineBrokerServiceEngine } from "./engineBrokerService.js"; +import { EngineBrokerTurnFailure } from "./grokEngineBroker.js"; + +const completedAccounting = { outcome: "completed", usage: { input: 8, cacheRead: 2, cacheWrite: 0, output: 1, total: 11 }, model: "grok-4.6", requests: 1, limitReason: "none" } as const; test("broker backend serves a turn and preserves worker attestation", async () => { await withService(async (client) => {await client.ready();assert.equal(await client.turn("agent-a","wake-a","hello","http://127.0.0.1:44001/mcp"),"answer");}); @@ -33,8 +36,23 @@ test("service shutdown aborts turns and closes connected clients", async () => { await started;await service.close();await rejected;assert.equal(aborted,true);await rm(directory,{recursive:true,force:true}); }); -async function withService(run:(client:EngineBrokerControlClient)=>Promise,turn:EngineBrokerServiceEngine["turn"]=async()=>({text:"answer",workerPid:22,workerUid:2200,workerStartTime:"123"}),readiness:EngineBrokerServiceEngine["readiness"]=()=>({providerProxyPort:43123,mcpFacadePort:43124,registrations:1,credentialStale:false,realmLease:true,workerIsolation:true})):Promise{ +async function withService(run:(client:EngineBrokerControlClient)=>Promise,turn:EngineBrokerServiceEngine["turn"]=async()=>({text:"answer",workerPid:22,workerUid:2200,workerStartTime:"123",...completedAccounting}),readiness:EngineBrokerServiceEngine["readiness"]=()=>({providerProxyPort:43123,mcpFacadePort:43124,registrations:1,credentialStale:false,realmLease:true,workerIsolation:true})):Promise{ const directory=await mkdtemp(path.join(tmpdir(),"daimon-broker-service-")),socketPath=path.join(directory,"broker.sock"),engine=makeEngine(turn,readiness);const service=await startEngineBrokerServiceWithIdentity(engine,socketPath,process.getuid!()); try{await run(new EngineBrokerControlClient(socketPath));}finally{await service.close();await rm(directory,{recursive:true,force:true});} } function makeEngine(turn:EngineBrokerServiceEngine["turn"],readiness:EngineBrokerServiceEngine["readiness"]=()=>({providerProxyPort:43123,mcpFacadePort:43124,registrations:1,credentialStale:false,realmLease:true,workerIsolation:true})):EngineBrokerServiceEngine{return {turn,readiness,close:async()=>undefined};} + +test("the wake's lowering limits reach the broker, and the client verifies the declared model", async () => { + let seen: unknown; + await withService(async (client) => { + assert.equal(await client.turn("agent-a","wake-a","hello","http://127.0.0.1:44001/mcp",undefined,{limits:{maxTokens:1_000,timeoutMs:5_000},model:"grok-4.6"}),"answer"); + assert.deepEqual(seen,{maxTokens:1_000,timeoutMs:5_000}); + await assert.rejects(client.turn("agent-a","wake-b","hello","http://127.0.0.1:44001/mcp",undefined,{model:"grok-4.5"}),/model grok-4.6, not the declared grok-4.5/u); + },async(_agent,_wake,_prompt,_endpoint,_signal,limits)=>{seen=limits;return {text:"answer",workerPid:22,workerUid:2200,workerStartTime:"123",...completedAccounting};}); +}); + +test("a limit failure reaches the client with its code and limit reason", async () => { + await withService(async (client) => { + await assert.rejects(client.turn("agent-a","wake-a","hello","http://127.0.0.1:44001/mcp"),/engine broker turn failed \(limit_exceeded; limit=requests\)/u); + },async()=>{throw new EngineBrokerTurnFailure("limit_exceeded",undefined,{outcome:"failed",usage:{input:1,cacheRead:0,cacheWrite:0,output:1,total:2},model:"grok-4.6",requests:3,limitReason:"requests"});}); +}); diff --git a/src/runtime/engineBrokerService.ts b/src/runtime/engineBrokerService.ts index 9faa5c4..2b3f4c5 100644 --- a/src/runtime/engineBrokerService.ts +++ b/src/runtime/engineBrokerService.ts @@ -1,9 +1,10 @@ import { chmod, lstat, unlink } from "node:fs/promises"; import { createServer, type Server, type Socket } from "node:net"; import { encodeEngineBrokerFrame,EngineBrokerFrameDecoder,parseEngineBrokerRequest,type EngineBrokerResponse } from "./engineBrokerProtocol.js"; +import type { EngineBrokerTurnAccounting, EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; import { EngineBrokerTurnFailure } from "./grokEngineBroker.js"; export interface EngineBrokerServiceEngine { - turn(agentId:string,wakeId:string,prompt:string,mcpEndpoint:string,signal?:AbortSignal):Promise>; + turn(agentId:string,wakeId:string,prompt:string,mcpEndpoint:string,signal?:AbortSignal,limits?:EngineBrokerTurnLimitOverrides):Promise&EngineBrokerTurnAccounting>; readiness():Readonly<{providerProxyPort:number;mcpFacadePort:number;registrations:number;credentialStale:boolean;realmLease:boolean;workerIsolation:boolean}>; close():Promise; } @@ -36,7 +37,20 @@ export async function startEngineBrokerServiceWithIdentity(broker:EngineBrokerSe * reader has disconnected. */ function handleSocketError(socket:Socket,owned:()=>Readonly<{turnId:string;controller:AbortController}>|undefined):void{socket.on("error",()=>{owned()?.controller.abort();socket.destroy();});} -function handle(socket:Socket,broker:EngineBrokerServiceEngine,active:Map):void{const decoder=new EngineBrokerFrameDecoder();let started=false,owned:Readonly<{turnId:string;controller:AbortController}>|undefined;handleSocketError(socket,()=>owned);socket.once("close",()=>owned?.controller.abort());socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const request=parseEngineBrokerRequest(value);if(request.kind==="health"){if(started)throw new Error();started=true;const ready=broker.readiness();if(ready.providerProxyPort!==43123||ready.mcpFacadePort!==43124||ready.registrations<1||ready.credentialStale||!ready.realmLease||!ready.workerIsolation)throw new Error();return send(socket,{version:request.version,kind:"ready",requestId:request.requestId,brokerUid:2100,providerProxyPort:43123,mcpFacadePort:43124,registrations:ready.registrations,credentialStale:false,realmLease:true,workerIsolation:true});}if(request.kind==="cancel_turn"){active.get(request.turnId)?.abort();continue;}if(started||active.has(request.turnId))throw new Error();started=true;const controller=new AbortController();owned={turnId:request.turnId,controller};active.set(request.turnId,controller);socket.write(encodeEngineBrokerFrame({version:request.version,kind:"accepted",requestId:request.requestId,turnId:request.turnId}));void broker.turn(request.agentId,request.wakeId,request.prompt,request.mcpEndpoint,controller.signal).then((result)=>send(socket,{version:request.version,kind:"completed",requestId:request.requestId,turnId:request.turnId,...result}),(error:unknown)=>send(socket,{version:request.version,kind:"failed",requestId:request.requestId,turnId:request.turnId,code:error instanceof EngineBrokerTurnFailure?error.code:controller.signal.aborted?"cancelled":"engine_failed",...(error instanceof EngineBrokerTurnFailure&&error.diagnostic?{diagnostic:error.diagnostic}:{})})).finally(()=>{if(active.get(request.turnId)===controller)active.delete(request.turnId);owned=undefined;});}}catch{socket.destroy();}});} +function handle(socket:Socket,broker:EngineBrokerServiceEngine,active:Map):void{const decoder=new EngineBrokerFrameDecoder();let started=false,owned:Readonly<{turnId:string;controller:AbortController}>|undefined;handleSocketError(socket,()=>owned);socket.once("close",()=>owned?.controller.abort());socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const request=parseEngineBrokerRequest(value);if(request.kind==="health"){if(started)throw new Error();started=true;const ready=broker.readiness();if(ready.providerProxyPort!==43123||ready.mcpFacadePort!==43124||ready.registrations<1||ready.credentialStale||!ready.realmLease||!ready.workerIsolation)throw new Error();return send(socket,{version:request.version,kind:"ready",requestId:request.requestId,brokerUid:2100,providerProxyPort:43123,mcpFacadePort:43124,registrations:ready.registrations,credentialStale:false,realmLease:true,workerIsolation:true});}if(request.kind==="cancel_turn"){active.get(request.turnId)?.abort();continue;}if(started||active.has(request.turnId))throw new Error();started=true;const controller=new AbortController();owned={turnId:request.turnId,controller};active.set(request.turnId,controller);socket.write(encodeEngineBrokerFrame({version:request.version,kind:"accepted",requestId:request.requestId,turnId:request.turnId}));void broker.turn(request.agentId,request.wakeId,request.prompt,request.mcpEndpoint,controller.signal,request.limits).then((result)=>send(socket,{version:request.version,kind:"completed",requestId:request.requestId,turnId:request.turnId,text:result.text,workerPid:result.workerPid,workerUid:result.workerUid,workerStartTime:result.workerStartTime,outcome:"completed",usage:result.usage,model:result.model,requests:result.requests,limitReason:"none"}),(error:unknown)=>failed(socket,request,error,controller.signal.aborted)).finally(()=>{if(active.get(request.turnId)===controller)active.delete(request.turnId);owned=undefined;});}}catch{socket.destroy();}});} +/** + * Every failure the broker raises for a known registration carries its + * accounting (a wake that tried to raise a limit: `usage: null`, zero + * requests). A failure with no registration behind it (unknown agent, closed + * broker) has no declared model to report, so it is refused as a bare + * connection close and the client reports the broker unavailable. + */ +function failed(socket:Socket,request:Extract,{kind:"start_turn"}>,error:unknown,aborted:boolean):void{ + const accounting=error instanceof EngineBrokerTurnFailure?error.accounting:undefined; + if(accounting===undefined){socket.destroy();return;} + const code=error instanceof EngineBrokerTurnFailure?error.code:aborted?"cancelled":"engine_failed"; + send(socket,{version:request.version,kind:"failed",requestId:request.requestId,turnId:request.turnId,code,...(error instanceof EngineBrokerTurnFailure&&error.diagnostic?{diagnostic:error.diagnostic}:{}),outcome:"failed",usage:accounting.usage,model:accounting.model,requests:accounting.requests,limitReason:accounting.limitReason}); +} function send(socket:Socket,response:EngineBrokerResponse):void{if(!socket.destroyed)socket.end(encodeEngineBrokerFrame(response));} async function removeOwnedSocket(file:string,uid:number):Promise{try{const entry=await lstat(file);if(!entry.isSocket()||Number(entry.uid)!==uid)throw new Error("unsafe broker socket");await unlink(file);}catch(error){if((error as NodeJS.ErrnoException).code!=="ENOENT")throw error;}} async function verifySocket(file:string,uid:number):Promise{const entry=await lstat(file);if(!entry.isSocket()||Number(entry.uid)!==uid||(Number(entry.mode)&0o777)!==0o600)throw new Error("unsafe broker socket");} diff --git a/src/runtime/engineBrokerServiceCli.test.ts b/src/runtime/engineBrokerServiceCli.test.ts index 5d11cf6..db7771d 100644 --- a/src/runtime/engineBrokerServiceCli.test.ts +++ b/src/runtime/engineBrokerServiceCli.test.ts @@ -1,18 +1,55 @@ import assert from "node:assert/strict"; import test from "node:test"; import { parseEngineBrokerServiceConfig } from "./engineBrokerServiceCli.js"; +import { engineBrokerRequestLedgerPathFor } from "./engineBrokerServiceConfig.js"; -test("parses the closed broker service configuration",()=>{ - const registration=reg("agent-a",0);assert.deepEqual(parseEngineBrokerServiceConfig({version:"noopolis.daimon.engine-broker-service.v1",credentialHome:"/var/lib/daimon-engine-broker/credential",turnStore:"/var/lib/daimon-engine-broker/turns",registrations:[registration]}),{credentialHome:"/var/lib/daimon-engine-broker/credential",turnStore:"/var/lib/daimon-engine-broker/turns",registrations:[registration]}); +const paths = { credentialHome: "/var/lib/daimon-engine-broker/credential", turnStore: "/var/lib/daimon-engine-broker/turns" }; +const reg = (agentId: string, slot: number) => ({ agentId, slot, workerUid: 2200 + slot, workspace: `/workspace/${slot}`, profilePath: `/workers/${slot}/.grok/sandbox.toml`, eventsPath: `/workers/${slot}/.grok/sessions/sandbox-events.jsonl`, profileSha256: "a".repeat(64) }); +const v2 = (agentId: string, slot: number) => ({ ...reg(agentId, slot), usageLedgerPath: `/run/slots/${slot}/usage/usage.jsonl`, limits: { maxRequests: 12, maxTokens: 400_000, timeoutMs: 480_000 }, model: { id: "grok-4.6", reasoningEffort: "low" } }); +const config = (version: string, registrations: readonly unknown[]) => ({ version: `noopolis.daimon.engine-broker-service.${version}`, ...paths, registrations }); + +test("v1 service config is still accepted and receives today's defaults", () => { + assert.deepEqual(parseEngineBrokerServiceConfig(config("v1", [reg("agent-a", 0)])), { ...paths, registrations: [{ + ...reg("agent-a", 0), usageLedgerPath: "/var/lib/spawnfile/daimon/usage/usage.jsonl", + limits: { maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }, model: { model: "grok-4.6", reasoningEffort: "low" } + }] }); +}); + +test("v2 declares a per-slot ledger, limits and a closed-list model per registration", () => { + const parsed = parseEngineBrokerServiceConfig(config("v2", [v2("agent-a", 0), v2("agent-b", 1)])); + assert.deepEqual(parsed.registrations[1], { ...reg("agent-b", 1), usageLedgerPath: "/run/slots/1/usage/usage.jsonl", limits: { maxRequests: 12, maxTokens: 400_000, timeoutMs: 480_000 }, model: { model: "grok-4.6", reasoningEffort: "low" } }); + assert.equal(engineBrokerRequestLedgerPathFor(parsed.registrations[0]!.usageLedgerPath), "/run/slots/0/usage/requests.jsonl"); +}); + +test("v2 rejects unknown keys at every level, off-list models, missing fields and out-of-bound limits", () => { + const base = v2("agent-a", 0); + // Mutation guard: loosening any exact-member check accepts one of these. + for (const registration of [ + { ...base, extra: true }, + { ...base, limits: { ...base.limits, maxWakes: 1 } }, + { ...base, model: { ...base.model, provider: "xai" } }, + { ...base, model: { id: "grok-4.6-build", reasoningEffort: "low" } }, + { ...base, model: { id: "grok-4.6", reasoningEffort: "xhigh" } }, + { ...base, limits: { ...base.limits, maxRequests: 49 } }, + { ...base, limits: { ...base.limits, maxTokens: 0 } }, + { ...base, usageLedgerPath: "relative/usage.jsonl" }, + { ...base, usageLedgerPath: "/run/slots/0/requests.jsonl" }, + { ...base, usageLedgerPath: "/run/slots/0/usage" }, + (({ model: _omit, ...rest }) => rest)(base), + reg("agent-a", 0) + ]) assert.throws(() => parseEngineBrokerServiceConfig(config("v2", [registration])), /invalid engine broker service config/u); + assert.throws(() => parseEngineBrokerServiceConfig({ ...config("v2", [base]), grokCommand: "evil" }), /invalid engine broker service config/u); + assert.throws(() => parseEngineBrokerServiceConfig(config("v1", [base])), /invalid engine broker service config/u, "v2 members are not accepted under v1"); + assert.throws(() => parseEngineBrokerServiceConfig(config("v3", [base])), /invalid engine broker service config/u); }); -test("rejects caller-selected commands, duplicate identities, and traversal",()=>{ - const base={version:"noopolis.daimon.engine-broker-service.v1",credentialHome:"/var/lib/daimon-engine-broker/credential",turnStore:"/var/lib/daimon-engine-broker/turns",registrations:[reg("agent-a",0)]}; - assert.throws(()=>parseEngineBrokerServiceConfig({...base,grokCommand:"evil"})); - assert.throws(()=>parseEngineBrokerServiceConfig({...base,turnStore:"/var/lib/../secret"})); - assert.throws(()=>parseEngineBrokerServiceConfig({...base,registrations:[reg("agent-a",0),reg("agent-a",1)]})); +test("rejects caller-selected commands, duplicate identities, and traversal", () => { + const base = config("v1", [reg("agent-a", 0)]); + assert.throws(() => parseEngineBrokerServiceConfig({ ...base, grokCommand: "evil" })); + assert.throws(() => parseEngineBrokerServiceConfig({ ...base, turnStore: "/var/lib/../secret" })); + assert.throws(() => parseEngineBrokerServiceConfig({ ...base, registrations: [reg("agent-a", 0), reg("agent-a", 1)] })); + assert.throws(() => parseEngineBrokerServiceConfig({ ...base, registrations: [reg("agent-a", 0), reg("agent-b", 0)] }), /invalid/u, "one slot is one worker"); // Grok 1.0.34 logs sandbox events under $GROK_HOME/sessions/; the 1.0.13 root path stays empty and must not be attested. - assert.throws(()=>parseEngineBrokerServiceConfig({...base,registrations:[{...reg("agent-a",0),eventsPath:"/workers/0/.grok/sandbox-events.jsonl"}]})); - assert.throws(()=>parseEngineBrokerServiceConfig({...base,registrations:[{...reg("agent-a",0),eventsPath:"/workers/1/.grok/sessions/sandbox-events.jsonl"}]})); + assert.throws(() => parseEngineBrokerServiceConfig({ ...base, registrations: [{ ...reg("agent-a", 0), eventsPath: "/workers/0/.grok/sandbox-events.jsonl" }] })); + assert.throws(() => parseEngineBrokerServiceConfig({ ...base, registrations: [{ ...reg("agent-a", 0), eventsPath: "/workers/1/.grok/sessions/sandbox-events.jsonl" }] })); }); -const reg=(agentId:string,slot:number)=>({agentId,slot,workerUid:2200+slot,workspace:`/workspace/${slot}`,profilePath:`/workers/${slot}/.grok/sandbox.toml`,eventsPath:`/workers/${slot}/.grok/sessions/sandbox-events.jsonl`,profileSha256:"a".repeat(64)}); diff --git a/src/runtime/engineBrokerServiceCli.ts b/src/runtime/engineBrokerServiceCli.ts index d70839c..2a3967a 100644 --- a/src/runtime/engineBrokerServiceCli.ts +++ b/src/runtime/engineBrokerServiceCli.ts @@ -1,8 +1,10 @@ import { constants } from "node:fs"; import { open } from "node:fs/promises"; import { startEngineBrokerService } from "./engineBrokerService.js"; -import { startGrokEngineBroker, type GrokEngineBrokerRegistration } from "./grokEngineBroker.js"; -import { grokWorkerEventsPathFor } from "./grokWorkerSandboxProfile.js"; +import { parseEngineBrokerServiceConfig } from "./engineBrokerServiceConfig.js"; +import { startGrokEngineBroker } from "./grokEngineBroker.js"; + +export { parseEngineBrokerServiceConfig }; export const ENGINE_BROKER_SERVICE_CONFIG = "/etc/daimon-engine-broker/service.json"; const MAX_CONFIG_BYTES=65_536; @@ -16,12 +18,4 @@ export async function runEngineBrokerServiceCli():Promise{ const onSignal=()=>{void stop().catch(()=>{process.exitCode=1;});};process.once("SIGINT",onSignal);process.once("SIGTERM",onSignal); } -export function parseEngineBrokerServiceConfig(value:unknown):Readonly<{credentialHome:string;turnStore:string;registrations:readonly GrokEngineBrokerRegistration[]}>{ - if(value===null||typeof value!=="object"||Array.isArray(value))throw new TypeError("invalid engine broker service config");const input=value as Record; - if(Object.keys(input).length!==4||input.version!=="noopolis.daimon.engine-broker-service.v1"||typeof input.credentialHome!=="string"||typeof input.turnStore!=="string"||!Array.isArray(input.registrations))throw new TypeError("invalid engine broker service config"); - const absolute=(item:string)=>item.startsWith("/")&&!item.includes("/../")&&!item.endsWith("/..");if(!absolute(input.credentialHome)||!absolute(input.turnStore))throw new TypeError("invalid engine broker service config"); - const seen=new Set();const registrations=input.registrations.map((entry)=>{if(entry===null||typeof entry!=="object"||Array.isArray(entry))throw new TypeError("invalid engine broker service config");const item=entry as Record;if(Object.keys(item).length!==7||typeof item.agentId!=="string"||!item.agentId.trim()||!Number.isSafeInteger(item.slot)||(item.slot as number)<0||!Number.isSafeInteger(item.workerUid)||(item.workerUid as number)<2200||typeof item.workspace!=="string"||!absolute(item.workspace)||typeof item.profilePath!=="string"||!absolute(item.profilePath)||typeof item.eventsPath!=="string"||!absolute(item.eventsPath)||typeof item.profileSha256!=="string"||!/^[a-f0-9]{64}$/u.test(item.profileSha256)||item.eventsPath!==grokWorkerEventsPathFor(item.profilePath)||!item.profilePath.endsWith("/sandbox.toml")||seen.has(item.agentId))throw new TypeError("invalid engine broker service config");seen.add(item.agentId);return {agentId:item.agentId,slot:item.slot as number,workerUid:item.workerUid as number,workspace:item.workspace,profilePath:item.profilePath,eventsPath:item.eventsPath,profileSha256:item.profileSha256};}); - if(registrations.length===0)throw new TypeError("invalid engine broker service config");return {credentialHome:input.credentialHome,turnStore:input.turnStore,registrations}; -} - async function readRootConfig(file:string):Promise{const handle=await open(file,constants.O_RDONLY|constants.O_NOFOLLOW);try{const stat=await handle.stat();if(!stat.isFile()||stat.uid!==0||stat.gid!==2100||(stat.mode&0o777)!==0o440||stat.size<2||stat.size>MAX_CONFIG_BYTES)throw new Error("unsafe engine broker service config");const bytes=await handle.readFile();if(bytes.length>MAX_CONFIG_BYTES)throw new Error("unsafe engine broker service config");return JSON.parse(bytes.toString("utf8"));}finally{await handle.close();}} diff --git a/src/runtime/engineBrokerServiceConfig.ts b/src/runtime/engineBrokerServiceConfig.ts new file mode 100644 index 0000000..836963c --- /dev/null +++ b/src/runtime/engineBrokerServiceConfig.ts @@ -0,0 +1,69 @@ +import path from "node:path"; + +import { DEFAULT_GROK_BROKER_TURN_LIMITS, parseEngineBrokerTurnLimits, type EngineBrokerTurnLimits } from "./engineBrokerTurnAccounting.js"; +import { DEFAULT_GROK_BROKER_MODEL_POLICY, GROK_BROKER_MODELS, GROK_BROKER_REASONING_EFFORTS, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; +import { grokWorkerEventsPathFor } from "./grokWorkerSandboxProfile.js"; +import { TURN_USAGE_LEDGER } from "./turnUsageLedger.js"; + +export const ENGINE_BROKER_SERVICE_V1 = "noopolis.daimon.engine-broker-service.v1" as const; +export const ENGINE_BROKER_SERVICE_V2 = "noopolis.daimon.engine-broker-service.v2" as const; + +/** One root-provisioned broker slot. Every field is fixed at provisioning time; a wake can only lower `limits`. */ +export type EngineBrokerServiceRegistration = Readonly<{ + agentId: string; slot: number; workerUid: number; workspace: string; profilePath: string; eventsPath: string; profileSha256: string; + /** Per-slot usage ledger the broker appends turn rows to; per-request rows go to `requests.jsonl` beside it. */ + usageLedgerPath: string; + limits: EngineBrokerTurnLimits; + model: GrokBrokerModelPolicy; +}>; +export type EngineBrokerServiceConfig = Readonly<{ credentialHome: string; turnStore: string; registrations: readonly EngineBrokerServiceRegistration[] }>; + +const V1_REGISTRATION = ["agentId", "slot", "workerUid", "workspace", "profilePath", "eventsPath", "profileSha256"] as const; +const V2_REGISTRATION = [...V1_REGISTRATION, "usageLedgerPath", "limits", "model"] as const; +const invalid = (): TypeError => new TypeError("invalid engine broker service config"); +const plain = (value: unknown): value is Record => value !== null && typeof value === "object" && !Array.isArray(value); +const exact = (value: Record, fields: readonly string[]): void => { if (Object.keys(value).length !== fields.length || fields.some((field) => !Object.hasOwn(value, field))) throw invalid(); }; +const absolute = (item: unknown): item is string => typeof item === "string" && item.startsWith("/") && !item.includes("/../") && !item.endsWith("/..") && !item.includes("\0"); + +/** The per-request stream written beside a registration's usage ledger. */ +export const engineBrokerRequestLedgerPathFor = (usageLedgerPath: string): string => path.posix.join(path.posix.dirname(usageLedgerPath), "requests.jsonl"); + +/** + * Strict `service.json` parser. + * + * v2 requires every registration to declare its usage ledger, limits and model + * (`model.id`/`model.reasoningEffort` from the closed lists); unknown keys at + * any level are refused. v1 is still accepted and receives today's defaults: + * the container ledger, {@link DEFAULT_GROK_BROKER_TURN_LIMITS}, and + * `grok-4.6`/`low`. + */ +export function parseEngineBrokerServiceConfig(value: unknown): EngineBrokerServiceConfig { + if (!plain(value)) throw invalid(); + const v2 = value.version === ENGINE_BROKER_SERVICE_V2; + if (!v2 && value.version !== ENGINE_BROKER_SERVICE_V1) throw invalid(); + exact(value, ["version", "credentialHome", "turnStore", "registrations"]); + if (!absolute(value.credentialHome) || !absolute(value.turnStore) || !Array.isArray(value.registrations) || value.registrations.length === 0) throw invalid(); + const seen = new Set(), slots = new Set(); + const registrations = value.registrations.map((entry: unknown): EngineBrokerServiceRegistration => { + if (!plain(entry)) throw invalid(); + exact(entry, v2 ? V2_REGISTRATION : V1_REGISTRATION); + const { agentId, slot, workerUid, workspace, profilePath, eventsPath, profileSha256 } = entry; + if (typeof agentId !== "string" || !agentId.trim() || seen.has(agentId) || !Number.isSafeInteger(slot) || (slot as number) < 0 || slots.has(slot as number) || !Number.isSafeInteger(workerUid) || (workerUid as number) < 2200 || !absolute(workspace) || !absolute(profilePath) || !absolute(eventsPath) || typeof profileSha256 !== "string" || !/^[a-f0-9]{64}$/u.test(profileSha256) || eventsPath !== grokWorkerEventsPathFor(profilePath) || !profilePath.endsWith("/sandbox.toml")) throw invalid(); + seen.add(agentId); slots.add(slot as number); + const base = { agentId, slot: slot as number, workerUid: workerUid as number, workspace, profilePath, eventsPath, profileSha256 }; + if (!v2) return { ...base, usageLedgerPath: TURN_USAGE_LEDGER.filePath, limits: DEFAULT_GROK_BROKER_TURN_LIMITS, model: DEFAULT_GROK_BROKER_MODEL_POLICY }; + const usageLedgerPath = entry.usageLedgerPath; + if (!absolute(usageLedgerPath) || !usageLedgerPath.endsWith(".jsonl") || usageLedgerPath === engineBrokerRequestLedgerPathFor(usageLedgerPath) || path.posix.normalize(usageLedgerPath) !== usageLedgerPath) throw invalid(); + let limits: EngineBrokerTurnLimits; + try { limits = parseEngineBrokerTurnLimits(entry.limits); } catch { throw invalid(); } + return { ...base, usageLedgerPath, limits, model: parseServiceModel(entry.model) }; + }); + return { credentialHome: value.credentialHome, turnStore: value.turnStore, registrations }; +} + +function parseServiceModel(value: unknown): GrokBrokerModelPolicy { + if (!plain(value)) throw invalid(); + exact(value, ["id", "reasoningEffort"]); + if (!(GROK_BROKER_MODELS as readonly unknown[]).includes(value.id) || !(GROK_BROKER_REASONING_EFFORTS as readonly unknown[]).includes(value.reasoningEffort)) throw invalid(); + return Object.freeze({ model: value.id as GrokBrokerModelPolicy["model"], reasoningEffort: value.reasoningEffort as GrokBrokerModelPolicy["reasoningEffort"] }); +} diff --git a/src/runtime/engineBrokerTurnRegistry.test.ts b/src/runtime/engineBrokerTurnRegistry.test.ts index 16172c7..3de4e20 100644 --- a/src/runtime/engineBrokerTurnRegistry.test.ts +++ b/src/runtime/engineBrokerTurnRegistry.test.ts @@ -5,18 +5,49 @@ import path from "node:path"; import test from "node:test"; import { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; -const start = (prompt = "work") => ({ version: "noopolis.daimon.engine-broker.v1", kind: "start_turn", requestId: "request-1", turnId: "turn-1", agentId: "agent-1", wakeId: "wake-1", prompt,mcpEndpoint:"http://127.0.0.1:4567/mcp" } as const); +const start = (prompt = "work") => ({ version: "noopolis.daimon.engine-broker.v2", kind: "start_turn", requestId: "request-1", turnId: "turn-1", agentId: "agent-1", wakeId: "wake-1", prompt,mcpEndpoint:"http://127.0.0.1:4567/mcp" } as const); test("turn registry replays terminal results across restart and rejects conflicts", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-turns-")); try { - const first = new EngineBrokerTurnRegistry(root,"boot-a"); assert.equal(await first.begin(start()), "start"); - await assert.rejects(first.begin(start()), /already active/); - const response = { version: "noopolis.daimon.engine-broker.v1", kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 11, workerUid: 2200, workerStartTime: "123" } as const; + const first = new EngineBrokerTurnRegistry(root,"boot-a"); assert.equal(await first.begin(start(),"grok-4.6"), "start"); + await assert.rejects(first.begin(start(),"grok-4.6"), /already active/); + const response = { version: "noopolis.daimon.engine-broker.v2", kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 11, workerUid: 2200, workerStartTime: "123", outcome: "completed", usage: { input: 3, cacheRead: 2, cacheWrite: 0, output: 1, total: 6 }, model: "grok-4.6", requests: 1, limitReason: "none" } as const; await first.finish(start(), response); - assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-b").begin(start()), { replay: response }); - await assert.rejects(first.begin(start("different")), /conflict/); + assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-b").begin(start(),"grok-4.6"), { replay: response }); + await assert.rejects(first.begin(start("different"),"grok-4.6"), /conflict/); } finally { await rm(root, { recursive: true, force: true }); } }); -test("turn registry fails an orphaned active turn once after broker restart",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{assert.equal(await new EngineBrokerTurnRegistry(root,"boot-a").begin(start()),"start");const replay=await new EngineBrokerTurnRegistry(root,"boot-b").begin(start());assert.equal(typeof replay,"object");if(typeof replay==="object")assert.equal(replay.replay.kind,"failed");assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-c").begin(start()),replay);}finally{await rm(root,{recursive:true,force:true});}}); -test("turn registry durably replays a sanitized pre-attestation failure",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{const registry=new EngineBrokerTurnRegistry(root,"boot-a");assert.equal(await registry.begin(start()),"start");const response={version:"noopolis.daimon.engine-broker.v1",kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",diagnostic:{status:"worker_failed",stage:"attestation",failureClass:"profile_missing",profileApplied:false,exitCode:0,termSignal:0,workerPid:42,workerUid:2200,startTicks:"123"}} as const;await registry.finish(start(),response);assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-b").begin(start()),{replay:response});}finally{await rm(root,{recursive:true,force:true});}}); -test("turn registry rejects a persisted diagnostic with undeclared secret-bearing fields",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{const registry=new EngineBrokerTurnRegistry(root,"boot-a");assert.equal(await registry.begin(start()),"start");const [name]=await readdir(root);const file=path.join(root,name!);const record=JSON.parse(await readFile(file,"utf8")) as Record;record.state="terminal";record.response={version:"noopolis.daimon.engine-broker.v1",kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",rawOutput:"secret"};await writeFile(file,JSON.stringify(record));await assert.rejects(new EngineBrokerTurnRegistry(root,"boot-b").begin(start()),/invalid broker frame/u);}finally{await rm(root,{recursive:true,force:true});}}); +test("turn registry fails an orphaned active turn once after broker restart",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{assert.equal(await new EngineBrokerTurnRegistry(root,"boot-a").begin(start(),"grok-4.6"),"start");const replay=await new EngineBrokerTurnRegistry(root,"boot-b").begin(start(),"grok-4.6");assert.equal(typeof replay,"object");if(typeof replay==="object")assert.equal(replay.replay.kind,"failed");assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-c").begin(start(),"grok-4.6"),replay);}finally{await rm(root,{recursive:true,force:true});}}); +test("turn registry durably replays a sanitized pre-attestation failure",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{const registry=new EngineBrokerTurnRegistry(root,"boot-a");assert.equal(await registry.begin(start(),"grok-4.6"),"start");const response={version:"noopolis.daimon.engine-broker.v2",kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",diagnostic:{status:"worker_failed",stage:"attestation",failureClass:"profile_missing",profileApplied:false,exitCode:0,termSignal:0,workerPid:42,workerUid:2200,startTicks:"123"},outcome:"failed",usage:null,model:"grok-4.6",requests:0,limitReason:"none"} as const;await registry.finish(start(),response);assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-b").begin(start(),"grok-4.6"),{replay:response});}finally{await rm(root,{recursive:true,force:true});}}); +test("turn registry rejects a persisted diagnostic with undeclared secret-bearing fields",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{const registry=new EngineBrokerTurnRegistry(root,"boot-a");assert.equal(await registry.begin(start(),"grok-4.6"),"start");const [name]=await readdir(root);const file=path.join(root,name!);const record=JSON.parse(await readFile(file,"utf8")) as Record;record.state="terminal";record.response={version:"noopolis.daimon.engine-broker.v2",kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",rawOutput:"secret",outcome:"failed",usage:null,model:"grok-4.6",requests:0,limitReason:"none"};await writeFile(file,JSON.stringify(record));await assert.rejects(new EngineBrokerTurnRegistry(root,"boot-b").begin(start(),"grok-4.6"),/registry unavailable/u);}finally{await rm(root,{recursive:true,force:true});}}); + +const withRoot = async (run: (root: string) => Promise): Promise => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-turns-")); try { await run(root); } finally { await rm(root, { recursive: true, force: true }); } }; +const recordFile = async (root: string): Promise => { const [name] = await readdir(root); return path.join(root, name!); }; + +test("a v1 record sealed before the upgrade still replays, upgraded with no usage and never re-metered", async () => { + await withRoot(async (root) => { + const registry = new EngineBrokerTurnRegistry(root, "boot-a"); + assert.equal(await registry.begin(start(), "grok-4.5"), "start"); + const file = await recordFile(root); const record = JSON.parse(await readFile(file, "utf8")) as Record; + const v1 = { version: "noopolis.daimon.engine-broker.v1", kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 11, workerUid: 2200, workerStartTime: "123" }; + await writeFile(file, JSON.stringify({ version: "noopolis.daimon.engine-broker-turn.v1", digest: record.digest, state: "terminal", bootId: "boot-a", response: v1 })); + assert.deepEqual(await new EngineBrokerTurnRegistry(root, "boot-b").begin(start(), "grok-4.5"), { replay: { ...v1, version: "noopolis.daimon.engine-broker.v2", outcome: "completed", usage: null, model: "grok-4.5", requests: 0, limitReason: "none" } }); + }); +}); + +test("the v2 record parser is strict: an unknown member or a v1 frame inside a v2 record is refused", async () => { + await withRoot(async (root) => { + const registry = new EngineBrokerTurnRegistry(root, "boot-a"); + assert.equal(await registry.begin(start(), "grok-4.6"), "start"); + const response = { version: "noopolis.daimon.engine-broker.v2", kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 11, workerUid: 2200, workerStartTime: "123", outcome: "completed", usage: null, model: "grok-4.6", requests: 0, limitReason: "none" } as const; + await registry.finish(start(), response); + const file = await recordFile(root); const record = JSON.parse(await readFile(file, "utf8")) as Record; + assert.equal(record.version, "noopolis.daimon.engine-broker-turn.v2"); + // Mutation guard: dropping the exact-member check accepts this record. + await writeFile(file, JSON.stringify({ ...record, usageRow: "extra" })); + await assert.rejects(new EngineBrokerTurnRegistry(root, "boot-b").begin(start(), "grok-4.6"), /registry unavailable/u); + const { outcome: _o, usage: _u, model: _m, requests: _r, limitReason: _l, ...legacy } = response; + await writeFile(file, JSON.stringify({ ...record, response: { ...legacy, version: "noopolis.daimon.engine-broker.v1" } })); + await assert.rejects(new EngineBrokerTurnRegistry(root, "boot-b").begin(start(), "grok-4.6"), /registry unavailable/u); + }); +}); diff --git a/src/runtime/engineBrokerTurnRegistry.ts b/src/runtime/engineBrokerTurnRegistry.ts index bdee3c3..1ddd01c 100644 --- a/src/runtime/engineBrokerTurnRegistry.ts +++ b/src/runtime/engineBrokerTurnRegistry.ts @@ -2,33 +2,76 @@ import { createHash, randomUUID } from "node:crypto"; import { constants } from "node:fs"; import { mkdir, open, readFile, rename, unlink } from "node:fs/promises"; import path from "node:path"; -import { parseEngineBrokerResponse,type EngineBrokerRequest, type EngineBrokerResponse } from "./engineBrokerProtocol.js"; +import { parseEngineBrokerResponse, parseEngineBrokerV1TerminalResponse, type EngineBrokerRequest, type EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; +import type { GrokBrokerModel } from "./grokBrokerModelPolicy.js"; type Start = Extract; -type Terminal = Extract; -type Record = { version: "noopolis.daimon.engine-broker-turn.v1"; digest: string; state: "active" | "terminal"; bootId: string; response?: Terminal }; +type Terminal = EngineBrokerTerminalResponse; +export const ENGINE_BROKER_TURN_RECORD_V1 = "noopolis.daimon.engine-broker-turn.v1" as const; +export const ENGINE_BROKER_TURN_RECORD_V2 = "noopolis.daimon.engine-broker-turn.v2" as const; +// The digest deliberately excludes `limits` and the protocol version, so a v1 +// record written before the upgrade still identifies the same turn. const digest = (request: Start): string => createHash("sha256").update(JSON.stringify([request.turnId, request.agentId, request.wakeId, request.prompt,request.mcpEndpoint])).digest("hex"); const safe = (turnId: string): string => `${createHash("sha256").update(turnId).digest("hex")}.json`; +type Observed = Readonly<{ version: typeof ENGINE_BROKER_TURN_RECORD_V1 | typeof ENGINE_BROKER_TURN_RECORD_V2; digest: string; state: "active" | "terminal"; bootId: string; response?: Terminal }>; +/** + * Durable per-turn state. Record v2 stores the terminal response *with* its + * sealed accounting (usage, outcome, model, requests, limitReason), so a replay + * returns exactly what was metered and never meters again: metering happens + * only on the path that returned `"start"`. + */ export class EngineBrokerTurnRegistry { constructor(private readonly root: string,private readonly bootId:string=randomUUID()) {} - async begin(request: Start): Promise<"start" | { replay: Terminal }> { + /** `model` is the registration's declared model, used only to upgrade a v1 record on replay. */ + async begin(request: Start, model: GrokBrokerModel): Promise<"start" | { replay: Terminal }> { await mkdir(this.root, { recursive: true, mode: 0o700 }); const file = path.join(this.root, safe(request.turnId)); const expected = digest(request); - try { const handle = await open(file, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); try { await handle.writeFile(`${JSON.stringify({ version: "noopolis.daimon.engine-broker-turn.v1", digest: expected, state: "active",bootId:this.bootId })}\n`); await handle.sync(); } finally { await handle.close(); } await syncDirectory(this.root); return "start"; } + try { const handle = await open(file, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); try { await handle.writeFile(`${JSON.stringify({ version: ENGINE_BROKER_TURN_RECORD_V2, digest: expected, state: "active",bootId:this.bootId })}\n`); await handle.sync(); } finally { await handle.close(); } await syncDirectory(this.root); return "start"; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw new Error("broker turn registry unavailable"); } - const observed = JSON.parse(await readFile(file, "utf8")) as Record; - if (observed.version !== "noopolis.daimon.engine-broker-turn.v1" || observed.digest !== expected) throw new Error("broker turn conflict"); - if (observed.state === "terminal" && observed.response !== undefined) { const response=parseEngineBrokerResponse(observed.response);if(response.kind!=="completed"&&response.kind!=="failed")throw new Error("broker turn registry unavailable");return { replay: response }; } - if(observed.state==="active"&&observed.bootId!==this.bootId){const response={version:request.version,kind:"failed",requestId:request.requestId,turnId:request.turnId,code:"engine_failed"} as const;await this.finish(request,response);return {replay:response};} + const observed = parseEngineBrokerTurnRecord(await readFile(file, "utf8"), model); + if (observed.digest !== expected) throw new Error("broker turn conflict"); + if (observed.state === "terminal" && observed.response !== undefined) return { replay: observed.response }; + if(observed.state==="active"&&observed.bootId!==this.bootId){const response={version:request.version,kind:"failed",requestId:request.requestId,turnId:request.turnId,code:"engine_failed",outcome:"failed",usage:null,model,requests:0,limitReason:"none"} as const;await this.finish(request,response);return {replay:response};} throw new Error("broker turn already active"); } async finish(request: Start, response: Terminal): Promise { const file = path.join(this.root, safe(request.turnId)); const temporary = `${file}.${randomUUID()}.tmp`; const handle = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); - try { await handle.writeFile(`${JSON.stringify({ version: "noopolis.daimon.engine-broker-turn.v1", digest: digest(request), state: "terminal",bootId:this.bootId, response })}\n`); await handle.sync(); } finally { await handle.close(); } + try { await handle.writeFile(`${JSON.stringify({ version: ENGINE_BROKER_TURN_RECORD_V2, digest: digest(request), state: "terminal",bootId:this.bootId, response })}\n`); await handle.sync(); } finally { await handle.close(); } try { await rename(temporary, file); await syncDirectory(this.root); } finally { await unlink(temporary).catch(() => undefined); } } } +/** + * Strict record parser. v2 accepts exactly `{version,digest,state,bootId}` + * plus `response` when terminal, and the response must be a v2 terminal frame. + * v1 records keep their historical looser shape and are upgraded on read: no + * usage (`null`), zero requests, `limitReason: "none"`, the declared model. + */ +export function parseEngineBrokerTurnRecord(text: string, model: GrokBrokerModel): Observed { + let value: unknown; + try { value = JSON.parse(text); } catch { throw new Error("broker turn registry unavailable"); } + if (value === null || typeof value !== "object" || Array.isArray(value)) throw new Error("broker turn registry unavailable"); + const input = value as Record; + if (typeof input.digest !== "string" || !/^[a-f0-9]{64}$/u.test(input.digest) || typeof input.bootId !== "string" || (input.state !== "active" && input.state !== "terminal")) throw new Error("broker turn registry unavailable"); + const base = { digest: input.digest, state: input.state, bootId: input.bootId } as const; + if (input.version === ENGINE_BROKER_TURN_RECORD_V1) { + if (input.state !== "terminal" || input.response === undefined) return { version: ENGINE_BROKER_TURN_RECORD_V1, ...base }; + let legacy; + try { legacy = parseEngineBrokerV1TerminalResponse(input.response); } catch { throw new Error("broker turn registry unavailable"); } + const accounting = { outcome: legacy.kind, usage: null, model, requests: 0, limitReason: "none" } as const; + const response: Terminal = { ...legacy, ...accounting, version: "noopolis.daimon.engine-broker.v2" } as Terminal; + return { version: ENGINE_BROKER_TURN_RECORD_V1, ...base, response }; + } + if (input.version !== ENGINE_BROKER_TURN_RECORD_V2) throw new Error("broker turn conflict"); + const fields = input.state === "terminal" ? ["version", "digest", "state", "bootId", "response"] : ["version", "digest", "state", "bootId"]; + if (Object.keys(input).length !== fields.length || fields.some((field) => !Object.hasOwn(input, field))) throw new Error("broker turn registry unavailable"); + if (input.state === "active") return { version: ENGINE_BROKER_TURN_RECORD_V2, ...base }; + let response; + try { response = parseEngineBrokerResponse(input.response); } catch { throw new Error("broker turn registry unavailable"); } + if (response.kind !== "completed" && response.kind !== "failed") throw new Error("broker turn registry unavailable"); + return { version: ENGINE_BROKER_TURN_RECORD_V2, ...base, response }; +} + async function syncDirectory(directory: string): Promise { const handle = await open(directory, constants.O_RDONLY); try { await handle.sync(); } finally { await handle.close(); } } diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index 6c541b6..0e6e6de 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -3,6 +3,14 @@ import { request as httpRequest } from "node:http"; import test from "node:test"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; import { GROK_SESSION_TITLE_SINK_KEY } from "./grokBrokerWorkerConfig.js"; +import { GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; + +type Proxy = Awaited>; +const arm = (proxy: Proxy, guard: () => Promise, meter = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 })): GrokBrokerTurnMeter => { + proxy.registerIsolationGuard("turn", guard); + proxy.registerTurn("turn", { policy: { model: "grok-4.6", reasoningEffort: "low" }, meter }); + return meter; +}; const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); const leanBody = (overrides: Record = {}): string => JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: lean, ...overrides }); @@ -13,29 +21,29 @@ test("proxy retries one 401 with refreshed broker bearer and shuts down", async calls.push(request.headers.authorization); return calls.length === 1 ? { status: 401, headers: { "content-type": "application/json" }, body: new Uint8Array() } : { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from("data: done\n\n") }; }); const token = proxy.capabilities.issue("agent", "turn"); - proxy.registerIsolationGuard("turn", async () => undefined); + arm(proxy, async () => undefined); const result = await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`, { method: "POST", headers: { authorization: `Bearer ${token}`, "content-type": "application/json", "x-grok-client-version": "1.0.34" }, body: leanBody() }); assert.equal(result.status, 200); assert.equal(await result.text(), "data: done\n\n"); assert.deepEqual(calls, ["Bearer first", "Bearer second"]); assert.equal(refreshes, 0); await proxy.close(); await assert.rejects(fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`)); }); -test("proxy stale-fences a refreshed credential rejected by upstream",async()=>{let rejected=0;const proxy=await startGrokBrokerProxy({accessToken:async(force)=>force?"second":"first",markRejected:async()=>{rejected++;throw new Error("stale");}},async()=>({status:401,headers:{"content-type":"application/json"},body:new Uint8Array()}));const token=proxy.capabilities.issue("agent","turn");proxy.registerIsolationGuard("turn",async()=>undefined);const response=await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`,{method:"POST",headers:{authorization:`Bearer ${token}`,"x-grok-client-version":"1.0.34"},body:leanBody()});assert.equal(response.status,503);assert.equal(rejected,1);await proxy.close();}); +test("proxy stale-fences a refreshed credential rejected by upstream",async()=>{let rejected=0;const proxy=await startGrokBrokerProxy({accessToken:async(force)=>force?"second":"first",markRejected:async()=>{rejected++;throw new Error("stale");}},async()=>({status:401,headers:{"content-type":"application/json"},body:new Uint8Array()}));const token=proxy.capabilities.issue("agent","turn");arm(proxy, async()=>undefined);const response=await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`,{method:"POST",headers:{authorization:`Bearer ${token}`,"x-grok-client-version":"1.0.34"},body:leanBody()});assert.equal(response.status,503);assert.equal(rejected,1);await proxy.close();}); test("proxy failures expose only a fixed diagnostic", async () => { const proxy = await startGrokBrokerProxy({ accessToken: async () => { throw new Error("secret-token"); }, markRejected: async () => undefined }, async () => { throw new Error("unreachable"); }); const token = proxy.capabilities.issue("agent", "turn"); - proxy.registerIsolationGuard("turn", async () => undefined); + arm(proxy, async () => undefined); const result = await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`, { method: "POST", headers: { authorization: `Bearer ${token}`, "x-grok-client-version": "1.0.34" }, body: leanBody() }); assert.equal(result.status, 503); const body = await result.text(); assert.equal(body, '{"error":"broker unavailable"}'); assert.doesNotMatch(body, /secret/u); await proxy.close(); }); -test("one turn capability supports multiple guarded cognition requests",async()=>{let guarded=0,calls=0;const proxy=await startGrokBrokerProxy({accessToken:async()=>"provider-token",markRejected:async()=>undefined},async()=>{calls++;return{status:200,headers:{"content-type":"application/json"},body:Buffer.from("{}")};});try{const token=proxy.capabilities.issue("agent","turn");proxy.registerIsolationGuard("turn",async()=>{guarded++;});for(let index=0;index<2;index++){const response=await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`,{method:"POST",headers:{authorization:`Bearer ${token}`,"x-grok-client-version":"1.0.34"},body:leanBody()});assert.equal(response.status,200);}assert.equal(calls,2);assert.equal(guarded,2);}finally{await proxy.close();}}); +test("one turn capability supports multiple guarded cognition requests",async()=>{let guarded=0,calls=0;const proxy=await startGrokBrokerProxy({accessToken:async()=>"provider-token",markRejected:async()=>undefined},async()=>{calls++;return{status:200,headers:{"content-type":"application/json"},body:Buffer.from("{}")};});try{const token=proxy.capabilities.issue("agent","turn");arm(proxy, async()=>{guarded++;});for(let index=0;index<2;index++){const response=await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`,{method:"POST",headers:{authorization:`Bearer ${token}`,"x-grok-client-version":"1.0.34"},body:leanBody()});assert.equal(response.status,200);}assert.equal(calls,2);assert.equal(guarded,2);}finally{await proxy.close();}}); test("proxy refuses a fail-open tool set or an undeclared effort without calling upstream", async () => { let calls = 0; let accessed = 0; const proxy = await startGrokBrokerProxy({ accessToken: async () => { accessed++; return "provider-token"; }, markRejected: async () => undefined }, async () => { calls++; return { status: 200, headers: { "content-type": "application/json" }, body: Buffer.from("{}") }; }, { model: "grok-4.6", reasoningEffort: "low" }); try { - const token = proxy.capabilities.issue("agent", "turn"); proxy.registerIsolationGuard("turn", async () => undefined); + const token = proxy.capabilities.issue("agent", "turn"); arm(proxy, async () => undefined); const full = [...lean, ...["search_replace", "todo_write", "write", "monitor"].map((name) => ({ type: "function", function: { name } }))]; for (const payload of [leanBody({ tools: full }), leanBody({ tools: [{ type: "function", function: { name: "session_title" } }] }), leanBody({ reasoning_effort: "high" }), leanBody({ reasoning_effort: undefined })]) { assert.equal(await post(proxy.port, token, payload), 503); @@ -58,7 +66,7 @@ test("the session-title sink is refused before capability, guard, credential, or let calls = 0, accessed = 0, guarded = 0; const proxy = await startGrokBrokerProxy({ accessToken: async () => { accessed++; return "provider-token"; }, markRejected: async () => undefined }, async () => { calls++; return { status: 200, headers: { "content-type": "application/json" }, body: Buffer.from("{}") }; }); try { - const token = proxy.capabilities.issue("agent", "turn", 60_000, 1); proxy.registerIsolationGuard("turn", async () => { guarded++; }); + const token = proxy.capabilities.issue("agent", "turn", 60_000, 1); arm(proxy, async () => { guarded++; }); const title = JSON.stringify({ model: "disabled", max_tokens: 100, temperature: 0, stream: true, messages: [{ role: "user", content: "prompt-derived" }], tool_choice: { type: "function", function: { name: "session_title" } }, tools: [{ type: "function", function: { name: "session_title" } }] }); assert.equal(await post(proxy.port, GROK_SESSION_TITLE_SINK_KEY, title), 503); assert.deepEqual({ calls, accessed, guarded }, { calls: 0, accessed: 0, guarded: 0 }); @@ -73,7 +81,7 @@ test("the isolation guard is awaited before the first upstream call, and a faili const proxy = await startGrokBrokerProxy({ accessToken: async () => { order.push("credential"); return "provider-token"; }, markRejected: async () => undefined }, async () => { upstreamCalls++; order.push("upstream"); return { status: 200, headers: { "content-type": "application/json" }, body: Buffer.from("{}") }; }); try { const token = proxy.capabilities.issue("agent", "turn"); - proxy.registerIsolationGuard("turn", async () => { + arm(proxy, async () => { order.push("guard-start"); await new Promise((resolve) => setTimeout(resolve, 30)); order.push("guard-end"); if (fail) throw new Error("no enforcement evidence"); }); diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index 023a474..4f53103 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -4,28 +4,44 @@ import type { AddressInfo } from "node:net"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; import { DEFAULT_GROK_BROKER_MODEL_POLICY, parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; import { authorizeGrokBrokerProxyRequest } from "./grokBrokerProxyRequest.js"; +import { parseGrokUpstreamUsage, type GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; + +/** One running turn as the proxy sees it: its declared model/effort and its spend gate. */ +export type GrokBrokerProxyTurn = Readonly<{ policy: GrokBrokerModelPolicy; meter: GrokBrokerTurnMeter }>; export type GrokBrokerCredentialAuthority = Readonly<{ accessToken(forceRefresh: boolean): Promise; refreshAfterRejection?(rejectedTokenDigest:string):Promise; markRejected(rejectedTokenDigest?:string): Promise }>; export type GrokBrokerUpstream = (request: ReturnType) => Promise>; body: Uint8Array }>>; -/** `policy` is the declared model/effort every forwarded body must carry (closed list; defaults grok-4.6/low). */ -export async function startGrokBrokerProxy(authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream = defaultUpstream, policy: GrokBrokerModelPolicy = DEFAULT_GROK_BROKER_MODEL_POLICY): PromisePromise):void; revokeIsolationGuard(turnId:string):void; close(): Promise }>> { +/** + * `policy` is the fallback declared model/effort (closed list); a registered + * turn's own policy wins. A request whose turn has no registered meter is + * refused like one without an isolation guard: nothing is forwarded unmetered. + * `listenPort` exists for tests that must not contend for the production port. + */ +export async function startGrokBrokerProxy(authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream = defaultUpstream, policy: GrokBrokerModelPolicy = DEFAULT_GROK_BROKER_MODEL_POLICY, listenPort = 43_123): PromisePromise):void; revokeIsolationGuard(turnId:string):void; registerTurn(turnId:string,turn:GrokBrokerProxyTurn):void; revokeTurn(turnId:string):void; close(): Promise }>> { const declared = parseGrokBrokerModelPolicy(policy); const capabilities = new EngineBrokerCapabilities(); - const guards=new MapPromise>();const server = createServer((request, response) => { void serve(request, response, authority, upstream, capabilities,guards,declared); }); - await new Promise((resolve, reject) => { server.once("error", reject); server.listen(43_123, "127.0.0.1", () => { server.off("error", reject); resolve(); }); }); + const guards=new MapPromise>();const turns=new Map();const server = createServer((request, response) => { void serve(request, response, authority, upstream, capabilities,guards,turns,declared); }); + await new Promise((resolve, reject) => { server.once("error", reject); server.listen(listenPort, "127.0.0.1", () => { server.off("error", reject); resolve(); }); }); const address = server.address() as AddressInfo; - return { port: address.port, capabilities,registerIsolationGuard(turnId,guard){guards.set(turnId,guard);},revokeIsolationGuard(turnId){guards.delete(turnId);}, close: () => new Promise((resolve, reject) => server.close((error) => error === undefined ? resolve() : reject(error))) }; + return { port: address.port, capabilities,registerIsolationGuard(turnId,guard){guards.set(turnId,guard);},revokeIsolationGuard(turnId){guards.delete(turnId);},registerTurn(turnId,turn){turns.set(turnId,{policy:parseGrokBrokerModelPolicy(turn.policy),meter:turn.meter});},revokeTurn(turnId){turns.delete(turnId);}, close: () => new Promise((resolve, reject) => server.close((error) => error === undefined ? resolve() : reject(error))) }; } -async function serve(request: IncomingMessage, response: ServerResponse, authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream, capabilities: EngineBrokerCapabilities,guards:MapPromise>,policy:GrokBrokerModelPolicy): Promise { +async function serve(request: IncomingMessage, response: ServerResponse, authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream, capabilities: EngineBrokerCapabilities,guards:MapPromise>,turns:Map,fallback:GrokBrokerModelPolicy): Promise { + let settle:((usage:ReturnType)=>void)|undefined; try { const body = await readBody(request); const headers = Object.fromEntries(Object.entries(request.headers).map(([key, value]) => [key, Array.isArray(value) ? value[0] : value])); - const match=headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u),scope=match?capabilities.inspectToken(match[1]!):undefined;if(!scope)throw new Error();const guard=guards.get(scope.turnId);if(!guard)throw new Error();await guard(); - let token = await authority.accessToken(false);const rejectedDigest=createHash("sha256").update(token).digest("hex"); let prepared = authorizeGrokBrokerProxyRequest({ method: request.method ?? "", pathname: new URL(request.url ?? "/", "http://127.0.0.1").pathname, headers, body }, capabilities, token, policy); token = ""; + const match=headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u),scope=match?capabilities.inspectToken(match[1]!):undefined;if(!scope)throw new Error();const guard=guards.get(scope.turnId),turn=turns.get(scope.turnId);if(!guard||!turn)throw new Error();await guard(); + let token = await authority.accessToken(false);const rejectedDigest=createHash("sha256").update(token).digest("hex"); let prepared = authorizeGrokBrokerProxyRequest({ method: request.method ?? "", pathname: new URL(request.url ?? "/", "http://127.0.0.1").pathname, headers, body }, capabilities, token, turn.policy ?? fallback); token = ""; + // The spend gate runs after the body is proven a real lean worker request + // (a refused session-title body never counts) and before any upstream call. + const admission=turn.meter.admit(); + if("refused" in admission){response.writeHead(429,{"content-type":"application/json","cache-control":"no-store"});response.end(JSON.stringify({error:"turn limit reached",limit:admission.refused}));return;} + settle=(usage)=>{turn.meter.settle(admission.index,usage);settle=undefined;}; let result = await upstream(prepared); if (result.status === 401) { token = authority.refreshAfterRejection?await authority.refreshAfterRejection(rejectedDigest):await authority.accessToken(true);const refreshedDigest=createHash("sha256").update(token).digest("hex"); prepared = { ...prepared, headers: { ...prepared.headers, authorization: `Bearer ${token}` } }; token = ""; result = await upstream(prepared);if(result.status===401)await authority.markRejected(refreshedDigest); } + settle?.(parseGrokUpstreamUsage(result.body,result.headers["content-type"])); response.writeHead(result.status, { "content-type": result.headers["content-type"] ?? "application/json", "cache-control": "no-store" }); response.end(result.body); - } catch { response.writeHead(503, { "content-type": "application/json", "cache-control": "no-store" }); response.end('{"error":"broker unavailable"}'); } + } catch { settle?.(undefined); response.writeHead(503, { "content-type": "application/json", "cache-control": "no-store" }); response.end('{"error":"broker unavailable"}'); } } async function readBody(request: IncomingMessage): Promise { const chunks: Buffer[] = []; let bytes = 0; for await (const chunk of request) { const value = Buffer.from(chunk); bytes += value.length; if (bytes > 2 * 1024 * 1024) throw new Error("too large"); chunks.push(value); } return Buffer.concat(chunks); } const defaultUpstream: GrokBrokerUpstream = async (request) => { const result = await fetch(request.url, { method: "POST", headers: request.headers, body: Buffer.from(request.body) }); return { status: result.status, headers: { "content-type": result.headers.get("content-type") ?? "application/json" }, body: new Uint8Array(await result.arrayBuffer()) }; }; diff --git a/src/runtime/grokBrokerTurnMeter.test.ts b/src/runtime/grokBrokerTurnMeter.test.ts new file mode 100644 index 0000000..04c0a6a --- /dev/null +++ b/src/runtime/grokBrokerTurnMeter.test.ts @@ -0,0 +1,103 @@ +import assert from "node:assert/strict"; +import { request as httpRequest } from "node:http"; +import test from "node:test"; + +import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; +import { GrokBrokerTurnMeter, parseGrokUpstreamUsage } from "./grokBrokerTurnMeter.js"; + +const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); +const body = JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: lean }); +const sse = (usage: Record): Uint8Array => Buffer.from([{ choices: [{ index: 0, delta: { content: "x" } }] }, { choices: [], usage }].map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("") + "data: [DONE]\n\n"); + +function post(port: number, token: string): Promise> { + return new Promise((resolve, reject) => { + const req = httpRequest({ host: "127.0.0.1", port, path: "/v1/chat/completions", method: "POST", agent: false, headers: { authorization: `Bearer ${token}`, "x-grok-client-version": "1.0.34", "content-type": "application/json" } }, (response) => { + const chunks: Buffer[] = []; response.on("data", (chunk: Buffer) => chunks.push(chunk)); response.on("end", () => resolve({ status: response.statusCode ?? 0, text: Buffer.concat(chunks).toString("utf8") })); + }); + req.on("error", reject); req.end(body); + }); +} + +const withProxy = async (usage: Record | undefined, meter: GrokBrokerTurnMeter, run: (post: () => Promise>, calls: () => number) => Promise): Promise => { + let calls = 0; + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; return { status: 200, headers: { "content-type": "text/event-stream" }, body: usage === undefined ? Buffer.from("data: [DONE]\n\n") : sse(usage) }; }, undefined, 0); + try { + const token = proxy.capabilities.issue("agent", "turn"); + proxy.registerIsolationGuard("turn", async () => undefined); + proxy.registerTurn("turn", { policy: { model: "grok-4.6", reasoningEffort: "low" }, meter }); + await run(() => post(proxy.port, token), () => calls); + } finally { await proxy.close(); } +}; + +test("maxRequests is hard: request N+1 is refused with 429 before any upstream call, and the limit trips once", async () => { + const tripped: string[] = []; + const meter = new GrokBrokerTurnMeter({ maxRequests: 2, maxTokens: 1_000_000, timeoutMs: 60_000 }, (reason) => tripped.push(reason)); + await withProxy({ prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, meter, async (send, calls) => { + assert.equal((await send()).status, 200); + assert.equal((await send()).status, 200); + const refused = await send(); + assert.equal(refused.status, 429); + assert.deepEqual(JSON.parse(refused.text), { error: "turn limit reached", limit: "requests" }); + assert.equal((await send()).status, 429); + assert.equal(calls(), 2, "upstream never sees a request past maxRequests"); + }); + assert.deepEqual(tripped, ["requests"]); + const snapshot = meter.snapshot(); + assert.equal(snapshot.requests, 2); + assert.equal(snapshot.limitReason, "requests"); + assert.deepEqual(snapshot.usage, { input: 20, cacheRead: 0, cacheWrite: 0, output: 10, total: 30 }); +}); + +test("the token ceiling overshoots by at most one request, counting cached input", async () => { + // 60 tokens per request (40 cached) against a 100-token ceiling: requests 1 and + // 2 are admitted (0 and 60 < 100 before each), request 3 is refused at 120. + // Mutation guard: checking the ceiling after forwarding, or ignoring cached + // tokens, admits a third request and this goes red. + const meter = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 100, timeoutMs: 60_000 }); + await withProxy({ prompt_tokens: 50, completion_tokens: 10, total_tokens: 60, prompt_tokens_details: { cached_tokens: 40 } }, meter, async (send, calls) => { + const statuses = []; + for (let index = 0; index < 5; index++) statuses.push((await send()).status); + assert.deepEqual(statuses, [200, 200, 429, 429, 429]); + assert.equal(calls(), 2); + }); + const snapshot = meter.snapshot(); + assert.equal(snapshot.limitReason, "tokens"); + assert.equal(snapshot.tokens, 120); + assert.ok(snapshot.tokens - meter.limits.maxTokens <= 60, "overshoot is bounded by the last admitted request"); + assert.deepEqual(snapshot.usage, { input: 20, cacheRead: 80, cacheWrite: 0, output: 20, total: 120 }); +}); + +test("a request after the elapsed deadline is refused, and every admitted request is timed", async () => { + let now = 1_000_000; + const meter = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 1_000_000, timeoutMs: 5_000 }, undefined, () => now); + await withProxy(undefined, meter, async (send, calls) => { + assert.equal((await send()).status, 200); + now += 5_000; + assert.equal((await send()).status, 429); + assert.equal(calls(), 1); + }); + const snapshot = meter.snapshot(); + assert.equal(snapshot.limitReason, "timeout"); + assert.equal(snapshot.usage, null, "a body without usage contributes no invented zero"); + assert.deepEqual(snapshot.timings, [{ startedAt: new Date(1_000_000).toISOString(), endedAt: new Date(1_000_000).toISOString() }]); +}); + +test("a turn without a registered meter is never forwarded", async () => { + let calls = 0; + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; return { status: 200, headers: {}, body: Buffer.from("{}") }; }, undefined, 0); + try { + const token = proxy.capabilities.issue("agent", "turn"); + proxy.registerIsolationGuard("turn", async () => undefined); + assert.equal((await post(proxy.port, token)).status, 503); + assert.equal(calls, 0); + } finally { await proxy.close(); } +}); + +test("upstream usage parsing takes the last usage block and never zero-fills", () => { + assert.deepEqual(parseGrokUpstreamUsage(sse({ prompt_tokens: 100, completion_tokens: 7, total_tokens: 120, prompt_tokens_details: { cached_tokens: 30 }, completion_tokens_details: { reasoning_tokens: 13 } }), "text/event-stream"), + { input: 70, cacheRead: 30, cacheWrite: 0, output: 20, total: 120, reasoning: 13 }); + assert.deepEqual(parseGrokUpstreamUsage(Buffer.from(JSON.stringify({ usage: { prompt_tokens: 4, completion_tokens: 1 } })), "application/json"), { input: 4, cacheRead: 0, cacheWrite: 0, output: 1, total: 5 }); + assert.equal(parseGrokUpstreamUsage(sse({ prompt_tokens: "4", completion_tokens: 1 }), "text/event-stream"), undefined); + assert.equal(parseGrokUpstreamUsage(sse({ prompt_tokens: 4, completion_tokens: 1, prompt_tokens_details: { cached_tokens: 9 } }), "text/event-stream"), undefined); + assert.equal(parseGrokUpstreamUsage(Buffer.from("not json"), "application/json"), undefined); +}); diff --git a/src/runtime/grokBrokerTurnMeter.ts b/src/runtime/grokBrokerTurnMeter.ts new file mode 100644 index 0000000..203a9be --- /dev/null +++ b/src/runtime/grokBrokerTurnMeter.ts @@ -0,0 +1,118 @@ +import { sumEngineBrokerTurnUsage, type EngineBrokerLimitReason, type EngineBrokerTurnLimits, type EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; + +export type GrokBrokerRequestTiming = Readonly<{ startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage }>; +export type GrokBrokerTurnMeterSnapshot = Readonly<{ requests: number; tokens: number; limitReason: EngineBrokerLimitReason; usage: EngineBrokerTurnUsage | null; timings: readonly GrokBrokerRequestTiming[] }>; + +/** + * The proxy's per-turn spend gate. + * + * Every forwarded model request of one turn passes through {@link admit} + * *before* a bearer is attached, so the checks are hard for requests and + * elapsed time and between-requests for tokens: + * + * - `maxRequests`: request `maxRequests + 1` is refused; upstream never sees it. + * - `timeoutMs`: a request arriving after the deadline is refused (the broker's + * own timer additionally kills a worker that is mid-request). + * - `maxTokens`: checked against the running total of upstream-reported usage + * of the requests already answered. A request is admitted while that total is + * still below the ceiling, so the overshoot is bounded by exactly one + * request's usage — the last admitted one. Usage counts total input + * *including* cached tokens (P0 observed an uncached replay at +55%). + * + * The first limit that fires is sticky: every later request is refused with + * the same reason, and `onLimit` runs once. + */ +export class GrokBrokerTurnMeter { + private readonly startedAt: number; + private readonly timings: { startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage }[] = []; + private tokens = 0; + private reason: EngineBrokerLimitReason = "none"; + constructor(readonly limits: EngineBrokerTurnLimits, private readonly onLimit: (reason: Exclude) => void = () => undefined, private readonly now: () => number = Date.now) { + this.startedAt = now(); + } + + /** Returns the request index when admitted, or the limit that refused it. */ + admit(): Readonly<{ index: number } | { refused: Exclude }> { + if (this.reason === "none") { + if (this.now() - this.startedAt >= this.limits.timeoutMs) this.trip("timeout"); + else if (this.timings.length >= this.limits.maxRequests) this.trip("requests"); + else if (this.tokens >= this.limits.maxTokens) this.trip("tokens"); + } + if (this.reason !== "none") return { refused: this.reason }; + this.timings.push({ startedAt: new Date(this.now()).toISOString() }); + return { index: this.timings.length - 1 }; + } + + /** Records one admitted request's end and its upstream-reported usage, when the body carried any. */ + settle(index: number, usage: EngineBrokerTurnUsage | undefined): void { + const timing = this.timings[index]; + if (timing === undefined || timing.endedAt !== undefined) return; + timing.endedAt = new Date(this.now()).toISOString(); + if (usage === undefined) return; + timing.usage = usage; + this.tokens += usage.total; + } + + /** Trips a limit from outside the request path (the broker's wall-clock timer). */ + trip(reason: Exclude): void { + if (this.reason !== "none") return; + this.reason = reason; + this.onLimit(reason); + } + + snapshot(): GrokBrokerTurnMeterSnapshot { + const measured = this.timings.flatMap((timing) => timing.usage === undefined ? [] : [timing.usage]); + return { requests: this.timings.length, tokens: this.tokens, limitReason: this.reason, usage: sumEngineBrokerTurnUsage(measured), timings: this.timings.map((timing) => Object.freeze({ ...timing })) }; + } +} + +type JsonRecord = Record; +const isRecord = (value: unknown): value is JsonRecord => value !== null && typeof value === "object" && !Array.isArray(value); +const count = (value: unknown): number | undefined => typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : undefined; + +/** + * Upstream-reported usage of one chat-completions response, or `undefined`. + * + * The proxy buffers the whole upstream body, so the last `usage` object of an + * SSE stream (`stream_options.include_usage`) or of a JSON body is available + * before the body is returned to the worker. OpenAI-shaped `prompt_tokens` + * include cached tokens; they are split into disjoint buckets here, and any + * reasoning tokens reported outside `completion_tokens` (visible as + * `total_tokens` above prompt + completion) are folded into `output` so the + * total invariant holds. A malformed block is ignored, never zero-filled. + */ +export function parseGrokUpstreamUsage(body: Uint8Array, contentType: string | undefined): EngineBrokerTurnUsage | undefined { + const text = Buffer.from(body.buffer, body.byteOffset, body.byteLength).toString("utf8"); + const candidates: unknown[] = []; + if (contentType?.includes("text/event-stream") === true || text.startsWith("data:")) { + for (const line of text.split(/\r?\n/u)) { + if (!line.startsWith("data:")) continue; + const payload = line.slice(5).trim(); + if (payload === "[DONE]" || payload.length === 0) continue; + try { candidates.push(JSON.parse(payload)); } catch { /* a non-JSON event carries no usage */ } + } + } else { + try { candidates.push(JSON.parse(text)); } catch { return undefined; } + } + let found: EngineBrokerTurnUsage | undefined; + for (const candidate of candidates) { + if (!isRecord(candidate) || !isRecord(candidate.usage)) continue; + const decoded = decodeOpenAiUsage(candidate.usage); + if (decoded !== undefined) found = decoded; + } + return found; +} + +function decodeOpenAiUsage(usage: JsonRecord): EngineBrokerTurnUsage | undefined { + const prompt = count(usage.prompt_tokens), completion = count(usage.completion_tokens); + if (prompt === undefined || completion === undefined) return undefined; + const details = isRecord(usage.prompt_tokens_details) ? usage.prompt_tokens_details : {}; + const cached = details.cached_tokens === undefined ? 0 : count(details.cached_tokens); + const reported = usage.total_tokens === undefined ? prompt + completion : count(usage.total_tokens); + if (cached === undefined || reported === undefined || cached > prompt) return undefined; + const total = Math.max(reported, prompt + completion); + const completionDetails = isRecord(usage.completion_tokens_details) ? usage.completion_tokens_details : {}; + const reasoning = count(completionDetails.reasoning_tokens); + const output = total - prompt; + return { input: prompt - cached, cacheRead: cached, cacheWrite: 0, output, total, ...(reasoning === undefined || reasoning > output ? {} : { reasoning }) }; +} diff --git a/src/runtime/grokEngineBroker.ts b/src/runtime/grokEngineBroker.ts index 98a8384..d6d5125 100644 --- a/src/runtime/grokEngineBroker.ts +++ b/src/runtime/grokEngineBroker.ts @@ -1,57 +1,46 @@ -import { createHash, randomUUID } from "node:crypto"; -import { decodeGrokHeadlessTurn } from "../pi/grokHeadlessResult.js"; -import { recordTurnUsage, TURN_USAGE_LEDGER, type TurnUsageEntry } from "./turnUsageLedger.js"; import { DurableGrokBrokerCredentialAuthority } from "./grokBrokerCredentialAuthority.js"; -import { NativeBrokerTurnFailure, runNativeBrokerTurn, type NativeBrokerDiagnostic } from "./engineBrokerNativeClient.js"; +import { runNativeBrokerTurn } from "./engineBrokerNativeClient.js"; +import type { EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +import type { EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; import { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; import { startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; import { acquireGrokBrokerRealmLease } from "./grokBrokerRealmLease.js"; import { grokBrokerWorkerConfigSha256 } from "./grokBrokerWorkerConfig.js"; -import { parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; -import { createGrokWorkerIsolationGuard,GrokWorkerAttestationFailure,prepareGrokWorkerAttestation } from "./grokWorkerAttestation.js"; +import { parseGrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; +import { runGrokEngineBrokerTurn, type GrokEngineBrokerTurnResult } from "./grokEngineBrokerTurn.js"; +import { createGrokWorkerIsolationGuard,prepareGrokWorkerAttestation } from "./grokWorkerAttestation.js"; -export type GrokEngineBrokerRegistration = Readonly<{ agentId:string;slot:number;workerUid:number;workspace:string;profilePath:string;eventsPath:string;profileSha256:string }>; +export { EngineBrokerTurnFailure, type GrokEngineBrokerTurnResult } from "./grokEngineBrokerTurn.js"; +export { finishBrokerTurnWithUsage } from "./grokEngineBrokerMetering.js"; +export type GrokEngineBrokerRegistration = EngineBrokerServiceRegistration; export type GrokEngineBroker = Awaited>; -export class EngineBrokerTurnFailure extends Error{constructor(readonly code:"auth_stale"|"cancelled"|"engine_failed",readonly diagnostic?:NativeBrokerDiagnostic){super("engine broker turn failed");}} /** - * Seal a completed turn, then meter it. - * - * Order is load-bearing. `turns.finish` publishes the durable *completed* - * record; only after that does the advisory usage line get appended. A replayed - * turn returns before the enclosing `try` block and never reaches here, so a - * crash-recovered turn cannot double-count. Usage is deliberately kept out of - * the `completed` frame itself: that record is re-validated by the strict wire - * parser on the next `begin()`, whose exact field set would reject an extra key - * and break crash-recovery replay permanently. - * - * `recordTurnUsage` never rejects, so an append failure cannot escape into the - * caller's `catch` and rewrite this already-completed turn as failed. + * The Grok engine broker: one credential realm, one provider proxy, one MCP + * facade, and the root-provisioned registrations. Each registration declares + * its own model/effort (whose worker config bytes are attested), usage ledger, + * and turn limits (`engineBrokerServiceConfig.ts`). */ -export async function finishBrokerTurnWithUsage(turns: EngineBrokerTurnRegistry, request: Parameters[0], completed: Parameters[1], usageLedgerPath: string, usage: TurnUsageEntry["usage"] | undefined, agentId: string, wakeId: string): Promise { - await turns.finish(request, completed); - if (usage === undefined) return; - await recordTurnUsage(usageLedgerPath, { agent: agentId, wake: wakeId, engine: "grok", usage }); -} -export async function startGrokEngineBroker(options: Readonly<{ grokCommand: string; nativeClient: string; credentialHome: string; turnStore: string; registrations: readonly GrokEngineBrokerRegistration[]; usageLedgerPath?: string; modelPolicy?: GrokBrokerModelPolicy }>) { - // One declared model/effort drives both the worker config bytes the broker attests and the bodies the proxy forwards. - const modelPolicy = parseGrokBrokerModelPolicy(options.modelPolicy ?? {}); const configSha256 = grokBrokerWorkerConfigSha256(modelPolicy); - const usageLedgerPath = options.usageLedgerPath ?? TURN_USAGE_LEDGER.filePath; - const registrations = new Map(options.registrations.map((entry) => [entry.agentId, entry])); if (registrations.size !== options.registrations.length) throw new Error("engine broker registration conflict"); - const lease=await acquireGrokBrokerRealmLease(options.credentialHome);const authority = new DurableGrokBrokerCredentialAuthority(options.grokCommand, options.credentialHome);try{await authority.initialize();}catch(error){await lease.close();throw error;} let proxy:Awaited>;try{proxy=await startGrokBrokerProxy(authority,undefined,modelPolicy);}catch(error){await lease.close();throw error;}let mcp:Awaited>|undefined;try{mcp=await startEngineBrokerMcpFacade();for(const registration of registrations.values())await prepareGrokWorkerAttestation({...registration,brokerGid:2100,configSha256});}catch(error){if(mcp)await mcp.close().catch(()=>undefined);await proxy.close();await lease.close();throw error;}if(!mcp)throw new Error("engine broker unavailable");const turns = new EngineBrokerTurnRegistry(options.turnStore); const active = new Map; resolve: () => void }>(); let closed = false; +export async function startGrokEngineBroker(options: Readonly<{ grokCommand: string; nativeClient: string; credentialHome: string; turnStore: string; registrations: readonly GrokEngineBrokerRegistration[] }>) { + const registrations = new Map(options.registrations.map((entry) => [entry.agentId, { ...entry, model: parseGrokBrokerModelPolicy(entry.model) }])); if (registrations.size !== options.registrations.length) throw new Error("engine broker registration conflict"); + const attestationFor = (registration: GrokEngineBrokerRegistration) => ({ ...registration, brokerGid: 2100, configSha256: grokBrokerWorkerConfigSha256(registration.model) }); + const lease=await acquireGrokBrokerRealmLease(options.credentialHome);const authority = new DurableGrokBrokerCredentialAuthority(options.grokCommand, options.credentialHome);try{await authority.initialize();}catch(error){await lease.close();throw error;} let proxy:Awaited>;try{proxy=await startGrokBrokerProxy(authority);}catch(error){await lease.close();throw error;}let mcp:Awaited>|undefined;try{mcp=await startEngineBrokerMcpFacade();for(const registration of registrations.values())await prepareGrokWorkerAttestation(attestationFor(registration));}catch(error){if(mcp)await mcp.close().catch(()=>undefined);await proxy.close();await lease.close();throw error;}if(!mcp)throw new Error("engine broker unavailable");const turns = new EngineBrokerTurnRegistry(options.turnStore); const active = new Map }>(); let closed = false; + const facade = mcp; + const deps = { + turns, proxy, mcp: facade, credentialStale: () => authority.isStale(), + prepareIsolation: async (registration: GrokEngineBrokerRegistration) => { const attestation = attestationFor(registration); return createGrokWorkerIsolationGuard(attestation, await prepareGrokWorkerAttestation(attestation)); }, + runNative: (input: Parameters[1], signal: AbortSignal) => runNativeBrokerTurn(options.nativeClient, input, signal) + }; return { - async turn(agentId: string, wakeId: string, prompt: string, mcpEndpoint: string, signal?: AbortSignal): Promise> { + async turn(agentId: string, wakeId: string, prompt: string, mcpEndpoint: string, signal?: AbortSignal, limits?: EngineBrokerTurnLimitOverrides): Promise { if (closed) throw new Error("engine broker unavailable"); const registration = registrations.get(agentId); if (registration === undefined) throw new Error("engine broker unavailable"); - const turnId = createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"); const request = { version: "noopolis.daimon.engine-broker.v1", kind: "start_turn", requestId: randomUUID(), turnId, agentId, wakeId, prompt,mcpEndpoint } as const; - const begun = await turns.begin(request); if (begun !== "start") { if (begun.replay.kind === "completed") return {text:begun.replay.text,workerPid:begun.replay.workerPid,workerUid:begun.replay.workerUid,workerStartTime:begun.replay.workerStartTime};const code=begun.replay.code==="auth_stale"||begun.replay.code==="cancelled"?begun.replay.code:"engine_failed";throw new EngineBrokerTurnFailure(code,begun.replay.diagnostic as NativeBrokerDiagnostic|undefined); } - const attestation={...registration,brokerGid:2100,configSha256};const isolation=await prepareGrokWorkerAttestation(attestation);const isolationGuard=createGrokWorkerIsolationGuard(attestation,isolation);proxy.registerIsolationGuard(turnId,isolationGuard);const providerCapability = proxy.capabilities.issue(agentId, turnId);const mcpCapability=mcp.register(agentId,turnId,mcpEndpoint); const controller = new AbortController();const onAbort=()=>controller.abort();signal?.addEventListener("abort",onAbort,{once:true});if(signal?.aborted)controller.abort(); let resolve!:()=>void;const done=new Promise((value)=>{resolve=value;});active.set(turnId,{controller,done,resolve}); - let nativeDiagnostic:NativeBrokerDiagnostic|undefined,attested=false; - try { const result = await runNativeBrokerTurn(options.nativeClient, { slot: registration.slot, requestId: request.requestId, turnId, agentId, wakeId, prompt, providerCapability,mcpCapability }, controller.signal);nativeDiagnostic=result.diagnostic;if(result.workerUid!==registration.workerUid)throw new Error();await isolationGuard();attested=true; const decoded = decodeGrokHeadlessTurn(result.text);const text = decoded.text;const completed={ version: request.version, kind: "completed", requestId: request.requestId, turnId, text, workerPid: result.workerPid, workerUid: result.workerUid, workerStartTime: result.startTicks.toString() } as const; await finishBrokerTurnWithUsage(turns,request,completed,usageLedgerPath,decoded.usage,agentId,wakeId); return {text,workerPid:result.workerPid,workerUid:result.workerUid,workerStartTime:result.startTicks.toString()}; } - catch(error) { const code=authority.isStale()?"auth_stale":controller.signal.aborted?"cancelled":"engine_failed";const diagnostic=error instanceof NativeBrokerTurnFailure?error.diagnostic:nativeDiagnostic&&!attested?{...nativeDiagnostic,status:"worker_failed" as const,stage:"attestation" as const,failureClass:error instanceof GrokWorkerAttestationFailure?error.failureClass:"profile_invalid" as const,profileApplied:false}:undefined;await turns.finish(request, { version: request.version, kind: "failed", requestId: request.requestId, turnId, code,...(diagnostic?{diagnostic}:{}) }); throw new EngineBrokerTurnFailure(code,diagnostic); } - finally { signal?.removeEventListener("abort",onAbort);active.get(turnId)?.resolve(); active.delete(turnId); proxy.revokeIsolationGuard(turnId);proxy.capabilities.revoke(turnId);mcp.revoke(turnId); } + const controller = new AbortController(); const onAbort = () => controller.abort(); signal?.addEventListener("abort", onAbort, { once: true }); if (signal?.aborted) controller.abort(); + const key = `${agentId}\0${wakeId}`; const done = runGrokEngineBrokerTurn(deps, registration, wakeId, prompt, mcpEndpoint, controller.signal, limits); + active.set(key, { controller, done: done.then(() => undefined, () => undefined) }); + try { return await done; } finally { signal?.removeEventListener("abort", onAbort); active.delete(key); } }, - async close(): Promise { if (closed) return; closed = true; const running=[...active.values()];for (const entry of running) entry.controller.abort();await Promise.allSettled(running.map((entry)=>entry.done));const results=await Promise.allSettled([mcp.close(),proxy.close(),lease.close()]);const failures=results.flatMap((entry)=>entry.status==="rejected"?[entry.reason]:[]);if(failures.length)throw new AggregateError(failures,"engine broker shutdown failed"); }, + async close(): Promise { if (closed) return; closed = true; const running=[...active.values()];for (const entry of running) entry.controller.abort();await Promise.allSettled(running.map((entry)=>entry.done));const results=await Promise.allSettled([facade.close(),proxy.close(),lease.close()]);const failures=results.flatMap((entry)=>entry.status==="rejected"?[entry.reason]:[]);if(failures.length)throw new AggregateError(failures,"engine broker shutdown failed"); }, readiness: () => ({ providerProxyPort: proxy.port, mcpFacadePort:43_124, registrations: registrations.size,credentialStale:authority.isStale(),realmLease:true,workerIsolation:true }) }; } diff --git a/src/runtime/grokEngineBrokerMetering.ts b/src/runtime/grokEngineBrokerMetering.ts new file mode 100644 index 0000000..f254554 --- /dev/null +++ b/src/runtime/grokEngineBrokerMetering.ts @@ -0,0 +1,43 @@ +import type { EngineBrokerRequest, EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; +import type { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; +import { recordGrokTurnRequests, type GrokTurnRequest } from "./turnRequestLedger.js"; +import { recordTurnUsage, type TurnUsageFailureReason } from "./turnUsageLedger.js"; + +export type BrokerTurnMetering = Readonly<{ + usageLedgerPath: string; + requestLedgerPath: string; + agentId: string; + wakeId: string; +}>; +export type BrokerTurnMeteringDetail = Readonly<{ notionalUsd: number; complete: boolean; reason?: TurnUsageFailureReason; requests: readonly GrokTurnRequest[]; session?: string }>; + +/** + * Seal a terminal turn, then meter it. The broker is the single writer. + * + * Order is load-bearing. `turns.finish` publishes the durable terminal record + * *with* its accounting; only after that are the advisory ledger rows appended. + * A replayed turn returns before the broker's `try` block and never reaches + * here, so a crash-recovered or repeated turn cannot double-count; every row + * also carries the turn id as `turn`, so a reader that sees one twice counts + * it once. + * + * Both terminal kinds meter: a failed turn spent real tokens, so its partial + * usage is written with `outcome: failed` and its closed `limitReason`. A turn + * with no usage at all (`usage: null`) writes nothing — a zero row is + * byte-identical to a measured zero. + * + * `recordTurnUsage`/`recordGrokTurnRequests` never reject, so an append failure + * cannot escape into the caller's `catch` and rewrite a completed turn as failed. + */ +export async function finishBrokerTurnWithUsage(turns: EngineBrokerTurnRegistry, request: Extract, terminal: EngineBrokerTerminalResponse, metering: BrokerTurnMetering, detail: BrokerTurnMeteringDetail): Promise { + await turns.finish(request, terminal); + if (terminal.usage === null) return; + const { usage } = terminal; + await recordTurnUsage(metering.usageLedgerPath, { + agent: metering.agentId, wake: metering.wakeId, engine: "grok", + usage: { input: usage.input, output: usage.output, cacheRead: usage.cacheRead, cacheWrite: usage.cacheWrite, total: usage.total, calls: terminal.requests, notionalUsd: detail.notionalUsd, complete: detail.complete }, + outcome: terminal.kind === "completed" ? { status: "completed" } : { status: "failed", reason: detail.reason ?? "unknown" }, + turn: terminal.turnId, limitReason: terminal.limitReason, model: terminal.model + }); + await recordGrokTurnRequests(metering.requestLedgerPath, { agent: metering.agentId, wake: metering.wakeId, turn: terminal.turnId, model: terminal.model, requests: detail.requests, ...(detail.session === undefined ? {} : { session: detail.session }) }); +} diff --git a/src/runtime/grokEngineBrokerTurn.ts b/src/runtime/grokEngineBrokerTurn.ts new file mode 100644 index 0000000..105ff02 --- /dev/null +++ b/src/runtime/grokEngineBrokerTurn.ts @@ -0,0 +1,124 @@ +import { createHash, randomUUID } from "node:crypto"; + +import { decodeGrokHeadlessTurn } from "../pi/grokHeadlessResult.js"; +import { decodeGrokStreamUsage, type GrokStreamUsage } from "../pi/grokStreamUsage.js"; +import { ENGINE_BROKER_VERSION, type EngineBrokerFailureCode, type EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; +import { lowerEngineBrokerTurnLimits, mapGrokReportedModel, type EngineBrokerTurnAccounting, type EngineBrokerTurnLimitOverrides, type EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; +import { NativeBrokerTurnFailure, type NativeBrokerDiagnostic, type NativeBrokerTurn, type NativeBrokerTurnResult } from "./engineBrokerNativeClient.js"; +import type { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; +import { engineBrokerRequestLedgerPathFor, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +import { finishBrokerTurnWithUsage, type BrokerTurnMetering, type BrokerTurnMeteringDetail } from "./grokEngineBrokerMetering.js"; +import type { GrokBrokerProxyTurn } from "./grokBrokerProxy.js"; +import { GrokBrokerTurnMeter, type GrokBrokerTurnMeterSnapshot } from "./grokBrokerTurnMeter.js"; +import { GrokWorkerAttestationFailure } from "./grokWorkerAttestation.js"; + +export type GrokEngineBrokerTurnResult = Readonly<{ text: string; workerPid: number; workerUid: number; workerStartTime: string }> & EngineBrokerTurnAccounting; +export class EngineBrokerTurnFailure extends Error { + constructor(readonly code: Exclude, readonly diagnostic?: NativeBrokerDiagnostic, readonly accounting?: EngineBrokerTurnAccounting) { super("engine broker turn failed"); } +} + +/** Everything one broker turn touches, injected so the accounting and limit paths run under test without a native launcher. */ +export type GrokEngineBrokerTurnDependencies = Readonly<{ + turns: EngineBrokerTurnRegistry; + proxy: Readonly<{ capabilities: Readonly<{ issue(agentId: string, turnId: string): string; revoke(turnId: string): void }>; registerIsolationGuard(turnId: string, guard: () => Promise): void; revokeIsolationGuard(turnId: string): void; registerTurn(turnId: string, turn: GrokBrokerProxyTurn): void; revokeTurn(turnId: string): void }>; + mcp: Readonly<{ register(agentId: string, turnId: string, endpoint: string): string; revoke(turnId: string): void }>; + credentialStale(): boolean; + prepareIsolation(registration: EngineBrokerServiceRegistration): Promise<() => Promise>; + runNative(input: NativeBrokerTurn, signal: AbortSignal): Promise>; +}>; + +const limitReasonFor = { tokens: "token_ceiling", requests: "request_ceiling", timeout: "wake_timeout" } as const; +const usageOf = (usage: Readonly<{ input: number; cacheRead: number; cacheWrite: number; output: number; total: number }>): EngineBrokerTurnUsage => ({ input: usage.input, cacheRead: usage.cacheRead, cacheWrite: usage.cacheWrite, output: usage.output, total: usage.total }); + +/** + * One brokered Grok turn under its declared limits. + * + * The limits are the registration's, lowered (never raised) by the wake. The + * proxy meter refuses model requests past `maxRequests`, past the elapsed + * deadline, or once the upstream-reported running total reached `maxTokens`, + * and a tripped limit aborts the worker through the same cancel/kill path a + * client cancellation uses. A wall-clock timer trips `timeout` for a worker + * that is mid-request. + * + * Every terminal path — completed, failed, limit, cancelled — is sealed and + * metered through {@link finishBrokerTurnWithUsage}; a replayed turn returns its + * sealed accounting before any of this runs and never meters again. + */ +export async function runGrokEngineBrokerTurn(deps: GrokEngineBrokerTurnDependencies, registration: EngineBrokerServiceRegistration, wakeId: string, prompt: string, mcpEndpoint: string, signal?: AbortSignal, overrides?: EngineBrokerTurnLimitOverrides): Promise { + const { agentId } = registration, declared = registration.model.model; + let limits; + try { limits = lowerEngineBrokerTurnLimits(registration.limits, overrides); } catch { throw new EngineBrokerTurnFailure("invalid_request", undefined, { outcome: "failed", usage: null, model: declared, requests: 0, limitReason: "none" }); } + const turnId = createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"); + const request = { version: ENGINE_BROKER_VERSION, kind: "start_turn", requestId: randomUUID(), turnId, agentId, wakeId, prompt, mcpEndpoint, ...(overrides === undefined ? {} : { limits: overrides }) } as const; + const begun = await deps.turns.begin(request, declared); + if (begun !== "start") return replay(begun.replay); + const controller = new AbortController(); + const meter = new GrokBrokerTurnMeter(limits, () => controller.abort()); + const onAbort = () => controller.abort(); signal?.addEventListener("abort", onAbort, { once: true }); if (signal?.aborted) controller.abort(); + const timer = setTimeout(() => meter.trip("timeout"), limits.timeoutMs); timer.unref?.(); + const metering: BrokerTurnMetering = { usageLedgerPath: registration.usageLedgerPath, requestLedgerPath: engineBrokerRequestLedgerPathFor(registration.usageLedgerPath), agentId, wakeId }; + let nativeDiagnostic: NativeBrokerDiagnostic | undefined, attested = false, output: string | undefined, rejected = false; + try { + const isolationGuard = await deps.prepareIsolation(registration); + deps.proxy.registerIsolationGuard(turnId, isolationGuard); + deps.proxy.registerTurn(turnId, { policy: registration.model, meter }); + const providerCapability = deps.proxy.capabilities.issue(agentId, turnId), mcpCapability = deps.mcp.register(agentId, turnId, mcpEndpoint); + const result = await deps.runNative({ slot: registration.slot, requestId: request.requestId, turnId, agentId, wakeId, prompt, providerCapability, mcpCapability }, controller.signal); + nativeDiagnostic = result.diagnostic; output = result.text; + if (result.workerUid !== registration.workerUid) throw new Error("engine broker worker identity mismatch"); + await isolationGuard(); attested = true; + rejected = true; + const decoded = decodeGrokHeadlessTurn(result.text), stream = decodeGrokStreamUsage(result.text); + if (stream.reportedModels.some((reported) => mapGrokReportedModel(reported, declared) === undefined)) throw new Error("engine broker reported an undeclared model"); + rejected = false; + const snapshot = meter.snapshot(); + if (snapshot.limitReason !== "none") throw new Error("engine broker turn limit reached"); + const usage = decoded.usage === undefined ? streamOrMeterUsage(stream, snapshot) : usageOf(decoded.usage); + const accounting = { outcome: "completed", usage, model: declared, requests: requestCount(stream, snapshot), limitReason: "none" } as const; + const completed = { version: request.version, kind: "completed", requestId: request.requestId, turnId, text: decoded.text, workerPid: result.workerPid, workerUid: result.workerUid, workerStartTime: result.startTicks.toString(), ...accounting } as const; + await finishBrokerTurnWithUsage(deps.turns, request, completed, metering, { notionalUsd: decoded.usage?.notionalUsd ?? 0, complete: decoded.usage?.complete ?? false, requests: requestRows(stream, snapshot), ...(stream.sessionId === undefined ? {} : { session: stream.sessionId }) }); + return { text: completed.text, workerPid: completed.workerPid, workerUid: completed.workerUid, workerStartTime: completed.workerStartTime, ...accounting }; + } catch (error) { + const snapshot = meter.snapshot(); + const code: EngineBrokerTurnFailure["code"] = snapshot.limitReason !== "none" ? "limit_exceeded" : deps.credentialStale() ? "auth_stale" : controller.signal.aborted ? "cancelled" : "engine_failed"; + const diagnostic = error instanceof NativeBrokerTurnFailure ? error.diagnostic : nativeDiagnostic && !attested ? { ...nativeDiagnostic, status: "worker_failed" as const, stage: "attestation" as const, failureClass: error instanceof GrokWorkerAttestationFailure ? error.failureClass : "profile_invalid" as const, profileApplied: false } : undefined; + const stream = output === undefined ? undefined : decodeGrokStreamUsage(output); + const accounting = { outcome: "failed", usage: streamOrMeterUsage(stream, snapshot), model: declared, requests: requestCount(stream, snapshot), limitReason: snapshot.limitReason } as const; + const failed: EngineBrokerTerminalResponse = { version: request.version, kind: "failed", requestId: request.requestId, turnId, code, ...(diagnostic ? { diagnostic } : {}), ...accounting }; + const reason = snapshot.limitReason !== "none" ? limitReasonFor[snapshot.limitReason] : rejected ? "turn_rejected" : "unknown"; + await finishBrokerTurnWithUsage(deps.turns, request, failed, metering, { notionalUsd: 0, complete: false, reason, requests: requestRows(stream, snapshot), ...(stream?.sessionId === undefined ? {} : { session: stream.sessionId }) }); + throw new EngineBrokerTurnFailure(code, diagnostic, accounting); + } finally { + clearTimeout(timer); signal?.removeEventListener("abort", onAbort); + deps.proxy.revokeTurn(turnId); deps.proxy.revokeIsolationGuard(turnId); deps.proxy.capabilities.revoke(turnId); deps.mcp.revoke(turnId); + } +} + +function replay(response: EngineBrokerTerminalResponse): GrokEngineBrokerTurnResult { + const accounting = { outcome: response.outcome, usage: response.usage, model: response.model, requests: response.requests, limitReason: response.limitReason }; + if (response.kind === "completed") return { text: response.text, workerPid: response.workerPid, workerUid: response.workerUid, workerStartTime: response.workerStartTime, ...accounting, outcome: "completed" }; + const code = response.code === "turn_conflict" || response.code === "unavailable" ? "engine_failed" : response.code; + throw new EngineBrokerTurnFailure(code, response.diagnostic as NativeBrokerDiagnostic | undefined, accounting); +} + +/** The proxy saw every forwarded request; the stream is the fallback when no request crossed this proxy. */ +const requestCount = (stream: GrokStreamUsage | undefined, snapshot: GrokBrokerTurnMeterSnapshot): number => snapshot.requests > 0 ? snapshot.requests : stream?.requests.length ?? 0; + +/** Best partial usage: the worker's own per-request frames when any arrived, else what upstream reported to the proxy. */ +function streamOrMeterUsage(stream: GrokStreamUsage | undefined, snapshot: GrokBrokerTurnMeterSnapshot): EngineBrokerTurnUsage | null { + if (stream !== undefined && stream.requests.length > 0) { + const sum = (pick: (value: GrokStreamUsage["requests"][number]) => number) => stream.requests.reduce((total, value) => total + pick(value), 0); + return { input: sum((value) => value.input), cacheRead: sum((value) => value.cacheRead), cacheWrite: sum((value) => value.cacheWrite), output: sum((value) => value.output), total: sum((value) => value.total) }; + } + return snapshot.usage; +} + +/** Per-request rows: stream usage with proxy timing when both describe the same requests, else the proxy's own measured requests. */ +function requestRows(stream: GrokStreamUsage | undefined, snapshot: GrokBrokerTurnMeterSnapshot): BrokerTurnMeteringDetail["requests"] { + if (stream !== undefined && stream.requests.length > 0) { + const timed = snapshot.timings.length === stream.requests.length; + return stream.requests.map((value, index) => ({ ...value, ...(timed ? clock(snapshot.timings[index]!) : {}) })); + } + return snapshot.timings.flatMap((timing, index) => timing.usage === undefined ? [] : [{ index, ...usageOf(timing.usage), ...clock(timing) }]); +} +const clock = (timing: GrokBrokerTurnMeterSnapshot["timings"][number]) => ({ startedAt: timing.startedAt, ...(timing.endedAt === undefined ? {} : { endedAt: timing.endedAt }) }); diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index 7be6215..f66631d 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -1,130 +1,169 @@ import assert from "node:assert/strict"; -import { createHash, randomUUID } from "node:crypto"; +import { createHash } from "node:crypto"; import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { request as httpRequest } from "node:http"; import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import test from "node:test"; +import type { NativeBrokerTurn, NativeBrokerTurnResult } from "./engineBrokerNativeClient.js"; +import type { EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; import { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; -import { finishBrokerTurnWithUsage } from "./grokEngineBroker.js"; +import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; +import { EngineBrokerTurnFailure, runGrokEngineBrokerTurn, type GrokEngineBrokerTurnDependencies } from "./grokEngineBrokerTurn.js"; +import { TURN_REQUEST_LEDGER_VERSION } from "./turnRequestLedger.js"; import { TURN_USAGE_LEDGER_VERSION } from "./turnUsageLedger.js"; -const usage = { input: 8_746, output: 29, cacheRead: 5_760, cacheWrite: 12, total: 14_547, calls: 1, notionalUsd: 0.0035, complete: true }; - -const startRequest = (agentId: string, wakeId: string) => ({ - version: "noopolis.daimon.engine-broker.v1", kind: "start_turn", requestId: randomUUID(), - turnId: createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"), - agentId, wakeId, prompt: "prompt", mcpEndpoint: "http://127.0.0.1:43124/mcp" -} as const); +const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); +const leanBody = JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: lean }); +const upstreamUsage = { prompt_tokens: 2_696, completion_tokens: 79, total_tokens: 2_775, prompt_tokens_details: { cached_tokens: 128 } }; +const turnIdFor = (agentId: string, wakeId: string): string => createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"); + +const assistant = (id: string, usage: Record, content: unknown[], stop: string) => ({ type: "assistant", message: { id, type: "message", role: "assistant", model: "daimon-broker-grok", content, stop_reason: stop, usage }, parent_tool_use_id: null, session_id: "01a0ad21-a90f-7f71-8054-93fdb4334d6a" }); +const first = { input_tokens: 2_568, output_tokens: 79, cache_read_input_tokens: 128, cache_creation_input_tokens: 0 }; +const second = { input_tokens: 109, output_tokens: 13, cache_read_input_tokens: 2_688, cache_creation_input_tokens: 0 }; +const stream = (modelKey = "grok-4.6-build"): string => [ + { type: "system", subtype: "init", session_id: "01a0ad21-a90f-7f71-8054-93fdb4334d6a" }, + assistant("msg_0", first, [{ type: "tool_use", id: "call-0", name: "use_tool", input: {} }], "tool_use"), + assistant("msg_1", second, [{ type: "text", text: "TANGERINE-7" }], "end_turn"), + { type: "result", subtype: "success", is_error: false, num_turns: 2, result: "TANGERINE-7", stop_reason: "end_turn", total_cost_usd: 0.00248676, usage: { input_tokens: 2_677, output_tokens: 92, cache_read_input_tokens: 2_816, cache_creation_input_tokens: 0 }, modelUsage: { [modelKey]: {} }, session_id: "01a0ad21-a90f-7f71-8054-93fdb4334d6a" } +].map((frame) => JSON.stringify(frame)).join("\n"); + +function post(port: number, token: string): Promise { + return new Promise((resolve, reject) => { + const req = httpRequest({ host: "127.0.0.1", port, path: "/v1/chat/completions", method: "POST", agent: false, headers: { authorization: `Bearer ${token}`, "x-grok-client-version": "1.0.34", "content-type": "application/json" } }, (response) => { response.resume(); response.on("end", () => resolve(response.statusCode ?? 0)); }); + req.on("error", reject); req.end(leanBody); + }); +} -const completedFor = (request: ReturnType) => ({ - version: request.version, kind: "completed", requestId: request.requestId, turnId: request.turnId, - text: "ACK", workerPid: 4_242, workerUid: 2_200, workerStartTime: "99" -} as const); +type Worker = (post: () => Promise, signal: AbortSignal) => Promise; +const nativeResult = (text: string): NativeBrokerTurnResult => ({ text, workerPid: 4_242, workerUid: 2_200, startTicks: 99n, diagnostic: { status: "ok", stage: "output", failureClass: "none", profileApplied: false, exitCode: 0, termSignal: 0, workerPid: 4_242, workerUid: 2_200, startTicks: "99" } }); +const untilAborted = (signal: AbortSignal): Promise => new Promise((_resolve, reject) => { const fail = () => reject(new Error("engine broker turn failed")); if (signal.aborted) fail(); else signal.addEventListener("abort", fail, { once: true }); }); /** - * Reproduces the broker's turn control flow around the registry: replayed turns - * return before any work, and a fresh turn seals the record and then meters it. - * Everything but the engine call itself is the real production code. + * The real turn registry, proxy, meter and ledgers around a scripted worker that + * talks to the proxy exactly as the native worker does (capability bearer, + * pinned client version, lean body). Only the launcher and attestation are fakes. */ -const runTurn = async (turns: EngineBrokerTurnRegistry, ledger: string, agentId: string, wakeId: string): Promise<"start" | "replay"> => { - const request = startRequest(agentId, wakeId); - const begun = await turns.begin(request); - if (begun !== "start") return "replay"; - await finishBrokerTurnWithUsage(turns, request, completedFor(request), ledger, usage, agentId, wakeId); - return "start"; -}; - -const withStore = async (body: (turnStore: string, ledger: string, root: string) => Promise): Promise => { +const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId: string, worker: Worker, overrides?: Parameters[6], limits?: EngineBrokerServiceRegistration["limits"], turnStore?: string) => ReturnType; usageRows: () => Promise[]>; requestRows: () => Promise[]>; upstreamCalls: () => number }>) => Promise, usageLedgerPath?: string): Promise => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-usage-")); - try { await body(path.join(root, "turns"), path.join(root, "usage.jsonl"), root); } finally { await rm(root, { recursive: true, force: true }); } -}; - -const ledgerLines = async (file: string): Promise[]> => { - const text = await readFile(file, "utf8").catch(() => ""); - return text.split("\n").filter((line) => line.length > 0).map((line) => JSON.parse(line) as Record); + let calls = 0; + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(`data: ${JSON.stringify({ choices: [], usage: upstreamUsage })}\n\ndata: [DONE]\n\n`) }; }, undefined, 0); + const ledger = usageLedgerPath ?? path.join(root, "usage.jsonl"); + const rows = async (file: string) => (await readFile(file, "utf8").catch(() => "")).split("\n").filter((line) => line.length > 0).map((line) => JSON.parse(line) as Record); + try { + await body({ + root, + turn: (wakeId, worker, overrides, limits = { maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }, turnStore = path.join(root, "turns")) => { + const registration: EngineBrokerServiceRegistration = { agentId: "foreman", slot: 0, workerUid: 2_200, workspace: "/workspace", profilePath: "/workers/0/.grok/sandbox.toml", eventsPath: "/workers/0/.grok/sessions/sandbox-events.jsonl", profileSha256: "a".repeat(64), usageLedgerPath: ledger, limits, model: { model: "grok-4.6", reasoningEffort: "low" } }; + const deps: GrokEngineBrokerTurnDependencies = { + turns: new EngineBrokerTurnRegistry(turnStore), proxy, credentialStale: () => false, + mcp: { register: () => "mcp-capability-0123456789abcdef", revoke: () => undefined }, + prepareIsolation: async () => async () => undefined, + runNative: async (input: NativeBrokerTurn, signal: AbortSignal) => nativeResult(await worker(() => post(proxy.port, input.providerCapability), signal)) + }; + return runGrokEngineBrokerTurn(deps, registration, wakeId, "prompt", "http://127.0.0.1:43124/mcp", undefined, overrides); + }, + usageRows: () => rows(ledger), + requestRows: () => rows(path.join(path.dirname(ledger), "requests.jsonl")), + upstreamCalls: () => calls + }); + } finally { await proxy.close(); await rm(root, { recursive: true, force: true }); } }; -test("a completed broker turn writes exactly one metered line, and a replayed turn writes no second one", async () => { - await withStore(async (turnStore, ledger) => { - const turns = new EngineBrokerTurnRegistry(turnStore); - assert.equal(await runTurn(turns, ledger, "cogsworth", "wake-1"), "start"); - assert.deepEqual(await runTurn(turns, ledger, "cogsworth", "wake-1"), "replay"); +const twoRequests: Worker = async (send) => { assert.equal(await send(), 200); assert.equal(await send(), 200); return stream(); }; + +test("a completed turn seals its accounting, writes one usage row and per-request rows, and a replay never re-meters", async () => { + await withBroker(async ({ root, turn, usageRows, requestRows }) => { + const result = await turn("wake-1", twoRequests); + assert.deepEqual({ ...result, text: undefined }, { text: undefined, workerPid: 4_242, workerUid: 2_200, workerStartTime: "99", outcome: "completed", usage: { input: 2_677, cacheRead: 2_816, cacheWrite: 0, output: 92, total: 5_585 }, model: "grok-4.6", requests: 2, limitReason: "none" }); + // Mutation guard: metering on the replay path appends a second row here. + let replayedWorker = false; + assert.deepEqual(await turn("wake-1", async () => { replayedWorker = true; return stream(); }), result); + assert.deepEqual(await turn("wake-1", twoRequests, undefined, undefined, path.join(root, "turns")), result, "a fresh registry boot replays the sealed accounting"); + assert.equal(replayedWorker, false); + const usage = await usageRows(); + assert.equal(usage.length, 1); + assert.deepEqual({ ...usage[0], at: undefined }, { v: TURN_USAGE_LEDGER_VERSION, agent: "foreman", wake: "wake-1", engine: "grok", at: undefined, input: 2_677, output: 92, cache_read: 2_816, cache_write: 0, total: 5_585, calls: 2, notional_usd: 0.00248676, complete: true, outcome: "completed", turn: turnIdFor("foreman", "wake-1"), limit_reason: "none", model: "grok-4.6" }); + const requests = await requestRows(); + assert.deepEqual(requests.map((row) => [row.v, row.engine, row.request, row.requests, row.input, row.fresh_input, row.cached_input, row.total, row.turn]), [ + [TURN_REQUEST_LEDGER_VERSION, "grok", 0, 2, 2_696, 2_568, 128, 2_775, turnIdFor("foreman", "wake-1")], + [TURN_REQUEST_LEDGER_VERSION, "grok", 1, 2, 2_797, 109, 2_688, 2_810, turnIdFor("foreman", "wake-1")] + ]); + // Mutation guard: stamping every request with the wake end collapses these. + for (const row of requests) assert.match(String(row.started_at), /^\d{4}-\d{2}-\d{2}T/u); + assert.ok(String(requests[0]!.ended_at) <= String(requests[1]!.started_at), "request 1 ends before request 2 starts"); + }); +}); - // Mutation guard: removing the replay suppression makes the same wake - // append a second line and double-count the subscription. - const written = await ledgerLines(ledger); - assert.equal(written.length, 1); - assert.deepEqual(written[0], { - v: TURN_USAGE_LEDGER_VERSION, agent: "cogsworth", wake: "wake-1", engine: "grok", - at: written[0]?.at, input: 8_746, output: 29, cache_read: 5_760, cache_write: 12, - total: 14_547, calls: 1, notional_usd: 0.0035, complete: true, - // The broker only ever appends for a turn it finished, so its rows are - // completed by construction; the field still states it explicitly. - outcome: "completed" +test("a turn past maxRequests is refused before upstream, killed, sealed as limit_exceeded, and its partial usage is metered", async () => { + await withBroker(async ({ turn, usageRows, upstreamCalls }) => { + const worker: Worker = async (send, signal) => { for (;;) { if (await send() === 429) return untilAborted(signal); } }; + await assert.rejects(turn("wake-2", worker, undefined, { maxRequests: 3, maxTokens: 300_000, timeoutMs: 240_000 }), (error: unknown) => { + assert.ok(error instanceof EngineBrokerTurnFailure); + assert.equal(error.code, "limit_exceeded"); + assert.deepEqual(error.accounting, { outcome: "failed", usage: { input: 7_704, cacheRead: 384, cacheWrite: 0, output: 237, total: 8_325 }, model: "grok-4.6", requests: 3, limitReason: "requests" }); + return true; }); - - assert.equal(await runTurn(turns, ledger, "cogsworth", "wake-2"), "start"); - assert.equal((await ledgerLines(ledger)).length, 2); + assert.equal(upstreamCalls(), 3); + // Mutation guard: metering only completed turns leaves this ledger empty. + const [row, extra] = await usageRows(); + assert.equal(extra, undefined); + assert.deepEqual([row?.outcome, row?.reason, row?.limit_reason, row?.total, row?.calls, row?.complete], ["failed", "request_ceiling", "requests", 8_325, 3, false]); + await assert.rejects(turn("wake-2", twoRequests), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.accounting?.limitReason === "requests"); + assert.equal((await usageRows()).length, 1, "the replayed failure is not metered again"); }); }); -test("crash recovery replays the same completed turn without metering it again", async () => { - await withStore(async (turnStore, ledger) => { - assert.equal(await runTurn(new EngineBrokerTurnRegistry(turnStore), ledger, "foreman", "wake-9"), "start"); - // A fresh boot id is what the broker gets after a crash. - assert.equal(await runTurn(new EngineBrokerTurnRegistry(turnStore), ledger, "foreman", "wake-9"), "replay"); - assert.equal((await ledgerLines(ledger)).length, 1); +test("the token ceiling stops a turn one request past the ceiling at most", async () => { + await withBroker(async ({ turn, upstreamCalls }) => { + const worker: Worker = async (send, signal) => { for (;;) { if (await send() === 429) return untilAborted(signal); } }; + // 2,775 tokens per request against 5,000: requests 1 and 2 are admitted, 3 is refused. + await assert.rejects(turn("wake-3", worker, { maxTokens: 5_000 }), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.accounting?.limitReason === "tokens" && error.accounting.usage?.total === 5_550); + assert.equal(upstreamCalls(), 2); }); }); -test("an unwritable ledger leaves the turn recorded as completed, not failed", async () => { - // Mutation guard: deleting the advisory try/catch in recordTurnUsage makes - // finishBrokerTurnWithUsage reject. In the broker that rejection lands in the - // catch that calls finish(..., failed), which renames over this already - // completed record — turning a published turn into a failed one. - await withStore(async (turnStore, _ledger, root) => { - const turns = new EngineBrokerTurnRegistry(turnStore); - const unwritable = path.join(root, "not-provisioned", "usage.jsonl"); - const request = startRequest("brass", "wake-3"); - assert.equal(await turns.begin(request), "start"); - await assert.doesNotReject(finishBrokerTurnWithUsage(turns, request, completedFor(request), unwritable, usage, "brass", "wake-3")); - - const replayed = await new EngineBrokerTurnRegistry(turnStore).begin(request); - assert.notEqual(replayed, "start"); - assert.equal((replayed as { replay: { kind: string } }).replay.kind, "completed"); +test("the wall-clock limit aborts a worker that is mid-request", async () => { + await withBroker(async ({ turn, usageRows }) => { + const started = Date.now(); + const worker: Worker = async (send, signal) => { assert.equal(await send(), 200); return untilAborted(signal); }; + await assert.rejects(turn("wake-4", worker, { timeoutMs: 1_000 }), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.code === "limit_exceeded" && error.accounting?.limitReason === "timeout"); + assert.ok(Date.now() - started < 5_000); + assert.deepEqual((await usageRows()).map((row) => [row.reason, row.limit_reason, row.total]), [["wake_timeout", "timeout", 2_775]]); }); }); -test("a turn whose usage could not be decoded is sealed but writes no line", async () => { - await withStore(async (turnStore, ledger) => { - const turns = new EngineBrokerTurnRegistry(turnStore); - const request = startRequest("brass", "wake-4"); - assert.equal(await turns.begin(request), "start"); - await finishBrokerTurnWithUsage(turns, request, completedFor(request), ledger, undefined, "brass", "wake-4"); - assert.deepEqual(await ledgerLines(ledger), []); - assert.notEqual(await new EngineBrokerTurnRegistry(turnStore).begin(request), "start"); +test("a wake may only lower a declared limit: raising one is refused before any turn record or worker", async () => { + await withBroker(async ({ turn, usageRows, upstreamCalls }) => { + let ran = false; + // Mutation guard: clamping or accepting the raise runs the worker. + await assert.rejects(turn("wake-5", async () => { ran = true; return stream(); }, { maxTokens: 300_001 }), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.code === "invalid_request"); + assert.equal(ran, false); assert.equal(upstreamCalls(), 0); assert.deepEqual(await usageRows(), []); + assert.equal((await turn("wake-5", twoRequests, { maxTokens: 299_999, timeoutMs: 1_000 })).outcome, "completed", "a lowered limit is accepted and the turn was never recorded"); }); }); -test("usage is never written into the completed frame the strict wire parser re-validates", async () => { - await withStore(async (turnStore, ledger) => { - const turns = new EngineBrokerTurnRegistry(turnStore); - assert.equal(await runTurn(turns, ledger, "cogsworth", "wake-5"), "start"); - // The durable record is re-parsed on the next begin(); an extra field there - // makes it throw permanently and breaks crash-recovery replay for good. - const replayed = await new EngineBrokerTurnRegistry(turnStore).begin(startRequest("cogsworth", "wake-5")); - const response = (replayed as { replay: Record }).replay; - assert.deepEqual(Object.keys(response).sort(), ["kind", "requestId", "text", "turnId", "version", "workerPid", "workerStartTime", "workerUid"]); +test("a turn whose stream reports an undeclared model fails as rejected but is still metered", async () => { + await withBroker(async ({ turn, usageRows }) => { + await assert.rejects(turn("wake-6", async (send) => { await send(); await send(); return stream("grok-4.5-build"); }), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.code === "engine_failed"); + assert.deepEqual((await usageRows()).map((row) => [row.outcome, row.reason, row.model, row.total]), [["failed", "turn_rejected", "grok-4.6", 5_585]]); }); }); -test("the broker meters only on the success path, through the single sealing helper", async () => { - const source = await readFile(path.join(path.dirname(fileURLToPath(import.meta.url)), "grokEngineBroker.ts"), "utf8"); - const body = source.slice(source.indexOf("async turn(")); - assert.equal(body.includes("recordTurnUsage("), false, "the broker must meter only through finishBrokerTurnWithUsage"); - assert.equal((body.match(/finishBrokerTurnWithUsage\(/gu) ?? []).length, 1, "exactly one metering call, in the success branch"); - assert.equal(body.includes("turns.finish(request,completed)"), false, "the success branch must seal through the metering helper"); - assert.ok(body.indexOf("finishBrokerTurnWithUsage(") < body.indexOf("catch(error)"), "metering belongs to the success branch"); +test("an unwritable ledger leaves the turn recorded as completed, not failed", async () => { + await withBroker(async ({ root, turn }) => { + assert.equal((await turn("wake-7", twoRequests)).outcome, "completed"); + assert.equal((await turn("wake-7", twoRequests, undefined, undefined, path.join(root, "turns"))).outcome, "completed"); + }, path.join(os.tmpdir(), `daimon-missing-${process.pid}`, "not-provisioned", "usage.jsonl")); +}); + +test("the broker meters only through the single sealing helper, on both terminal branches", async () => { + const source = await readFile(path.join(path.dirname(fileURLToPath(import.meta.url)), "grokEngineBrokerTurn.ts"), "utf8"); + const body = source.slice(source.indexOf("export async function runGrokEngineBrokerTurn"), source.indexOf("function replay(")); + assert.equal(body.includes("recordTurnUsage("), false); + assert.equal(body.includes("turns.finish("), false, "every terminal record is sealed through the metering helper"); + assert.equal((body.match(/finishBrokerTurnWithUsage\(/gu) ?? []).length, 2); + assert.ok(body.indexOf("return replay(") < body.indexOf("finishBrokerTurnWithUsage("), "a replay returns before any metering"); }); From 65d0144be9e1513a03b71af9191841c95117d88e Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:13:24 +0200 Subject: [PATCH 04/21] feat: pass wake limits and declared model to the Grok broker from the dispatcher --- scripts/liveGrokBrokerSession.ts | 4 ++++ src/runtime/engineDispatcher.test.ts | 9 +++++++-- src/runtime/engineDispatcher.ts | 13 +++++++++++-- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/scripts/liveGrokBrokerSession.ts b/scripts/liveGrokBrokerSession.ts index 05cf30d..83c8772 100644 --- a/scripts/liveGrokBrokerSession.ts +++ b/scripts/liveGrokBrokerSession.ts @@ -8,7 +8,9 @@ import { readChild } from "../src/pi/cliChildOutput.ts"; import { terminateChild, trackCliChild } from "../src/pi/cliProcess.ts"; import { decodeGrokHeadlessTurn } from "../src/pi/grokHeadlessResult.ts"; import { readGrokBrokerCredential } from "../src/runtime/grokBrokerCredentialReader.ts"; +import { DEFAULT_GROK_BROKER_TURN_LIMITS } from "../src/runtime/engineBrokerTurnAccounting.ts"; import { startGrokBrokerProxy } from "../src/runtime/grokBrokerProxy.ts"; +import { GrokBrokerTurnMeter } from "../src/runtime/grokBrokerTurnMeter.ts"; import { DEFAULT_GROK_BROKER_MODEL_POLICY } from "../src/runtime/grokBrokerModelPolicy.ts"; import { GROK_BROKER_PROVIDER_CAPABILITY_ENV, renderGrokBrokerWorkerArgs, renderGrokBrokerWorkerConfigWith } from "../src/runtime/grokBrokerWorkerConfig.ts"; @@ -38,6 +40,8 @@ try { const capability = proxy.capabilities.issue("local-auth-probe", turnId); // This local transport probe deliberately does not attest a native worker. proxy.registerIsolationGuard(turnId, async () => undefined); + // The proxy forwards nothing unmetered; the probe runs under the default v1 limits. + proxy.registerTurn(turnId, { policy: DEFAULT_GROK_BROKER_MODEL_POLICY, meter: new GrokBrokerTurnMeter(DEFAULT_GROK_BROKER_TURN_LIMITS) }); // No MCP tools are needed for this exact-reply authentication probe. await writeFile(path.join(home, "config.toml"), renderGrokBrokerWorkerConfigWith(DEFAULT_GROK_BROKER_MODEL_POLICY, { proxyPort: proxy.port, mcpUrl: "http://127.0.0.1:43124/mcp" }).split("[mcp_servers.daimon]")[0]); const prompt = path.join(home, "prompt.txt"); diff --git a/src/runtime/engineDispatcher.test.ts b/src/runtime/engineDispatcher.test.ts index ef80d32..a5649b5 100644 --- a/src/runtime/engineDispatcher.test.ts +++ b/src/runtime/engineDispatcher.test.ts @@ -243,9 +243,14 @@ test("production Grok dispatcher routes every wake through the broker without ag process.env.PATH = `${root}${path.delimiter}${priorPath ?? ""}`; process.env.NOOPOLIS_RUN_ID = "dispatcher-grok-realm-test"; const broker: EngineBrokerTurnClient = { - async turn(agentId,wakeId,prompt,endpoint,signal) { turns += 1;assert.equal(agentId,config.id);assert.match(wakeId,/^(first|second)$/u);assert.match(prompt,/work/u);assert.match(endpoint,/^http:\/\/127\.0\.0\.1:\d+\/mcp$/u);assert.equal(signal?.aborted,false);return "brokered"; } + async turn(agentId,wakeId,prompt,endpoint,signal,options) { turns += 1;assert.equal(agentId,config.id);assert.match(wakeId,/^(first|second)$/u);assert.match(prompt,/work/u);assert.match(endpoint,/^http:\/\/127\.0\.0\.1:\d+\/mcp$/u);assert.equal(signal?.aborted,false); + // The engine-neutral wake bound reaches the broker as a lowering limit. + assert.deepEqual(options,{limits:{maxTokens:123_456}});return "brokered"; } }; - const handle = await startOrganizationRuntimeEngine(config, "DAIMON_UNUSED_CONTROL", undefined, undefined, broker); + const priorCeiling = process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING; process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING = "123456"; + let handle: Awaited>; + try { handle = await startOrganizationRuntimeEngine(config, "DAIMON_UNUSED_CONTROL", undefined, undefined, broker); } + finally { if (priorCeiling === undefined) delete process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING; else process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING = priorCeiling; } assert.equal((await handle.wake({ id: "first", kind: "manual", text: "work" })).text, "brokered"); assert.equal((await handle.wake({ id: "second", kind: "manual", text: "work" })).text, "brokered"); assert.equal(turns, 2); diff --git a/src/runtime/engineDispatcher.ts b/src/runtime/engineDispatcher.ts index 9414752..cbb5ad2 100644 --- a/src/runtime/engineDispatcher.ts +++ b/src/runtime/engineDispatcher.ts @@ -2,7 +2,7 @@ import { attentionTools, type AttentionRegistry } from "./attention.js"; import path from "node:path"; import type { AgentHandle } from "../core/types.js"; -import { AGY_MAX_TOOL_TURNS, createCliSessionFactory, resolveCodexWakeTimeoutMs, resolveCodexWakeTokenCeiling } from "../pi/cliSession.js"; +import { AGY_MAX_TOOL_TURNS, createCliSessionFactory, resolveCodexWakeTimeoutMs, resolveCodexWakeTokenCeiling, resolveEngineWakeLimitOverrides } from "../pi/cliSession.js"; import { GROK_DAIMON_SANDBOX_PROFILE, prepareAndVerifyGrokSandbox @@ -151,7 +151,10 @@ function adapterFor(agent: OrganizationRuntimeAgentConfig, controlTokenEnv: stri }) } : {}), ...(engine==="grok"&&grokBroker!==undefined?{}:{credentialSecretValues: () => readPortableEngineCredentialSecrets(agent.id, engine, engineHomePath)}), - ...(engine==="grok"&&grokBroker!==undefined?{grokBrokerTurn:(prompt:string,endpoint:string,signal:AbortSignal)=>grokBroker.turn(agent.id,wakeEnvironmentContext.current??"wake",prompt,endpoint,signal)}:{}), + // The broker seals usage and enforces its registration's limits; the + // wake may only lower them (DAIMON_ENGINE_WAKE_*), and a declared model + // must be the one the broker reports it ran. + ...(engine==="grok"&&grokBroker!==undefined?{grokBrokerTurn:grokBrokerTurnFor(agent,grokBroker,wakeEnvironmentContext)}:{}), ...(engine === "grok" && verifyGrokSandbox ? { grokSandboxProfile: GROK_DAIMON_SANDBOX_PROFILE, verifyGrokSandbox @@ -160,6 +163,12 @@ function adapterFor(agent: OrganizationRuntimeAgentConfig, controlTokenEnv: stri return cliHarness(agent, sessionFactory, [controlTokenEnv], productionTools, wakeEnvironmentContext); } +function grokBrokerTurnFor(agent: OrganizationRuntimeAgentConfig, grokBroker: EngineBrokerTurnClient, wakeEnvironmentContext: import("../pi/piAgentWakeSupport.js").PiWakeEnvironmentContextRef) { + const limits = resolveEngineWakeLimitOverrides(); + const options = { ...(limits === undefined ? {} : { limits }), ...(agent.engine.model === undefined ? {} : { model: agent.engine.model }) }; + return (prompt: string, endpoint: string, signal: AbortSignal) => grokBroker.turn(agent.id, wakeEnvironmentContext.current ?? "wake", prompt, endpoint, signal, options); +} + /** * CLI engines do not consume Pi's resource loader. Frame the same immutable * identity in JSON so arbitrary names/instructions cannot change its shape. From ae8dd17276bd70453beda90e87b09edf788633d7 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:15:16 +0200 Subject: [PATCH 05/21] feat: accept closed-list Grok model and reasoning effort and pin the broker accounting contract in the manifest --- src/contracts/organizationRuntimeContract.ts | 8 ++- src/contracts/runtimeContractManifest.ts | 15 ++++++ src/runtime/engineBrokerContract.test.ts | 17 +++++++ src/runtime/engineBrokerTurnAccounting.ts | 20 ++++---- src/runtime/engineDispatcher.test.ts | 5 ++ src/runtime/engineDispatcher.ts | 3 ++ src/runtime/organizationRuntime.test.ts | 51 ++++++++++++++++---- src/runtime/organizationRuntime.ts | 8 +-- src/runtime/organizationRuntimeParsing.ts | 32 +++++++++--- 9 files changed, 127 insertions(+), 32 deletions(-) create mode 100644 src/runtime/engineBrokerContract.test.ts diff --git a/src/contracts/organizationRuntimeContract.ts b/src/contracts/organizationRuntimeContract.ts index 8f2b9ca..ce87b07 100644 --- a/src/contracts/organizationRuntimeContract.ts +++ b/src/contracts/organizationRuntimeContract.ts @@ -1,3 +1,5 @@ +import { GROK_BROKER_MODELS, GROK_BROKER_REASONING_EFFORTS } from "./grokWorkerContract.js"; + /** The data-only organization-runtime constants shared by product code and artifacts. */ export const ORGANIZATION_RUNTIME_VERSION = "noopolis.daimon.organization-runtime.v1" as const; export const ORGANIZATION_RUNTIME_V2_VERSION = "noopolis.daimon.organization-runtime.v2" as const; @@ -74,7 +76,11 @@ export const ORGANIZATION_RUNTIME_CONFIG_SCHEMA = { codexSandbox: { type: "object", additionalProperties: false, required: ["mode", "networkAccess", "webSearch"], properties: { mode: { const: "workspace-write" }, networkAccess: { const: false }, webSearch: { const: "disabled" } } } - } }, + }, allOf: [ + // grok: a declared model is the closed broker pair, both or neither; never a Codex sandbox. + { if: { properties: { kind: { const: "grok" } } }, then: { properties: { model: { enum: GROK_BROKER_MODELS }, reasoningEffort: { enum: GROK_BROKER_REASONING_EFFORTS }, codexSandbox: false }, dependentRequired: { model: ["reasoningEffort"], reasoningEffort: ["model"] } } }, + { if: { properties: { kind: { const: "agy" } } }, then: { properties: { model: false, reasoningEffort: false, codexSandbox: false } } } + ] }, ...PRODUCTION_TOOL_PROPERTIES } } } diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 20ae6be..ceef199 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -68,6 +68,21 @@ export const GROK_ENGINE_BROKER = { } }, bounds: { promptBytes: 65_536, capabilityBytes: 4_096, capabilityBundleBytes: 8_196, outputBytes: 65_536 }, + // Accounting and limits (P2). The broker is the single sealed usage writer. + controlProtocolVersion: "noopolis.daimon.engine-broker.v2", + turnRecordVersions: ["noopolis.daimon.engine-broker-turn.v1", "noopolis.daimon.engine-broker-turn.v2"], + serviceConfigVersions: ["noopolis.daimon.engine-broker-service.v1", "noopolis.daimon.engine-broker-service.v2"], + turnLimits: { + keys: ["maxRequests", "maxTokens", "timeoutMs"], + v1Defaults: { maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }, + bounds: { maxRequests: [1, GROK_WORKER_MAX_TURNS], maxTokens: [1, 10_000_000], timeoutMs: [1_000, 3_600_000] }, + limitReasons: ["tokens", "requests", "timeout", "none"], + wakeMayOnlyLower: true, + tokenCeilingOvershoot: "at-most-one-request" + }, + wakeLimitEnvironment: { timeoutMs: "DAIMON_ENGINE_WAKE_TIMEOUT_MS", maxTokens: "DAIMON_ENGINE_WAKE_TOKEN_CEILING" }, + projectionVersion: "noopolis.daimon.grok-broker-projection.v1", + slotPreflightVersion: "noopolis.daimon.grok-slot-preflight.v1", artifacts: { sourceSha256: "36f60689f0a8af0e3108f5f53d78ed52b7d4b6f934c75b6184606dfa82bc741e", x64Sha256: "36dc76b134eb59cf5a6720b6f94228eb279108e20ea3343fa6efd9ffcb60a4d3", diff --git a/src/runtime/engineBrokerContract.test.ts b/src/runtime/engineBrokerContract.test.ts new file mode 100644 index 0000000..17ae209 --- /dev/null +++ b/src/runtime/engineBrokerContract.test.ts @@ -0,0 +1,17 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import { ENGINE_BROKER_VERSION } from "./engineBrokerProtocol.js"; +import { ENGINE_BROKER_SERVICE_V1, ENGINE_BROKER_SERVICE_V2 } from "./engineBrokerServiceConfig.js"; +import { ENGINE_BROKER_TURN_RECORD_V1, ENGINE_BROKER_TURN_RECORD_V2 } from "./engineBrokerTurnRegistry.js"; +import { DEFAULT_CODEX_WAKE_TIMEOUT_MS, DEFAULT_CODEX_WAKE_TOKEN_CEILING, DAIMON_ENGINE_WAKE_TIMEOUT_MS_ENV, DAIMON_ENGINE_WAKE_TOKEN_CEILING_ENV } from "../pi/engineWakeLimits.js"; + +test("the manifest pins the broker accounting contract the runtime actually speaks", () => { + assert.equal(GROK_ENGINE_BROKER.controlProtocolVersion, ENGINE_BROKER_VERSION); + assert.deepEqual(GROK_ENGINE_BROKER.turnRecordVersions, [ENGINE_BROKER_TURN_RECORD_V1, ENGINE_BROKER_TURN_RECORD_V2]); + assert.deepEqual(GROK_ENGINE_BROKER.serviceConfigVersions, [ENGINE_BROKER_SERVICE_V1, ENGINE_BROKER_SERVICE_V2]); + assert.deepEqual(GROK_ENGINE_BROKER.wakeLimitEnvironment, { timeoutMs: DAIMON_ENGINE_WAKE_TIMEOUT_MS_ENV, maxTokens: DAIMON_ENGINE_WAKE_TOKEN_CEILING_ENV }); + assert.deepEqual([GROK_ENGINE_BROKER.turnLimits.v1Defaults.timeoutMs, GROK_ENGINE_BROKER.turnLimits.v1Defaults.maxTokens], [DEFAULT_CODEX_WAKE_TIMEOUT_MS, DEFAULT_CODEX_WAKE_TOKEN_CEILING]); + assert.ok(GROK_ENGINE_BROKER.turnLimits.bounds.maxRequests[1] <= GROK_ENGINE_BROKER.worker.maxTurns, "the broker request ceiling fires before the launcher --max-turns backstop"); +}); diff --git a/src/runtime/engineBrokerTurnAccounting.ts b/src/runtime/engineBrokerTurnAccounting.ts index 951c4f4..7f7f346 100644 --- a/src/runtime/engineBrokerTurnAccounting.ts +++ b/src/runtime/engineBrokerTurnAccounting.ts @@ -1,4 +1,5 @@ -import { GROK_BROKER_MODELS, GROK_WORKER_MAX_TURNS } from "../contracts/grokWorkerContract.js"; +import { GROK_BROKER_MODELS } from "../contracts/grokWorkerContract.js"; +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; import type { GrokBrokerModel } from "./grokBrokerModelPolicy.js"; /** @@ -15,7 +16,7 @@ import type { GrokBrokerModel } from "./grokBrokerModelPolicy.js"; * inside `output` and never added to `total`. */ export type EngineBrokerTurnUsage = Readonly<{ input: number; cacheRead: number; cacheWrite: number; output: number; total: number; reasoning?: number }>; -export const ENGINE_BROKER_LIMIT_REASONS = ["tokens", "requests", "timeout", "none"] as const; +export const ENGINE_BROKER_LIMIT_REASONS = GROK_ENGINE_BROKER.turnLimits.limitReasons; export type EngineBrokerLimitReason = (typeof ENGINE_BROKER_LIMIT_REASONS)[number]; export type EngineBrokerTurnLimits = Readonly<{ maxRequests: number; maxTokens: number; timeoutMs: number }>; export type EngineBrokerTurnLimitOverrides = Readonly>; @@ -28,18 +29,15 @@ export type EngineBrokerTurnAccounting = Readonly<{ }>; /** - * Bounds every declared limit must sit inside. `maxRequests` stays at or below - * the launcher's compiled `--max-turns` backstop, so the broker ceiling is the - * one that fires first. + * Bounds every declared limit must sit inside, and the v1 defaults, both from + * the runtime contract manifest. `maxRequests` stays at or below the + * launcher's compiled `--max-turns` backstop, so the broker ceiling is the one + * that fires first. */ -export const ENGINE_BROKER_LIMIT_BOUNDS = Object.freeze({ - maxRequests: [1, GROK_WORKER_MAX_TURNS], - maxTokens: [1, 10_000_000], - timeoutMs: [1_000, 3_600_000] -} as const); +export const ENGINE_BROKER_LIMIT_BOUNDS = GROK_ENGINE_BROKER.turnLimits.bounds; /** What a v1 `service.json` registration gets; equal to the Codex per-wake defaults. */ -export const DEFAULT_GROK_BROKER_TURN_LIMITS: EngineBrokerTurnLimits = Object.freeze({ maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }); +export const DEFAULT_GROK_BROKER_TURN_LIMITS: EngineBrokerTurnLimits = Object.freeze({ ...GROK_ENGINE_BROKER.turnLimits.v1Defaults }); const LIMIT_KEYS = ["maxRequests", "maxTokens", "timeoutMs"] as const; type JsonRecord = Record; diff --git a/src/runtime/engineDispatcher.test.ts b/src/runtime/engineDispatcher.test.ts index a5649b5..488a658 100644 --- a/src/runtime/engineDispatcher.test.ts +++ b/src/runtime/engineDispatcher.test.ts @@ -401,3 +401,8 @@ async function seedAuth(root: string, kind: "codex" | "grok" | "agy"): Promise { + const config = { ...rootConfig("/tmp/daimon-unused-direct-grok", "grok"), engine: { kind: "grok", model: "grok-4.6", reasoningEffort: "low" } } as OrganizationRuntimeAgentConfig; + await assert.rejects(startOrganizationRuntimeEngine(config, "DAIMON_UNUSED_CONTROL"), /requires the engine broker/u); +}); diff --git a/src/runtime/engineDispatcher.ts b/src/runtime/engineDispatcher.ts index cbb5ad2..4666aea 100644 --- a/src/runtime/engineDispatcher.ts +++ b/src/runtime/engineDispatcher.ts @@ -32,6 +32,9 @@ export async function startOrganizationRuntimeEngine( sharedProtectedPaths: readonly string[] = [], attention?: AttentionRegistry ): Promise { + // A declared Grok model is enforced by the broker proxy and worker config; + // the direct path has neither, so it refuses rather than silently ignoring it. + if (agent.engine.kind === "grok" && agent.engine.model !== undefined && grokBroker === undefined) throw new Error(`Agent ${agent.id} declares a Grok model, which requires the engine broker`); await paths?.verify(); const canonicalAgent = paths === undefined ? agent : { ...agent, workspacePath: paths.workspacePath, runtimeHomePath: paths.runtimeHomePath }; const readiness = canonicalAgent.engine.kind === "grok" && grokBroker !== undefined diff --git a/src/runtime/organizationRuntime.test.ts b/src/runtime/organizationRuntime.test.ts index f5f1c7f..3636225 100644 --- a/src/runtime/organizationRuntime.test.ts +++ b/src/runtime/organizationRuntime.test.ts @@ -271,6 +271,20 @@ test("the engine JSON Schema and the parser agree on model/reasoningEffort", () } }); +test("the engine JSON Schema and the parser agree on grok and agy model declarations", async () => { + const { Ajv2020 } = await import("ajv/dist/2020.js") as unknown as { Ajv2020: new (options: Record) => { compile(schema: unknown): (value: unknown) => boolean } }; + const validate = new Ajv2020({ strict: false }).compile(ORGANIZATION_RUNTIME_CONFIG_SCHEMA); + for (const engine of [ + { kind: "grok" }, { kind: "grok", model: "grok-4.6", reasoningEffort: "low" }, { kind: "grok", model: "grok-4.6" }, + { kind: "grok", model: "grok-4.6-build", reasoningEffort: "low" }, { kind: "grok", model: "grok-4.5", reasoningEffort: "xhigh" }, + { kind: "agy" }, { kind: "agy", model: "x" }, { kind: "codex", model: "gpt-5-codex", reasoningEffort: "xhigh" } + ]) { + const config = valid(); + config.agents[0]!.engine = engine as never; + assert.equal(validate(config), validateOrganizationRuntimeConfig(config), JSON.stringify(engine)); + } +}); + test("accepts only the narrow optional Codex workspace policy", () => { const config = valid(); config.agents[0]!.engine = { kind: "codex", codexSandbox: { mode: "workspace-write", networkAccess: false, webSearch: "disabled" } } as never; @@ -287,22 +301,41 @@ test("accepts only the narrow optional Codex workspace policy", () => { } }); -test("rejects model and reasoningEffort on a non-codex engine", () => { +test("rejects model and reasoningEffort on agy, and codexSandbox on every non-codex engine", () => { + const withModel = valid(); + withModel.agents[0]!.engine = { kind: "agy", model: "gpt-5-codex" } as never; + assert.throws(() => parseOrganizationRuntimeConfig(withModel), /codex-only/); + const withEffort = valid(); + withEffort.agents[0]!.engine = { kind: "agy", reasoningEffort: "high" } as never; + assert.throws(() => parseOrganizationRuntimeConfig(withEffort), /codex-only/); for (const kind of ["grok", "agy"] as const) { - const withModel = valid(); - withModel.agents[0]!.engine = { kind, model: "gpt-5-codex" } as never; - assert.throws(() => parseOrganizationRuntimeConfig(withModel), /codex-only/); - - const withEffort = valid(); - withEffort.agents[0]!.engine = { kind, reasoningEffort: "high" } as never; - assert.throws(() => parseOrganizationRuntimeConfig(withEffort), /codex-only/); - const withPolicy = valid(); withPolicy.agents[0]!.engine = { kind, codexSandbox: { mode: "workspace-write", networkAccess: false, webSearch: "disabled" } } as never; assert.throws(() => parseOrganizationRuntimeConfig(withPolicy), /codex-only/); } }); +test("grok declares model and reasoning effort together from the closed broker lists, never half-inherited", () => { + const declared = valid(); + declared.agents[0]!.engine = { kind: "grok", model: "grok-4.6", reasoningEffort: "low" } as never; + assert.deepEqual(parseOrganizationRuntimeConfig(declared).agents[0]!.engine, { kind: "grok", model: "grok-4.6", reasoningEffort: "low" }); + const bare = valid(); + bare.agents[0]!.engine = { kind: "grok" } as never; + assert.deepEqual(parseOrganizationRuntimeConfig(bare).agents[0]!.engine, { kind: "grok" }); + for (const engine of [ + { kind: "grok", model: "grok-4.6" }, + { kind: "grok", reasoningEffort: "low" }, + { kind: "grok", model: "grok-4.6-build", reasoningEffort: "low" }, + { kind: "grok", model: "gpt-5-codex", reasoningEffort: "low" }, + { kind: "grok", model: "grok-4.6", reasoningEffort: "xhigh" }, + { kind: "grok", model: "grok-4.6", reasoningEffort: "low", provider: "xai" } + ]) { + const invalid = valid(); + invalid.agents[0]!.engine = engine as never; + assert.throws(() => parseOrganizationRuntimeConfig(invalid), TypeError, JSON.stringify(engine)); + } +}); + const withMemory = (agent: Record, memory: unknown): Record => ({ ...agent, memory }); test("parses a declared memory bank and round-trips its fields", () => { diff --git a/src/runtime/organizationRuntime.ts b/src/runtime/organizationRuntime.ts index 76684fe..f201306 100644 --- a/src/runtime/organizationRuntime.ts +++ b/src/runtime/organizationRuntime.ts @@ -35,10 +35,10 @@ export { export type OrganizationRuntimeEngineKind = "codex" | "grok" | "agy"; /** - * `model`/`reasoningEffort` are codex-only: grok and agy own their own model - * selection (their subscription auth and model selection are Daimon-owned), - * and `organizationRuntimeParsing.ts` rejects either field on a non-codex - * engine at parse time rather than silently ignoring it. The type stays flat + * `model`/`reasoningEffort` are accepted for codex (open model name) and for + * grok (closed broker lists, declared together); agy owns its own model + * selection and `organizationRuntimeParsing.ts` rejects either field there at + * parse time rather than silently ignoring it. The type stays flat * — not a `kind`-discriminated union — because every parsed value already * satisfies the invariant; callers that need it narrow on `kind === "codex"`. */ diff --git a/src/runtime/organizationRuntimeParsing.ts b/src/runtime/organizationRuntimeParsing.ts index a6f1d39..cc65b39 100644 --- a/src/runtime/organizationRuntimeParsing.ts +++ b/src/runtime/organizationRuntimeParsing.ts @@ -1,3 +1,4 @@ +import { GROK_BROKER_MODELS, GROK_BROKER_REASONING_EFFORTS } from "../contracts/grokWorkerContract.js"; import { parseAttention } from "./attention.js"; import { ORGANIZATION_RUNTIME_CODEX_REASONING_EFFORTS, @@ -187,18 +188,19 @@ function cronValues(field: string, [minimum, maximum]: readonly [number, number] function engine(value: unknown, label: string): OrganizationRuntimeEngineIntent { const input = object(value, label); const kind = string(input.kind, `${label}.kind`); if (!ENGINE_KINDS.has(kind)) throw new TypeError(`${label}.kind is not a supported engine`); - // `model`/`reasoningEffort` are codex-only: grok and agy own their own model - // selection, so either field on a non-codex engine is rejected explicitly - // here (a clear, named error) rather than falling through to the generic - // "must contain exactly" rejection every other unexpected key gets below. - const allowed = kind === "codex" ? ["kind", "model", "reasoningEffort", "codexSandbox"] : ["kind"]; + // `codexSandbox` is codex-only; `model`/`reasoningEffort` are accepted for + // codex (open model name, Codex effort list) and for grok (closed broker + // lists, declared together or not at all). agy owns its own model selection. + const allowed = kind === "codex" ? ["kind", "model", "reasoningEffort", "codexSandbox"] : kind === "grok" ? ["kind", "model", "reasoningEffort"] : ["kind"]; const extras = Object.keys(input).filter((key) => !allowed.includes(key)); if (extras.length > 0) { - if (kind !== "codex" && (extras.includes("model") || extras.includes("reasoningEffort") || extras.includes("codexSandbox"))) { - throw new TypeError(`${label}.model, ${label}.reasoningEffort, and ${label}.codexSandbox are codex-only; their subscription auth, model selection, and sandbox policy are Daimon-owned`); + if (kind === "agy" && (extras.includes("model") || extras.includes("reasoningEffort") || extras.includes("codexSandbox"))) { + throw new TypeError(`${label}.model, ${label}.reasoningEffort, and ${label}.codexSandbox are codex-only for agy; its subscription auth, model selection, and sandbox policy are Daimon-owned`); } + if (kind === "grok" && extras.includes("codexSandbox")) throw new TypeError(`${label}.codexSandbox is codex-only; the Grok worker sandbox is Daimon-owned`); throw new TypeError(`${label} must contain exactly ${allowed.join(", ")}`); } + if (kind === "grok") return grokEngine(input, label); return { kind: kind as OrganizationRuntimeEngineKind, ...(input.model === undefined ? {} : { model: nonEmpty(input.model, `${label}.model`) }), @@ -207,6 +209,22 @@ function engine(value: unknown, label: string): OrganizationRuntimeEngineIntent }; } +/** + * A Grok model is declared explicitly or not at all: both members from the + * closed broker lists, never one of them with the other inherited. Omitting + * both keeps a config parseable by pre-declaration producers; the brokered + * paths that need a model (`grokBrokerProjection.ts`, `service.json` v2) + * require it there instead of defaulting it here. + */ +function grokEngine(input: RecordValue, label: string): OrganizationRuntimeEngineIntent { + if ((input.model === undefined) !== (input.reasoningEffort === undefined)) throw new TypeError(`${label}.model and ${label}.reasoningEffort must be declared together for grok`); + if (input.model === undefined) return { kind: "grok" }; + const model = string(input.model, `${label}.model`), effort = string(input.reasoningEffort, `${label}.reasoningEffort`); + if (!(GROK_BROKER_MODELS as readonly string[]).includes(model)) throw new TypeError(`${label}.model must be one of ${GROK_BROKER_MODELS.join(", ")}`); + if (!(GROK_BROKER_REASONING_EFFORTS as readonly string[]).includes(effort)) throw new TypeError(`${label}.reasoningEffort must be one of ${GROK_BROKER_REASONING_EFFORTS.join(", ")}`); + return { kind: "grok", model, reasoningEffort: effort }; +} + function codexSandbox(value: unknown, label: string): OrganizationRuntimeEngineIntent["codexSandbox"] { const input = object(value, label); exact(input, ["mode", "networkAccess", "webSearch"], label); From 48e9b8b6178610bce48289ea6277cccf1a3a3522 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:17:49 +0200 Subject: [PATCH 06/21] feat: add the public Grok broker projection --- src/runtime/grokBrokerProjection.test.ts | 52 +++++++++ src/runtime/grokBrokerProjection.ts | 130 +++++++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 src/runtime/grokBrokerProjection.test.ts create mode 100644 src/runtime/grokBrokerProjection.ts diff --git a/src/runtime/grokBrokerProjection.test.ts b/src/runtime/grokBrokerProjection.test.ts new file mode 100644 index 0000000..aa8689a --- /dev/null +++ b/src/runtime/grokBrokerProjection.test.ts @@ -0,0 +1,52 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { GROK_ENGINE_BROKER, GROK_SUBSCRIPTION_REALM } from "../contracts/runtimeContractManifest.js"; +import { parseEngineBrokerServiceConfig } from "./engineBrokerServiceConfig.js"; +import { grokBrokerProjectionSha256, grokBrokerServiceRegistrationFor, resolveOrganizationGrokBrokerProjection, verifyGrokBrokerRegistrationMatchesProjection } from "./grokBrokerProjection.js"; +import { grokBrokerWorkerConfigSha256 } from "./grokBrokerWorkerConfig.js"; +import { grokWorkerSandboxProfileSha256 } from "./grokWorkerSandboxProfile.js"; + +const agent = (id: string, engine: Record) => ({ id, name: id, instructions: "Unused", workspacePath: `/var/lib/spawnfile/instance/workspace/agents/${id}`, runtimeHomePath: `/var/lib/spawnfile/instance/homes/${id}`, schedule: { kind: "disabled" }, engine }); +const config = { version: "noopolis.daimon.organization-runtime.v2", host: { bindHost: "127.0.0.1", port: 19700, controlTokenEnv: "UNIT_CONTROL_TOKEN" }, + agents: [agent("foreman", { kind: "grok", model: "grok-4.6", reasoningEffort: "low" }), agent("peer", { kind: "codex" })] }; +const options = { slot: 0, workerUid: 2_200, workerHomePath: "/var/lib/daimon-workers/2200", architecture: "arm64", usageLedgerPath: "/run/slots/0/usage/usage.jsonl", + limits: { maxRequests: 24, maxTokens: 400_000, timeoutMs: 480_000 }, acceptanceStorePath: "/run/paideia/control", denyPaths: ["/run/paideia", "/run/training/inputs"] } as const; + +test("the projection is Daimon's own renderers and collectors, fully declared and deterministic", () => { + const projection = resolveOrganizationGrokBrokerProjection(config, "foreman", options); + const denyPaths = [GROK_SUBSCRIPTION_REALM.bootstrapMountPath, GROK_SUBSCRIPTION_REALM.durableMountPath, "/run/paideia/control", "/var/lib/spawnfile/instance/homes/peer", "/var/lib/spawnfile/instance/workspace/agents/peer", "/run/paideia", "/run/training/inputs"].sort(); + assert.deepEqual(projection, { + version: "noopolis.daimon.grok-broker-projection.v1", agentId: "foreman", + workspacePath: "/var/lib/spawnfile/instance/workspace/agents/foreman", runtimeHomePath: "/var/lib/spawnfile/instance/homes/foreman", + workerUid: 2_200, slot: 0, profilePath: "/var/lib/daimon-workers/2200/.grok/sandbox.toml", profileSha256: grokWorkerSandboxProfileSha256(denyPaths), denyPaths, + workerConfigSha256: grokBrokerWorkerConfigSha256({ model: "grok-4.6", reasoningEffort: "low" }), systemPromptSha256: GROK_ENGINE_BROKER.worker.systemPromptSha256, + grokCliVersion: "1.0.34", grokExecutableSha256: GROK_ENGINE_BROKER.grokCliArtifacts.arm64.sha256, nativeAbiVersion: GROK_ENGINE_BROKER.nativeAbiVersion, + model: "grok-4.6", reasoningEffort: "low", limits: options.limits, usageLedgerPath: options.usageLedgerPath, + attestation: { platform: "linux/landlock", enforced: true, restrictNetwork: true, profileName: "daimon-strict", eventsPath: "/var/lib/daimon-workers/2200/.grok/sessions/sandbox-events.jsonl" } + }); + assert.equal(grokBrokerProjectionSha256(resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, denyPaths: [...options.denyPaths].reverse() })), grokBrokerProjectionSha256(projection)); + assert.match(grokBrokerProjectionSha256(projection), /^[a-f0-9]{64}$/u); +}); + +test("the projection refuses undeclared models, non-Grok agents, and a profile digest it did not render", () => { + assert.throws(() => resolveOrganizationGrokBrokerProjection({ ...config, agents: [agent("foreman", { kind: "grok" })] }, "foreman", options), /declared model/u); + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "peer", options), /known Grok agent/u); + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "missing", options), /known Grok agent/u); + // Mutation guard: skipping the digest comparison accepts a weaker profile's digest. + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, profileSha256: grokWorkerSandboxProfileSha256([]) }), /profile digest mismatch/u); + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, denyPaths: ["relative"] }), /deny path/u); + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, limits: { ...options.limits, maxRequests: 49 } }), /invalid/u); + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, usageLedgerPath: "/run/slots/0/usage/requests.jsonl" }), /invalid engine broker service config/u); + assert.equal(resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, profileSha256: resolveOrganizationGrokBrokerProjection(config, "foreman", options).profileSha256 }).agentId, "foreman"); +}); + +test("a provisioned registration must describe its projection exactly", () => { + const projection = resolveOrganizationGrokBrokerProjection(config, "foreman", options); + const parse = (registration: Record) => parseEngineBrokerServiceConfig({ version: "noopolis.daimon.engine-broker-service.v2", credentialHome: "/c", turnStore: "/t", registrations: [registration] }).registrations[0]!; + const registration = grokBrokerServiceRegistrationFor(projection); + verifyGrokBrokerRegistrationMatchesProjection(parse(registration), projection); + for (const drift of [{ profileSha256: grokWorkerSandboxProfileSha256([]) }, { model: { id: "grok-4.5", reasoningEffort: "low" } }, { limits: { ...options.limits, maxTokens: 400_001 } }, { usageLedgerPath: "/run/slots/1/usage/usage.jsonl" }]) { + assert.throws(() => verifyGrokBrokerRegistrationMatchesProjection(parse({ ...registration, ...drift }), projection), /does not match/u, JSON.stringify(drift)); + } +}); diff --git a/src/runtime/grokBrokerProjection.ts b/src/runtime/grokBrokerProjection.ts new file mode 100644 index 0000000..ac1e5f6 --- /dev/null +++ b/src/runtime/grokBrokerProjection.ts @@ -0,0 +1,130 @@ +import { createHash } from "node:crypto"; +import path from "node:path"; + +import { canonicalJson } from "../contracts/canonicalJson.js"; +import { DAIMON_GROK_SYSTEM_PROMPT } from "../contracts/grokWorkerContract.js"; +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import { grokSandboxProtectedPaths } from "./engineDispatcher.js"; +import { parseEngineBrokerServiceConfig, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +import { parseEngineBrokerTurnLimits, type EngineBrokerTurnLimits } from "./engineBrokerTurnAccounting.js"; +import { grokBrokerWorkerConfigSha256 } from "./grokBrokerWorkerConfig.js"; +import type { GrokBrokerModel, GrokBrokerReasoningEffort } from "./grokBrokerModelPolicy.js"; +import { GROK_WORKER_SANDBOX_PROFILE, grokWorkerEventsPathFor, renderGrokWorkerSandboxProfile, grokWorkerSandboxProfileSha256 } from "./grokWorkerSandboxProfile.js"; +import { parseOrganizationRuntimeConfig } from "./organizationRuntime.js"; + +export const GROK_BROKER_PROJECTION_VERSION = GROK_ENGINE_BROKER.projectionVersion; + +/** + * Everything a consumer (Spawnfile provisioning, Paideia's native adapter, the + * root slot supervisor) needs to know about one brokered Grok agent's slot, + * computed by Daimon from the same renderers and collectors the broker attests + * against. It never reads credentials, runs a worker, or touches the realm. + */ +export type OrganizationGrokBrokerProjection = Readonly<{ + version: typeof GROK_BROKER_PROJECTION_VERSION; + agentId: string; + workspacePath: string; + runtimeHomePath: string; + workerUid: number; + slot: number; + profilePath: string; + profileSha256: string; + denyPaths: readonly string[]; + workerConfigSha256: string; + systemPromptSha256: string; + grokCliVersion: string; + grokExecutableSha256: string; + nativeAbiVersion: number; + model: GrokBrokerModel; + reasoningEffort: GrokBrokerReasoningEffort; + limits: EngineBrokerTurnLimits; + usageLedgerPath: string; + attestation: Readonly<{ platform: "linux/landlock"; enforced: true; restrictNetwork: true; profileName: typeof GROK_WORKER_SANDBOX_PROFILE; eventsPath: string }>; +}>; + +export type OrganizationGrokBrokerProjectionOptions = Readonly<{ + /** Deployment-assigned slot identity. */ + slot: number; + workerUid: number; + /** The worker's home; its `GROK_HOME` is `/.grok`. */ + workerHomePath: string; + architecture: "arm64" | "x64"; + usageLedgerPath: string; + limits: EngineBrokerTurnLimits; + /** The wake-acceptance store, always denied like the Codex projection's. */ + acceptanceStorePath: string; + /** Evaluator and host-bind paths the deployment must keep from the worker (R4). */ + denyPaths?: readonly string[]; + /** When the caller already holds a rendered profile digest, it must equal Daimon's. */ + profileSha256?: string; +}>; + +/** + * Resolve the public Grok broker projection for one agent. + * + * Deterministic and I/O-free on purpose: its digest + * ({@link grokBrokerProjectionSha256}) is what the slot preflight receipt + * binds, so the supervisor that writes the receipt and the evaluator that reads + * it must compute byte-equal projections from the same inputs. + * + * The agent must be a Grok agent that *declares* its model and reasoning + * effort; nothing is defaulted. The deny list is Daimon's own protected set for + * this agent (realm, bootstrap, peers, acceptance store) plus the caller's + * evaluator paths, sorted and deduplicated exactly as the profile renderer + * does. A supplied `profileSha256` that differs is refused. + */ +export function resolveOrganizationGrokBrokerProjection(config: unknown, agentId: string, options: OrganizationGrokBrokerProjectionOptions): OrganizationGrokBrokerProjection { + const parsed = parseOrganizationRuntimeConfig(config); + const agent = parsed.agents.find((entry) => entry.id === agentId); + if (agent === undefined || agent.engine.kind !== "grok") throw new Error("Grok broker projection requires a known Grok agent"); + if (agent.engine.model === undefined || agent.engine.reasoningEffort === undefined) throw new Error("Grok broker projection requires a declared model and reasoning effort"); + for (const [label, value] of [["workerHomePath", options.workerHomePath], ["acceptanceStorePath", options.acceptanceStorePath]] as const) { + if (!path.posix.isAbsolute(value) || path.posix.normalize(value) !== value || value === "/" || value.endsWith("/")) throw new Error(`Grok broker projection requires a canonical absolute ${label}`); + } + const model = agent.engine.model as GrokBrokerModel, reasoningEffort = agent.engine.reasoningEffort as GrokBrokerReasoningEffort; + const denyPaths = [...new Set([...grokSandboxProtectedPaths(agent.id, parsed.agents, [options.acceptanceStorePath]), ...(options.denyPaths ?? [])])].sort(); + renderGrokWorkerSandboxProfile(denyPaths); + const profileSha256 = grokWorkerSandboxProfileSha256(denyPaths); + if (options.profileSha256 !== undefined && options.profileSha256 !== profileSha256) throw new Error("Grok broker projection profile digest mismatch"); + const workerConfigSha256 = grokBrokerWorkerConfigSha256({ model, reasoningEffort }); + if (workerConfigSha256 !== GROK_ENGINE_BROKER.worker.configSha256[model][reasoningEffort]) throw new Error("Grok broker projection worker config drifted from the manifest"); + if (createHash("sha256").update(DAIMON_GROK_SYSTEM_PROMPT).digest("hex") !== GROK_ENGINE_BROKER.worker.systemPromptSha256) throw new Error("Grok broker projection system prompt drifted from the manifest"); + const artifact = GROK_ENGINE_BROKER.grokCliArtifacts[options.architecture]; + if (artifact === undefined) throw new Error("Grok broker projection requires a pinned architecture"); + const profilePath = path.posix.join(options.workerHomePath, ".grok", "sandbox.toml"); + const projection: OrganizationGrokBrokerProjection = { + version: GROK_BROKER_PROJECTION_VERSION, agentId, workspacePath: agent.workspacePath, runtimeHomePath: agent.runtimeHomePath, + workerUid: options.workerUid, slot: options.slot, profilePath, profileSha256, denyPaths, workerConfigSha256, + systemPromptSha256: GROK_ENGINE_BROKER.worker.systemPromptSha256, grokCliVersion: GROK_ENGINE_BROKER.grokCliVersion, grokExecutableSha256: artifact.sha256, + nativeAbiVersion: GROK_ENGINE_BROKER.nativeAbiVersion, model, reasoningEffort, limits: parseEngineBrokerTurnLimits(options.limits), usageLedgerPath: options.usageLedgerPath, + attestation: { platform: "linux/landlock", enforced: true, restrictNetwork: true, profileName: GROK_WORKER_SANDBOX_PROFILE, eventsPath: grokWorkerEventsPathFor(profilePath) } + }; + // The registration this projection implies must itself be a valid v2 service.json entry. + grokBrokerServiceRegistrationFor(projection); + return projection; +} + +/** sha256 over the projection's canonical JSON; what a slot preflight receipt binds. */ +export const grokBrokerProjectionSha256 = (projection: OrganizationGrokBrokerProjection): string => + createHash("sha256").update(canonicalJson(projection)).digest("hex"); + +/** The `service.json` v2 registration a deployment provisions for this projection, validated by the broker's own parser. */ +export function grokBrokerServiceRegistrationFor(projection: OrganizationGrokBrokerProjection): Readonly> { + const registration = { + agentId: projection.agentId, slot: projection.slot, workerUid: projection.workerUid, workspace: projection.workspacePath, + profilePath: projection.profilePath, eventsPath: projection.attestation.eventsPath, profileSha256: projection.profileSha256, + usageLedgerPath: projection.usageLedgerPath, limits: projection.limits, model: { id: projection.model, reasoningEffort: projection.reasoningEffort } + }; + parseEngineBrokerServiceConfig({ version: "noopolis.daimon.engine-broker-service.v2", credentialHome: GROK_ENGINE_BROKER.credentialHomePath, turnStore: GROK_ENGINE_BROKER.turnStorePath, registrations: [registration] }); + return registration; +} + +/** + * Refuses a provisioned registration that does not describe this projection: + * any differing member — a weaker profile digest, another model, raised + * limits, a different ledger — is a mismatch, never a merge. + */ +export function verifyGrokBrokerRegistrationMatchesProjection(registration: EngineBrokerServiceRegistration, projection: OrganizationGrokBrokerProjection): void { + const expected = parseEngineBrokerServiceConfig({ version: "noopolis.daimon.engine-broker-service.v2", credentialHome: GROK_ENGINE_BROKER.credentialHomePath, turnStore: GROK_ENGINE_BROKER.turnStorePath, registrations: [grokBrokerServiceRegistrationFor(projection)] }).registrations[0]!; + if (canonicalJson(expected) !== canonicalJson(registration)) throw new Error("Grok broker registration does not match its projection"); +} From b5be62ca96f7afcf792338901611e5a0d229d206 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:17:49 +0200 Subject: [PATCH 07/21] feat: define the Grok slot preflight receipt schema with fixtures and export the broker contract --- package-lock.json | 3 +- package.json | 3 +- .../grok-slot-preflight/projection-input.json | 20 +++++ .../receipt.missing-canary.json | 32 ++++++++ .../receipt.projection-mismatch.json | 37 ++++++++++ .../receipt.readable-canary.json | 37 ++++++++++ .../receipt.unknown-member.json | 38 ++++++++++ .../grok-slot-preflight/receipt.valid.v1.json | 37 ++++++++++ src/runtime/grokSlotPreflightReceipt.test.ts | 44 +++++++++++ src/runtime/grokSlotPreflightReceipt.ts | 74 +++++++++++++++++++ src/runtime/index.ts | 10 +++ 11 files changed, 333 insertions(+), 2 deletions(-) create mode 100644 src/runtime/fixtures/grok-slot-preflight/projection-input.json create mode 100644 src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json create mode 100644 src/runtime/fixtures/grok-slot-preflight/receipt.projection-mismatch.json create mode 100644 src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json create mode 100644 src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json create mode 100644 src/runtime/fixtures/grok-slot-preflight/receipt.valid.v1.json create mode 100644 src/runtime/grokSlotPreflightReceipt.test.ts create mode 100644 src/runtime/grokSlotPreflightReceipt.ts diff --git a/package-lock.json b/package-lock.json index d046ccf..ffeb234 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,8 @@ "@earendil-works/pi-coding-agent": "^0.79.10", "@modelcontextprotocol/sdk": "^1.29.0", "@noopolis/mneme": "^0.1.1", - "ajv": "^8.17.1" + "ajv": "^8.17.1", + "zod": "^4.4.3" }, "bin": { "daimon-runtime": "dist/runtime/cli.js" diff --git a/package.json b/package.json index dc42793..1f22e90 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,8 @@ "@earendil-works/pi-coding-agent": "^0.79.10", "@modelcontextprotocol/sdk": "^1.29.0", "@noopolis/mneme": "^0.1.1", - "ajv": "^8.17.1" + "ajv": "^8.17.1", + "zod": "^4.4.3" }, "devDependencies": { "@types/node": "^24.12.4", diff --git a/src/runtime/fixtures/grok-slot-preflight/projection-input.json b/src/runtime/fixtures/grok-slot-preflight/projection-input.json new file mode 100644 index 0000000..b75a102 --- /dev/null +++ b/src/runtime/fixtures/grok-slot-preflight/projection-input.json @@ -0,0 +1,20 @@ +{ + "config": { + "version": "noopolis.daimon.organization-runtime.v2", + "host": { "bindHost": "127.0.0.1", "port": 19700, "controlTokenEnv": "DAIMON_CONTROL_TOKEN" }, + "agents": [ + { "id": "foreman", "name": "Foreman", "instructions": "Fixture agent.", "workspacePath": "/var/lib/spawnfile/instance/workspace/agents/foreman", "runtimeHomePath": "/var/lib/spawnfile/instance/homes/foreman", "schedule": { "kind": "disabled" }, "engine": { "kind": "grok", "model": "grok-4.6", "reasoningEffort": "low" } } + ] + }, + "agentId": "foreman", + "options": { + "slot": 0, + "workerUid": 2200, + "workerHomePath": "/var/lib/daimon-workers/2200", + "architecture": "arm64", + "usageLedgerPath": "/run/daimon-slots/0/usage/usage.jsonl", + "limits": { "maxRequests": 24, "maxTokens": 400000, "timeoutMs": 480000 }, + "acceptanceStorePath": "/run/paideia/control", + "denyPaths": ["/run/paideia", "/run/training/inputs"] + } +} diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json new file mode 100644 index 0000000..6671c48 --- /dev/null +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json @@ -0,0 +1,32 @@ +{ + "version": "noopolis.daimon.grok-slot-preflight.v1", + "slot": 0, + "worker_uid": 2200, + "projection_sha256": "b41a6952dfe3319f014866e7468df69ca2e77cfd28d781ac39cfdf0d3d587bd8", + "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", + "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", + "canaries": [ + { + "path": "/run/paideia/control", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/training/inputs", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-bootstrap-auth", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-subscription-realm", + "method": "sandboxed-read", + "result": "denied" + } + ], + "created_at": "2026-09-17T12:00:00.000Z" +} diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.projection-mismatch.json b/src/runtime/fixtures/grok-slot-preflight/receipt.projection-mismatch.json new file mode 100644 index 0000000..cdcb916 --- /dev/null +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.projection-mismatch.json @@ -0,0 +1,37 @@ +{ + "version": "noopolis.daimon.grok-slot-preflight.v1", + "slot": 0, + "worker_uid": 2200, + "projection_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", + "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", + "canaries": [ + { + "path": "/run/paideia", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/paideia/control", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/training/inputs", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-bootstrap-auth", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-subscription-realm", + "method": "sandboxed-read", + "result": "denied" + } + ], + "created_at": "2026-09-17T12:00:00.000Z" +} diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json new file mode 100644 index 0000000..34d05ff --- /dev/null +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json @@ -0,0 +1,37 @@ +{ + "version": "noopolis.daimon.grok-slot-preflight.v1", + "slot": 0, + "worker_uid": 2200, + "projection_sha256": "b41a6952dfe3319f014866e7468df69ca2e77cfd28d781ac39cfdf0d3d587bd8", + "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", + "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", + "canaries": [ + { + "path": "/run/paideia", + "method": "sandboxed-read", + "result": "readable" + }, + { + "path": "/run/paideia/control", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/training/inputs", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-bootstrap-auth", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-subscription-realm", + "method": "sandboxed-read", + "result": "denied" + } + ], + "created_at": "2026-09-17T12:00:00.000Z" +} diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json b/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json new file mode 100644 index 0000000..c69658a --- /dev/null +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json @@ -0,0 +1,38 @@ +{ + "version": "noopolis.daimon.grok-slot-preflight.v1", + "slot": 0, + "worker_uid": 2200, + "projection_sha256": "b41a6952dfe3319f014866e7468df69ca2e77cfd28d781ac39cfdf0d3d587bd8", + "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", + "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", + "canaries": [ + { + "path": "/run/paideia", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/paideia/control", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/training/inputs", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-bootstrap-auth", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-subscription-realm", + "method": "sandboxed-read", + "result": "denied" + } + ], + "created_at": "2026-09-17T12:00:00.000Z", + "operator": "root" +} diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v1.json b/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v1.json new file mode 100644 index 0000000..46954bc --- /dev/null +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v1.json @@ -0,0 +1,37 @@ +{ + "version": "noopolis.daimon.grok-slot-preflight.v1", + "slot": 0, + "worker_uid": 2200, + "projection_sha256": "b41a6952dfe3319f014866e7468df69ca2e77cfd28d781ac39cfdf0d3d587bd8", + "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", + "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", + "canaries": [ + { + "path": "/run/paideia", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/paideia/control", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/training/inputs", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-bootstrap-auth", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-subscription-realm", + "method": "sandboxed-read", + "result": "denied" + } + ], + "created_at": "2026-09-17T12:00:00.000Z" +} diff --git a/src/runtime/grokSlotPreflightReceipt.test.ts b/src/runtime/grokSlotPreflightReceipt.test.ts new file mode 100644 index 0000000..77d80c1 --- /dev/null +++ b/src/runtime/grokSlotPreflightReceipt.test.ts @@ -0,0 +1,44 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { resolveOrganizationGrokBrokerProjection } from "./grokBrokerProjection.js"; +import { parseGrokSlotPreflightReceipt, verifyGrokSlotPreflightReceipt } from "./grokSlotPreflightReceipt.js"; + +const fixture = async (name: string): Promise> => JSON.parse(await readFile(new URL(`./fixtures/grok-slot-preflight/${name}`, import.meta.url), "utf8")) as Record; +const projection = async () => { const input = await fixture("projection-input.json") as { config: unknown; agentId: string; options: Parameters[2] }; return resolveOrganizationGrokBrokerProjection(input.config, input.agentId, input.options); }; + +test("the committed valid receipt fixture proves the committed projection input", async () => { + const receipt = verifyGrokSlotPreflightReceipt(await fixture("receipt.valid.v1.json"), await projection()); + assert.equal(receipt.canaries.length, (await projection()).denyPaths.length); + assert.ok(receipt.canaries.every((canary) => canary.method === "sandboxed-read" && canary.result === "denied")); +}); + +test("the schema refuses a readable canary, an unknown member, duplicates and malformed digests or times", async () => { + const valid = await fixture("receipt.valid.v1.json"); + await assert.rejects(async () => parseGrokSlotPreflightReceipt(await fixture("receipt.readable-canary.json")), /invalid Grok slot preflight receipt/u); + await assert.rejects(async () => parseGrokSlotPreflightReceipt(await fixture("receipt.unknown-member.json")), /invalid Grok slot preflight receipt/u); + const canaries = valid.canaries as Record[]; + for (const bad of [ + { ...valid, canaries: [...canaries, canaries[0]] }, + { ...valid, canaries: [{ ...canaries[0], method: "stat" }] }, + { ...valid, canaries: [{ ...canaries[0], path: "/run/../etc" }] }, + { ...valid, canaries: [{ ...canaries[0], extra: true }] }, + { ...valid, canaries: [] }, + { ...valid, projection_sha256: "A".repeat(64) }, + { ...valid, worker_uid: 2_000 }, + { ...valid, created_at: "2026-09-17T12:00:00Z" }, + { ...valid, version: "noopolis.daimon.grok-slot-preflight.v2" } + ]) assert.throws(() => parseGrokSlotPreflightReceipt(bad), /invalid Grok slot preflight receipt/u); +}); + +test("a receipt for a different projection, slot, profile or deny set is refused", async () => { + const projected = await projection(); + const valid = await fixture("receipt.valid.v1.json"); + // Mutation guard: dropping the digest comparison accepts this fixture. + await assert.rejects(async () => verifyGrokSlotPreflightReceipt(await fixture("receipt.projection-mismatch.json"), projected), /projection_sha256/u); + await assert.rejects(async () => verifyGrokSlotPreflightReceipt(await fixture("receipt.missing-canary.json"), projected), /canaries/u); + assert.throws(() => verifyGrokSlotPreflightReceipt(valid, { ...projected, limits: { ...projected.limits, maxTokens: 1 } }), /projection_sha256/u); + assert.throws(() => verifyGrokSlotPreflightReceipt({ ...valid, sandbox_profile_sha256: "1".repeat(64) }, projected), /sandbox_profile_sha256/u); + assert.throws(() => verifyGrokSlotPreflightReceipt({ ...valid, slot: 1 }, projected), /slot/u); +}); diff --git a/src/runtime/grokSlotPreflightReceipt.ts b/src/runtime/grokSlotPreflightReceipt.ts new file mode 100644 index 0000000..3bed48d --- /dev/null +++ b/src/runtime/grokSlotPreflightReceipt.ts @@ -0,0 +1,74 @@ +import path from "node:path"; +import { z } from "zod"; + +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import { grokBrokerProjectionSha256, type OrganizationGrokBrokerProjection } from "./grokBrokerProjection.js"; + +export const GROK_SLOT_PREFLIGHT_VERSION = GROK_ENGINE_BROKER.slotPreflightVersion; + +const sha256 = z.string().regex(/^[a-f0-9]{64}$/u); +const canonicalAbsolute = z.string().max(4_096).refine((value) => path.posix.isAbsolute(value) && path.posix.normalize(value) === value && value !== "/" && !value.endsWith("/") && !value.includes("\0"), "canonical absolute path"); + +/** + * One denied-path canary: the root supervisor ran a real sandboxed read of + * `path` as the slot's worker uid (a stub-model turn under the attested + * profile) and the read was denied. Only denials are representable — a + * supervisor that observed a readable path writes no receipt at all. + */ +export const grokSlotPreflightCanarySchema = z.strictObject({ + path: canonicalAbsolute, + method: z.literal("sandboxed-read"), + result: z.literal("denied") +}); + +/** + * `noopolis.daimon.grok-slot-preflight.v1`: what the root slot supervisor (P5) + * writes after provisioning or recycling one broker slot, and what an + * evaluator (Paideia, P4) must hold before it runs a Grok subject turn in that + * slot. It binds the slot to one exact projection by digest, so any change to + * the model, limits, deny list, profile, worker config, or pinned executable + * invalidates it. + */ +export const grokSlotPreflightReceiptSchema = z.strictObject({ + version: z.literal(GROK_SLOT_PREFLIGHT_VERSION), + slot: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER), + worker_uid: z.number().int().min(GROK_ENGINE_BROKER.identities.firstWorkerUid).max(4_294_967_294), + projection_sha256: sha256, + /** The bubblewrap/Landlock `daimon-strict` profile bytes' digest (the projection's `profileSha256`). */ + sandbox_profile_sha256: sha256, + /** The container seccomp profile the worker ran under. */ + seccomp_profile_sha256: sha256, + grok_executable_sha256: sha256, + canaries: z.array(grokSlotPreflightCanarySchema).min(1).max(256), + created_at: z.string().regex(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/u).refine((value) => !Number.isNaN(Date.parse(value)) && new Date(value).toISOString() === value, "exact RFC3339 timestamp") +}).superRefine((receipt, context) => { + const paths = receipt.canaries.map((canary) => canary.path); + if (new Set(paths).size !== paths.length) context.addIssue({ code: "custom", path: ["canaries"], message: "duplicate canary path" }); +}); + +export type GrokSlotPreflightReceipt = z.infer; + +/** Strict parse: unknown members, off-contract values, or duplicate canaries throw. */ +export function parseGrokSlotPreflightReceipt(value: unknown): GrokSlotPreflightReceipt { + const result = grokSlotPreflightReceiptSchema.safeParse(value); + if (!result.success) throw new TypeError(`invalid Grok slot preflight receipt: ${result.error.issues.map((issue) => `${issue.path.join(".") || "receipt"}: ${issue.message}`).join("; ")}`); + return result.data; +} + +/** + * Parse a receipt and require that it proves *this* projection's slot: same + * digest, slot, worker uid, profile and executable, and a denied canary for + * exactly every projected deny path (no more, no fewer). + */ +export function verifyGrokSlotPreflightReceipt(value: unknown, projection: OrganizationGrokBrokerProjection): GrokSlotPreflightReceipt { + const receipt = parseGrokSlotPreflightReceipt(value); + const mismatch = (member: string): never => { throw new Error(`Grok slot preflight receipt does not match the projection: ${member}`); }; + if (receipt.projection_sha256 !== grokBrokerProjectionSha256(projection)) mismatch("projection_sha256"); + if (receipt.slot !== projection.slot) mismatch("slot"); + if (receipt.worker_uid !== projection.workerUid) mismatch("worker_uid"); + if (receipt.sandbox_profile_sha256 !== projection.profileSha256) mismatch("sandbox_profile_sha256"); + if (receipt.grok_executable_sha256 !== projection.grokExecutableSha256) mismatch("grok_executable_sha256"); + const denied = receipt.canaries.map((canary) => canary.path).sort(); + if (denied.length !== projection.denyPaths.length || denied.some((entry, index) => entry !== projection.denyPaths[index])) mismatch("canaries"); + return receipt; +} diff --git a/src/runtime/index.ts b/src/runtime/index.ts index b72ee90..83d00a4 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -5,6 +5,16 @@ export { createOrganizationRuntimeHost } from "./organizationRuntimeHost.js"; export { createOrganizationRuntimeControlHost } from "./organizationRuntimeControl.js"; export { CODEX_SANDBOX_PROJECTION_VERSION, resolveOrganizationCodexSandboxProjection, type OrganizationCodexSandboxProjection } from "./codexSandboxProjection.js"; +export { GROK_BROKER_PROJECTION_VERSION, grokBrokerProjectionSha256, grokBrokerServiceRegistrationFor, resolveOrganizationGrokBrokerProjection, + verifyGrokBrokerRegistrationMatchesProjection, type OrganizationGrokBrokerProjection, type OrganizationGrokBrokerProjectionOptions } from "./grokBrokerProjection.js"; +export { GROK_SLOT_PREFLIGHT_VERSION, grokSlotPreflightCanarySchema, grokSlotPreflightReceiptSchema, parseGrokSlotPreflightReceipt, + verifyGrokSlotPreflightReceipt, type GrokSlotPreflightReceipt } from "./grokSlotPreflightReceipt.js"; +export { grokBrokerWorkerConfigSha256, renderGrokBrokerWorkerConfig } from "./grokBrokerWorkerConfig.js"; +export { grokWorkerSandboxProfileSha256, renderGrokWorkerSandboxProfile } from "./grokWorkerSandboxProfile.js"; +export { engineBrokerRequestLedgerPathFor, parseEngineBrokerServiceConfig, type EngineBrokerServiceConfig, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +export { DEFAULT_GROK_BROKER_TURN_LIMITS, ENGINE_BROKER_LIMIT_REASONS, type EngineBrokerLimitReason, type EngineBrokerTurnAccounting, + type EngineBrokerTurnLimits, type EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; +export { dedupeTurnUsageRows } from "./turnUsageLedger.js"; export { WakeTransitionLockBlockedError } from "./wakeAcceptanceStore.js"; export { OFFLINE_RECONCILIATION_BLOCKED_CODE, From 55852f3d24018b6d1cade6ddd1f5b3d1f8470fd6 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:18:19 +0200 Subject: [PATCH 08/21] docs: describe Grok broker limits, sealed accounting, projection and slot receipts --- docs/engines.md | 15 ++++++++++++ src/runtime/AGENTS.md | 56 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/docs/engines.md b/docs/engines.md index ba2b288..b2517f6 100644 --- a/docs/engines.md +++ b/docs/engines.md @@ -88,6 +88,21 @@ skills, workflows, plan mode, subagents, memory or web search, and a declared model and reasoning effort from a closed list (default `grok-4.6` at `low`). The broker proxy refuses any request outside that shape before it spends. +Each broker registration (`service.json` v2) declares its model and effort, +its usage ledger, and turn limits `{maxRequests, maxTokens, timeoutMs}`. A wake +may only lower them (`DAIMON_ENGINE_WAKE_TIMEOUT_MS`, +`DAIMON_ENGINE_WAKE_TOKEN_CEILING`; the `DAIMON_CODEX_WAKE_*` names are +aliases). The proxy refuses request `maxRequests + 1` and any request after the +deadline with HTTP 429 before upstream, and stops admitting requests once the +upstream-reported running total (cached input included) reaches `maxTokens`, so +a turn overshoots its token ceiling by at most one request. A tripped limit +kills the worker. The broker seals every terminal turn with its usage, request +count, declared model and limit reason, and writes one usage row (keyed by +`turn`) plus per-request rows for completed and failed turns alike; a replayed +turn is never metered twice. `resolveOrganizationGrokBrokerProjection` exposes +a slot's full declared shape, and `noopolis.daimon.grok-slot-preflight.v1` +receipts bind a slot's denied-path canaries to that projection's digest. + AGY uses OS-native secure storage through one private D-Bus and Secret Service realm. Enroll it once with: diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 4ddaa1b..ca8eb55 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -28,6 +28,51 @@ tool), and any body whose `model`/`reasoning_effort` differ from the declared `grokBrokerModelPolicy.ts` policy (closed lists; default `grok-4.6`/`low`). The model override header follows that declaration. +The proxy is the per-turn limit gate too. Every broker turn registers a +`grokBrokerTurnMeter.ts` meter with its registration's model policy, and the +proxy forwards nothing for a turn without one. After a body is proven a lean +worker request and before any upstream call, the meter refuses request +`maxRequests + 1`, any request past `timeoutMs`, and any request once the +upstream-reported running total (prompt tokens *including* cached, plus +completion) has reached `maxTokens` — HTTP 429, and the tripped limit aborts +the worker through the ordinary cancel/kill path. The token ceiling is checked +between requests, so a turn overshoots it by at most the last admitted +request; if an upstream body carries no `usage`, only `maxRequests` and +`timeoutMs` bound that turn mid-flight. A broker timer also trips `timeout` +for a worker that is mid-request. Limits come from `service.json` v2 +(`engineBrokerServiceConfig.ts`; v1 gets `GROK_ENGINE_BROKER.turnLimits.v1Defaults`) +and a wake may only lower them: a raise is refused as `invalid_request`, never +clamped. + +The broker stays the single sealed usage writer. `grokEngineBrokerTurn.ts` +seals every terminal turn — completed, failed, limit, cancelled — through +`finishBrokerTurnWithUsage` (`grokEngineBrokerMetering.ts`): the turn registry +record v2 stores the control-protocol v2 terminal response *with* its +numeric-only accounting (`usage`, `outcome`, declared `model`, `requests`, +closed `limitReason`), and only then are ledger rows appended. A replay +returns the sealed accounting and never meters again; v1 records still replay +(upgraded with `usage: null`). Completed usage is the terminal `result.usage`; +a failed turn's partial usage is its per-request stream frames +(`../pi/grokStreamUsage.ts`) when output arrived, else the upstream usage the +proxy saw. Usage rows carry `turn` (the idempotency key readers dedupe on — +`wakeFuse.ts` does), `limit_reason` and `model`; per-request rows go to +`requests.jsonl` beside the registration's `usageLedgerPath` with proxy-measured +`started_at`/`ended_at`. A provider-reported model key must map to the declared +model (`grok-4.6-build` → `grok-4.6`), otherwise the turn fails as rejected and +is still metered. Control protocol v2 is refused-v1 on the wire because both +ends ship in this package. + +`grokBrokerProjection.ts` is the public, I/O-free projection of one brokered +Grok agent's slot (`noopolis.daimon.grok-broker-projection.v1`): Daimon's own +deny collectors plus the caller's evaluator paths, profile/config/prompt +digests, pinned executable, model, limits and ledger. A Grok agent must declare +`model` and `reasoningEffort` for it; nothing is defaulted, and a supplied +profile digest that differs is refused. `grokSlotPreflightReceipt.ts` is the +zod schema a root slot supervisor's receipt must satisfy +(`noopolis.daimon.grok-slot-preflight.v1`, fixtures under +`fixtures/grok-slot-preflight/`); `verifyGrokSlotPreflightReceipt` binds it to +the projection digest and requires a denied canary for exactly every deny path. + `grokBrokerWorkerConfig.ts` is the only source of worker `config.toml` bytes; the manifest pins the sha256 of every model/effort combination and the broker refuses a turn whose worker config does not hash to the declared one. Three @@ -91,8 +136,10 @@ the only place its per-wake tool-call bound is decided. `maxToolTurns` only mediates daimon-MCP tool calls; Codex's own shell (`exec_command`) is never routed through it, so Codex gets its own bounds instead — `DEFAULT_CODEX_WAKE_TIMEOUT_MS` (wall clock) and -`DEFAULT_CODEX_WAKE_TOKEN_CEILING`, both in `../pi/cliSession.ts`, overridable -via `DAIMON_CODEX_WAKE_TIMEOUT_MS`/`DAIMON_CODEX_WAKE_TOKEN_CEILING`. The token +`DEFAULT_CODEX_WAKE_TOKEN_CEILING`, both in `../pi/engineWakeLimits.ts`, overridable +via the engine-neutral `DAIMON_ENGINE_WAKE_TIMEOUT_MS`/`DAIMON_ENGINE_WAKE_TOKEN_CEILING` +(the `DAIMON_CODEX_*` names are aliases; conflicting values are refused), which +the dispatcher also passes to the Grok broker as lowering limits. The token ceiling can only be checked when Codex reports it: its `--json` stream carries usage exactly once, on the turn's own `turn.completed`, so crossing it kills the child immediately and fails the wake instead of letting an over-budget @@ -133,7 +180,10 @@ request count) look like the first without proving it. `../pi/cliChildOutput.ts` carries the thread id off Codex's own `thread.started` frame, and `../pi/codexRolloutUsage.ts` reads that thread's rollout under `$CODEX_HOME/sessions/**` for the per-request `token_usage_record` frames the -`--json` stream never emits. Rows go to `requests.jsonl` beside `usage.jsonl` +`--json` stream never emits. Each Codex row carries its own `started_at`/`ended_at` +from the rollout frame timestamps (end = the usage frame; start = the first +non-usage frame after the previous request's usage frame, else that request's +end), absent rather than substituted when a frame has no valid timestamp. Rows go to `requests.jsonl` beside `usage.jsonl` (`DAIMON_TURN_REQUESTS_LEDGER_PATH` relocates it) under the same invariants: a wake whose rollout is absent, unreadable, or undecodable writes *nothing*, because a fabricated zero is byte-identical to a measured one; and every failure From 50a932557797e780dd24d4bf945b00de168f1903 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:19:39 +0200 Subject: [PATCH 09/21] test: prove Grok per-request rows carry measured proxy intervals, not the append time --- src/runtime/grokEngineBrokerUsage.test.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index f66631d..20ab62d 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -49,7 +49,7 @@ const untilAborted = (signal: AbortSignal): Promise => new Promise((_reso const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId: string, worker: Worker, overrides?: Parameters[6], limits?: EngineBrokerServiceRegistration["limits"], turnStore?: string) => ReturnType; usageRows: () => Promise[]>; requestRows: () => Promise[]>; upstreamCalls: () => number }>) => Promise, usageLedgerPath?: string): Promise => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-usage-")); let calls = 0; - const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(`data: ${JSON.stringify({ choices: [], usage: upstreamUsage })}\n\ndata: [DONE]\n\n`) }; }, undefined, 0); + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; await new Promise((resolve) => setTimeout(resolve, 15)); return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(`data: ${JSON.stringify({ choices: [], usage: upstreamUsage })}\n\ndata: [DONE]\n\n`) }; }, undefined, 0); const ledger = usageLedgerPath ?? path.join(root, "usage.jsonl"); const rows = async (file: string) => (await readFile(file, "utf8").catch(() => "")).split("\n").filter((line) => line.length > 0).map((line) => JSON.parse(line) as Record); try { @@ -92,8 +92,10 @@ test("a completed turn seals its accounting, writes one usage row and per-reques [TURN_REQUEST_LEDGER_VERSION, "grok", 1, 2, 2_797, 109, 2_688, 2_810, turnIdFor("foreman", "wake-1")] ]); // Mutation guard: stamping every request with the wake end collapses these. - for (const row of requests) assert.match(String(row.started_at), /^\d{4}-\d{2}-\d{2}T/u); - assert.ok(String(requests[0]!.ended_at) <= String(requests[1]!.started_at), "request 1 ends before request 2 starts"); + // The upstream stub takes 15 ms per request, so each request has a measurable interval. + const [a, b] = requests.map((row) => [Date.parse(String(row.started_at)), Date.parse(String(row.ended_at))] as const); + assert.ok(a![0] < a![1] && a![1] <= b![0] && b![0] < b![1], JSON.stringify(requests.map((row) => [row.started_at, row.ended_at]))); + assert.ok(b![1] <= Date.parse(String(requests[1]!.at)), "every request ended before the rows were appended"); }); }); From 39deffaf3c8ea10c0a4cd11cadea95a4899cb3f1 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:23:41 +0200 Subject: [PATCH 10/21] fix: count a killed turn's in-flight request in every Grok per-request row --- src/runtime/grokEngineBrokerMetering.ts | 2 +- src/runtime/grokEngineBrokerUsage.test.ts | 19 +++++++++++-------- src/runtime/turnRequestLedger.ts | 9 +++++++-- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/runtime/grokEngineBrokerMetering.ts b/src/runtime/grokEngineBrokerMetering.ts index f254554..bd4e482 100644 --- a/src/runtime/grokEngineBrokerMetering.ts +++ b/src/runtime/grokEngineBrokerMetering.ts @@ -39,5 +39,5 @@ export async function finishBrokerTurnWithUsage(turns: EngineBrokerTurnRegistry, outcome: terminal.kind === "completed" ? { status: "completed" } : { status: "failed", reason: detail.reason ?? "unknown" }, turn: terminal.turnId, limitReason: terminal.limitReason, model: terminal.model }); - await recordGrokTurnRequests(metering.requestLedgerPath, { agent: metering.agentId, wake: metering.wakeId, turn: terminal.turnId, model: terminal.model, requests: detail.requests, ...(detail.session === undefined ? {} : { session: detail.session }) }); + await recordGrokTurnRequests(metering.requestLedgerPath, { agent: metering.agentId, wake: metering.wakeId, turn: terminal.turnId, model: terminal.model, requests: detail.requests, requestCount: terminal.requests, ...(detail.session === undefined ? {} : { session: detail.session }) }); } diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index 20ab62d..408850b 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -46,10 +46,10 @@ const untilAborted = (signal: AbortSignal): Promise => new Promise((_reso * talks to the proxy exactly as the native worker does (capability bearer, * pinned client version, lean body). Only the launcher and attestation are fakes. */ -const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId: string, worker: Worker, overrides?: Parameters[6], limits?: EngineBrokerServiceRegistration["limits"], turnStore?: string) => ReturnType; usageRows: () => Promise[]>; requestRows: () => Promise[]>; upstreamCalls: () => number }>) => Promise, usageLedgerPath?: string): Promise => { +const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId: string, worker: Worker, overrides?: Parameters[6], limits?: EngineBrokerServiceRegistration["limits"], turnStore?: string) => ReturnType; usageRows: () => Promise[]>; requestRows: () => Promise[]>; upstreamCalls: () => number }>) => Promise, usageLedgerPath?: string, upstreamDelayMs: (call: number) => number = () => 15): Promise => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-usage-")); let calls = 0; - const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; await new Promise((resolve) => setTimeout(resolve, 15)); return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(`data: ${JSON.stringify({ choices: [], usage: upstreamUsage })}\n\ndata: [DONE]\n\n`) }; }, undefined, 0); + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; await new Promise((resolve) => setTimeout(resolve, upstreamDelayMs(calls))); return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(`data: ${JSON.stringify({ choices: [], usage: upstreamUsage })}\n\ndata: [DONE]\n\n`) }; }, undefined, 0); const ledger = usageLedgerPath ?? path.join(root, "usage.jsonl"); const rows = async (file: string) => (await readFile(file, "utf8").catch(() => "")).split("\n").filter((line) => line.length > 0).map((line) => JSON.parse(line) as Record); try { @@ -128,13 +128,16 @@ test("the token ceiling stops a turn one request past the ceiling at most", asyn }); test("the wall-clock limit aborts a worker that is mid-request", async () => { - await withBroker(async ({ turn, usageRows }) => { + await withBroker(async ({ turn, usageRows, requestRows }) => { const started = Date.now(); - const worker: Worker = async (send, signal) => { assert.equal(await send(), 200); return untilAborted(signal); }; - await assert.rejects(turn("wake-4", worker, { timeoutMs: 1_000 }), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.code === "limit_exceeded" && error.accounting?.limitReason === "timeout"); - assert.ok(Date.now() - started < 5_000); - assert.deepEqual((await usageRows()).map((row) => [row.reason, row.limit_reason, row.total]), [["wake_timeout", "timeout", 2_775]]); - }); + // Request 2 is still upstream (1.5 s) when the 1 s wall clock fires. + const worker: Worker = async (send, signal) => { assert.equal(await send(), 200); void send().catch(() => undefined); return untilAborted(signal); }; + await assert.rejects(turn("wake-4", worker, { timeoutMs: 1_000 }), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.code === "limit_exceeded" && error.accounting?.limitReason === "timeout" && error.accounting.requests === 2); + assert.ok(Date.now() - started < 1_400, "the turn ends at the deadline, not when the in-flight request returns"); + assert.deepEqual((await usageRows()).map((row) => [row.reason, row.limit_reason, row.total, row.calls]), [["wake_timeout", "timeout", 2_775, 2]]); + // One measured row, but both admitted requests count: the killed one was sent upstream. + assert.deepEqual((await requestRows()).map((row) => [row.request, row.requests]), [[0, 2]]); + }, undefined, (call) => call === 2 ? 1_500 : 15); }); test("a wake may only lower a declared limit: raising one is refused before any turn record or worker", async () => { diff --git a/src/runtime/turnRequestLedger.ts b/src/runtime/turnRequestLedger.ts index a13bfdf..ef8dfca 100644 --- a/src/runtime/turnRequestLedger.ts +++ b/src/runtime/turnRequestLedger.ts @@ -106,7 +106,12 @@ const requestClockFields = (request: Readonly<{ startedAt?: string; endedAt?: st /** One Grok broker model request: usage from the worker stream, timing from the proxy. */ export type GrokTurnRequest = Readonly<{ index: number; input: number; cacheRead: number; cacheWrite: number; output: number; total: number; startedAt?: string; endedAt?: string }>; -export type GrokTurnRequestEntry = Readonly<{ agent: string; wake: string; turn: string; session?: string; model: GrokBrokerModel; requests: readonly GrokTurnRequest[]; at?: string }>; +/** + * `requestCount` is the turn's admitted request count when it exceeds the rows: + * a killed turn's in-flight request was sent upstream but never reported usage, + * so it has no row yet still counts in every row's `requests`. + */ +export type GrokTurnRequestEntry = Readonly<{ agent: string; wake: string; turn: string; session?: string; model: GrokBrokerModel; requests: readonly GrokTurnRequest[]; requestCount?: number; at?: string }>; /** * Grok rows share the Codex row's field meaning: `input` is the whole prompt @@ -128,7 +133,7 @@ export const renderGrokTurnRequestLines = (entry: GrokTurnRequestEntry): string ...(entry.session === undefined ? {} : { thread: bounded(entry.session) }), model: entry.model, request: request.index, - requests: entry.requests.length, + requests: Math.max(entry.requests.length, entry.requestCount ?? 0), input: request.input + request.cacheRead, cached_input: request.cacheRead, fresh_input: request.input, From cb1971f3318d67931bbfe5b804222f7b20290796 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:25:43 +0200 Subject: [PATCH 11/21] test: keep Grok engine declaration tests in their own file under the line limit --- src/runtime/engineDispatcher.test.ts | 13 +---- src/runtime/organizationRuntime.test.ts | 35 ----------- .../organizationRuntimeGrokEngine.test.ts | 58 +++++++++++++++++++ 3 files changed, 60 insertions(+), 46 deletions(-) create mode 100644 src/runtime/organizationRuntimeGrokEngine.test.ts diff --git a/src/runtime/engineDispatcher.test.ts b/src/runtime/engineDispatcher.test.ts index 488a658..0dae9a8 100644 --- a/src/runtime/engineDispatcher.test.ts +++ b/src/runtime/engineDispatcher.test.ts @@ -244,13 +244,9 @@ test("production Grok dispatcher routes every wake through the broker without ag process.env.NOOPOLIS_RUN_ID = "dispatcher-grok-realm-test"; const broker: EngineBrokerTurnClient = { async turn(agentId,wakeId,prompt,endpoint,signal,options) { turns += 1;assert.equal(agentId,config.id);assert.match(wakeId,/^(first|second)$/u);assert.match(prompt,/work/u);assert.match(endpoint,/^http:\/\/127\.0\.0\.1:\d+\/mcp$/u);assert.equal(signal?.aborted,false); - // The engine-neutral wake bound reaches the broker as a lowering limit. - assert.deepEqual(options,{limits:{maxTokens:123_456}});return "brokered"; } + assert.deepEqual(options,{limits:{maxTokens:123_456}});return "brokered"; } // the engine-neutral wake bound reaches the broker as a lowering limit }; - const priorCeiling = process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING; process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING = "123456"; - let handle: Awaited>; - try { handle = await startOrganizationRuntimeEngine(config, "DAIMON_UNUSED_CONTROL", undefined, undefined, broker); } - finally { if (priorCeiling === undefined) delete process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING; else process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING = priorCeiling; } + const priorCeiling = process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING; process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING = "123456"; const handle = await startOrganizationRuntimeEngine(config, "DAIMON_UNUSED_CONTROL", undefined, undefined, broker).finally(() => { if (priorCeiling === undefined) delete process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING; else process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING = priorCeiling; }); assert.equal((await handle.wake({ id: "first", kind: "manual", text: "work" })).text, "brokered"); assert.equal((await handle.wake({ id: "second", kind: "manual", text: "work" })).text, "brokered"); assert.equal(turns, 2); @@ -401,8 +397,3 @@ async function seedAuth(root: string, kind: "codex" | "grok" | "agy"): Promise { - const config = { ...rootConfig("/tmp/daimon-unused-direct-grok", "grok"), engine: { kind: "grok", model: "grok-4.6", reasoningEffort: "low" } } as OrganizationRuntimeAgentConfig; - await assert.rejects(startOrganizationRuntimeEngine(config, "DAIMON_UNUSED_CONTROL"), /requires the engine broker/u); -}); diff --git a/src/runtime/organizationRuntime.test.ts b/src/runtime/organizationRuntime.test.ts index 3636225..60f98f8 100644 --- a/src/runtime/organizationRuntime.test.ts +++ b/src/runtime/organizationRuntime.test.ts @@ -271,20 +271,6 @@ test("the engine JSON Schema and the parser agree on model/reasoningEffort", () } }); -test("the engine JSON Schema and the parser agree on grok and agy model declarations", async () => { - const { Ajv2020 } = await import("ajv/dist/2020.js") as unknown as { Ajv2020: new (options: Record) => { compile(schema: unknown): (value: unknown) => boolean } }; - const validate = new Ajv2020({ strict: false }).compile(ORGANIZATION_RUNTIME_CONFIG_SCHEMA); - for (const engine of [ - { kind: "grok" }, { kind: "grok", model: "grok-4.6", reasoningEffort: "low" }, { kind: "grok", model: "grok-4.6" }, - { kind: "grok", model: "grok-4.6-build", reasoningEffort: "low" }, { kind: "grok", model: "grok-4.5", reasoningEffort: "xhigh" }, - { kind: "agy" }, { kind: "agy", model: "x" }, { kind: "codex", model: "gpt-5-codex", reasoningEffort: "xhigh" } - ]) { - const config = valid(); - config.agents[0]!.engine = engine as never; - assert.equal(validate(config), validateOrganizationRuntimeConfig(config), JSON.stringify(engine)); - } -}); - test("accepts only the narrow optional Codex workspace policy", () => { const config = valid(); config.agents[0]!.engine = { kind: "codex", codexSandbox: { mode: "workspace-write", networkAccess: false, webSearch: "disabled" } } as never; @@ -315,27 +301,6 @@ test("rejects model and reasoningEffort on agy, and codexSandbox on every non-co } }); -test("grok declares model and reasoning effort together from the closed broker lists, never half-inherited", () => { - const declared = valid(); - declared.agents[0]!.engine = { kind: "grok", model: "grok-4.6", reasoningEffort: "low" } as never; - assert.deepEqual(parseOrganizationRuntimeConfig(declared).agents[0]!.engine, { kind: "grok", model: "grok-4.6", reasoningEffort: "low" }); - const bare = valid(); - bare.agents[0]!.engine = { kind: "grok" } as never; - assert.deepEqual(parseOrganizationRuntimeConfig(bare).agents[0]!.engine, { kind: "grok" }); - for (const engine of [ - { kind: "grok", model: "grok-4.6" }, - { kind: "grok", reasoningEffort: "low" }, - { kind: "grok", model: "grok-4.6-build", reasoningEffort: "low" }, - { kind: "grok", model: "gpt-5-codex", reasoningEffort: "low" }, - { kind: "grok", model: "grok-4.6", reasoningEffort: "xhigh" }, - { kind: "grok", model: "grok-4.6", reasoningEffort: "low", provider: "xai" } - ]) { - const invalid = valid(); - invalid.agents[0]!.engine = engine as never; - assert.throws(() => parseOrganizationRuntimeConfig(invalid), TypeError, JSON.stringify(engine)); - } -}); - const withMemory = (agent: Record, memory: unknown): Record => ({ ...agent, memory }); test("parses a declared memory bank and round-trips its fields", () => { diff --git a/src/runtime/organizationRuntimeGrokEngine.test.ts b/src/runtime/organizationRuntimeGrokEngine.test.ts new file mode 100644 index 0000000..93ea7ab --- /dev/null +++ b/src/runtime/organizationRuntimeGrokEngine.test.ts @@ -0,0 +1,58 @@ +import { strict as assert } from "node:assert"; +import test from "node:test"; + +import { startOrganizationRuntimeEngine } from "./engineDispatcher.js"; +import { + ORGANIZATION_RUNTIME_CONFIG_SCHEMA, + ORGANIZATION_RUNTIME_VERSION, + validateOrganizationRuntimeConfig, + parseOrganizationRuntimeConfig, + type OrganizationRuntimeAgentConfig, + type OrganizationRuntimeEngineIntent +} from "./organizationRuntime.js"; + +const valid = () => ({ + version: ORGANIZATION_RUNTIME_VERSION, + host: { bindHost: "127.0.0.1", port: 4318, controlTokenEnv: "DAIMON_CONTROL_TOKEN" }, + agents: [{ id: "editor", name: "Editor", instructions: "Write a concise report.", workspacePath: "/runtime/workspaces/editor", runtimeHomePath: "/runtime/homes/editor", engine: { kind: "codex" } as OrganizationRuntimeEngineIntent }] +}); + +test("grok declares model and reasoning effort together from the closed broker lists, never half-inherited", () => { + const declared = valid(); + declared.agents[0]!.engine = { kind: "grok", model: "grok-4.6", reasoningEffort: "low" } as never; + assert.deepEqual(parseOrganizationRuntimeConfig(declared).agents[0]!.engine, { kind: "grok", model: "grok-4.6", reasoningEffort: "low" }); + const bare = valid(); + bare.agents[0]!.engine = { kind: "grok" } as never; + assert.deepEqual(parseOrganizationRuntimeConfig(bare).agents[0]!.engine, { kind: "grok" }); + for (const engine of [ + { kind: "grok", model: "grok-4.6" }, + { kind: "grok", reasoningEffort: "low" }, + { kind: "grok", model: "grok-4.6-build", reasoningEffort: "low" }, + { kind: "grok", model: "gpt-5-codex", reasoningEffort: "low" }, + { kind: "grok", model: "grok-4.6", reasoningEffort: "xhigh" }, + { kind: "grok", model: "grok-4.6", reasoningEffort: "low", provider: "xai" } + ]) { + const invalid = valid(); + invalid.agents[0]!.engine = engine as never; + assert.throws(() => parseOrganizationRuntimeConfig(invalid), TypeError, JSON.stringify(engine)); + } +}); + +test("the engine JSON Schema and the parser agree on grok and agy model declarations", async () => { + const { Ajv2020 } = await import("ajv/dist/2020.js") as unknown as { Ajv2020: new (options: Record) => { compile(schema: unknown): (value: unknown) => boolean } }; + const validate = new Ajv2020({ strict: false }).compile(ORGANIZATION_RUNTIME_CONFIG_SCHEMA); + for (const engine of [ + { kind: "grok" }, { kind: "grok", model: "grok-4.6", reasoningEffort: "low" }, { kind: "grok", model: "grok-4.6" }, + { kind: "grok", model: "grok-4.6-build", reasoningEffort: "low" }, { kind: "grok", model: "grok-4.5", reasoningEffort: "xhigh" }, + { kind: "agy" }, { kind: "agy", model: "x" }, { kind: "codex", model: "gpt-5-codex", reasoningEffort: "xhigh" } + ]) { + const config = valid(); + config.agents[0]!.engine = engine as never; + assert.equal(validate(config), validateOrganizationRuntimeConfig(config), JSON.stringify(engine)); + } +}); + +test("a declared Grok model is refused on the direct path that cannot enforce it", async () => { + const config = { ...valid().agents[0]!, engine: { kind: "grok", model: "grok-4.6", reasoningEffort: "low" } } as OrganizationRuntimeAgentConfig; + await assert.rejects(startOrganizationRuntimeEngine(config, "DAIMON_UNUSED_CONTROL"), /requires the engine broker/u); +}); From a77a6d4be0b26f1ed1f8075af982656281b0bbed Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:36:52 +0200 Subject: [PATCH 12/21] fix: allow one in-flight upstream request per Grok turn and abort it when a limit trips --- src/runtime/grokBrokerProxy.ts | 9 ++-- src/runtime/grokBrokerTurnMeter.test.ts | 54 +++++++++++++++++++++++ src/runtime/grokBrokerTurnMeter.ts | 25 +++++++++-- src/runtime/grokEngineBrokerTurn.ts | 2 +- src/runtime/grokEngineBrokerUsage.test.ts | 12 ++--- 5 files changed, 89 insertions(+), 13 deletions(-) diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index 4f53103..57e9091 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -10,7 +10,7 @@ import { parseGrokUpstreamUsage, type GrokBrokerTurnMeter } from "./grokBrokerTu export type GrokBrokerProxyTurn = Readonly<{ policy: GrokBrokerModelPolicy; meter: GrokBrokerTurnMeter }>; export type GrokBrokerCredentialAuthority = Readonly<{ accessToken(forceRefresh: boolean): Promise; refreshAfterRejection?(rejectedTokenDigest:string):Promise; markRejected(rejectedTokenDigest?:string): Promise }>; -export type GrokBrokerUpstream = (request: ReturnType) => Promise>; body: Uint8Array }>>; +export type GrokBrokerUpstream = (request: ReturnType, signal?: AbortSignal) => Promise>; body: Uint8Array }>>; /** * `policy` is the fallback declared model/effort (closed list); a registered @@ -36,12 +36,13 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori // (a refused session-title body never counts) and before any upstream call. const admission=turn.meter.admit(); if("refused" in admission){response.writeHead(429,{"content-type":"application/json","cache-control":"no-store"});response.end(JSON.stringify({error:"turn limit reached",limit:admission.refused}));return;} + if("busy" in admission){response.writeHead(429,{"content-type":"application/json","cache-control":"no-store"});response.end('{"error":"turn request in flight"}');return;} settle=(usage)=>{turn.meter.settle(admission.index,usage);settle=undefined;}; - let result = await upstream(prepared); - if (result.status === 401) { token = authority.refreshAfterRejection?await authority.refreshAfterRejection(rejectedDigest):await authority.accessToken(true);const refreshedDigest=createHash("sha256").update(token).digest("hex"); prepared = { ...prepared, headers: { ...prepared.headers, authorization: `Bearer ${token}` } }; token = ""; result = await upstream(prepared);if(result.status===401)await authority.markRejected(refreshedDigest); } + let result = await upstream(prepared,admission.signal); + if (result.status === 401) { token = authority.refreshAfterRejection?await authority.refreshAfterRejection(rejectedDigest):await authority.accessToken(true);const refreshedDigest=createHash("sha256").update(token).digest("hex"); prepared = { ...prepared, headers: { ...prepared.headers, authorization: `Bearer ${token}` } }; token = ""; result = await upstream(prepared,admission.signal);if(result.status===401)await authority.markRejected(refreshedDigest); } settle?.(parseGrokUpstreamUsage(result.body,result.headers["content-type"])); response.writeHead(result.status, { "content-type": result.headers["content-type"] ?? "application/json", "cache-control": "no-store" }); response.end(result.body); } catch { settle?.(undefined); response.writeHead(503, { "content-type": "application/json", "cache-control": "no-store" }); response.end('{"error":"broker unavailable"}'); } } async function readBody(request: IncomingMessage): Promise { const chunks: Buffer[] = []; let bytes = 0; for await (const chunk of request) { const value = Buffer.from(chunk); bytes += value.length; if (bytes > 2 * 1024 * 1024) throw new Error("too large"); chunks.push(value); } return Buffer.concat(chunks); } -const defaultUpstream: GrokBrokerUpstream = async (request) => { const result = await fetch(request.url, { method: "POST", headers: request.headers, body: Buffer.from(request.body) }); return { status: result.status, headers: { "content-type": result.headers.get("content-type") ?? "application/json" }, body: new Uint8Array(await result.arrayBuffer()) }; }; +const defaultUpstream: GrokBrokerUpstream = async (request, signal) => { const result = await fetch(request.url, { method: "POST", headers: request.headers, body: Buffer.from(request.body), ...(signal === undefined ? {} : { signal }) }); return { status: result.status, headers: { "content-type": result.headers.get("content-type") ?? "application/json" }, body: new Uint8Array(await result.arrayBuffer()) }; }; diff --git a/src/runtime/grokBrokerTurnMeter.test.ts b/src/runtime/grokBrokerTurnMeter.test.ts index 04c0a6a..cd55082 100644 --- a/src/runtime/grokBrokerTurnMeter.test.ts +++ b/src/runtime/grokBrokerTurnMeter.test.ts @@ -101,3 +101,57 @@ test("upstream usage parsing takes the last usage block and never zero-fills", ( assert.equal(parseGrokUpstreamUsage(sse({ prompt_tokens: 4, completion_tokens: 1, prompt_tokens_details: { cached_tokens: 9 } }), "text/event-stream"), undefined); assert.equal(parseGrokUpstreamUsage(Buffer.from("not json"), "application/json"), undefined); }); + +test("at most one upstream request is in flight per turn: an overlapping request is refused, uncounted", async () => { + // Mutation guard: without the in-flight gate both overlapping requests pass on + // the same pre-settle token total and the one-request overshoot bound is gone. + const meter = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 100, timeoutMs: 60_000 }); + const first = meter.admit(); + assert.ok("index" in first); + assert.deepEqual(meter.admit(), { busy: true }); + assert.equal(meter.snapshot().requests, 1); + meter.settle(first.index, { input: 60, cacheRead: 0, cacheWrite: 0, output: 60, total: 120 }); + assert.deepEqual(meter.admit(), { refused: "tokens" }, "once settled, the next request sees the reported total"); + + let release!: () => void; let calls = 0; + const gate = new Promise((resolve) => { release = resolve; }); + const live = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 1_000_000, timeoutMs: 60_000 }); + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; await gate; return { status: 200, headers: { "content-type": "text/event-stream" }, body: sse({ prompt_tokens: 1, completion_tokens: 1 }) }; }, undefined, 0); + try { + const token = proxy.capabilities.issue("agent", "turn"); + proxy.registerIsolationGuard("turn", async () => undefined); + proxy.registerTurn("turn", { policy: { model: "grok-4.6", reasoningEffort: "low" }, meter: live }); + const pending = post(proxy.port, token); + while (calls === 0) await new Promise((resolve) => setTimeout(resolve, 5)); + const overlapping = await post(proxy.port, token); + assert.deepEqual([overlapping.status, JSON.parse(overlapping.text)], [429, { error: "turn request in flight" }]); + assert.equal(calls, 1); + release(); + assert.equal((await pending).status, 200); + assert.equal((await post(proxy.port, token)).status, 200); + assert.deepEqual([calls, live.snapshot().requests, live.snapshot().limitReason], [2, 2, "none"]); + } finally { release(); await proxy.close(); } +}); + +test("tripping a limit aborts the in-flight upstream call instead of letting it run", async () => { + let observed: AbortSignal | undefined; + const meter = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 1_000_000, timeoutMs: 60_000 }); + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async (_request, signal) => { + observed = signal; + await new Promise((_resolve, reject) => signal?.addEventListener("abort", () => reject(new Error("aborted")), { once: true })); + throw new Error("unreachable"); + }, undefined, 0); + try { + const token = proxy.capabilities.issue("agent", "turn"); + proxy.registerIsolationGuard("turn", async () => undefined); + proxy.registerTurn("turn", { policy: { model: "grok-4.6", reasoningEffort: "low" }, meter }); + const pending = post(proxy.port, token); + while (observed === undefined) await new Promise((resolve) => setTimeout(resolve, 5)); + assert.equal(observed.aborted, false); + // Mutation guard: a trip that leaves the upstream signal alone hangs this request. + meter.trip("timeout"); + assert.equal(observed.aborted, true); + assert.equal((await pending).status, 503); + assert.deepEqual(meter.admit(), { refused: "timeout" }); + } finally { await proxy.close(); } +}); diff --git a/src/runtime/grokBrokerTurnMeter.ts b/src/runtime/grokBrokerTurnMeter.ts index 203a9be..5136734 100644 --- a/src/runtime/grokBrokerTurnMeter.ts +++ b/src/runtime/grokBrokerTurnMeter.ts @@ -21,32 +21,47 @@ export type GrokBrokerTurnMeterSnapshot = Readonly<{ requests: number; tokens: n * * The first limit that fires is sticky: every later request is refused with * the same reason, and `onLimit` runs once. + * + * The token bound is only a bound if no request can be admitted on a total + * that an in-flight request has not yet reported into. So a turn has at most + * ONE upstream request in flight: a second request arriving before the first + * settled is refused (`busy`, HTTP 429) without being counted or tripping a + * limit. Grok's headless loop is sequential — every live capture (P1 + * live-round1, P2 live) shows each request ending before the next starts — so + * this refuses only a worker that is not behaving like Grok. Tripping a limit + * (including the broker's timer) aborts that in-flight upstream call through + * its own `AbortSignal` rather than letting it run to completion. */ export class GrokBrokerTurnMeter { private readonly startedAt: number; private readonly timings: { startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage }[] = []; private tokens = 0; private reason: EngineBrokerLimitReason = "none"; + private inFlight: { index: number; controller: AbortController } | undefined; constructor(readonly limits: EngineBrokerTurnLimits, private readonly onLimit: (reason: Exclude) => void = () => undefined, private readonly now: () => number = Date.now) { this.startedAt = now(); } - /** Returns the request index when admitted, or the limit that refused it. */ - admit(): Readonly<{ index: number } | { refused: Exclude }> { + /** Returns the request index and its upstream abort signal when admitted, the limit that refused it, or `busy` while another request is in flight. */ + admit(): Readonly<{ index: number; signal: AbortSignal } | { refused: Exclude } | { busy: true }> { if (this.reason === "none") { if (this.now() - this.startedAt >= this.limits.timeoutMs) this.trip("timeout"); else if (this.timings.length >= this.limits.maxRequests) this.trip("requests"); else if (this.tokens >= this.limits.maxTokens) this.trip("tokens"); } if (this.reason !== "none") return { refused: this.reason }; + if (this.inFlight !== undefined) return { busy: true }; this.timings.push({ startedAt: new Date(this.now()).toISOString() }); - return { index: this.timings.length - 1 }; + const controller = new AbortController(); + this.inFlight = { index: this.timings.length - 1, controller }; + return { index: this.inFlight.index, signal: controller.signal }; } /** Records one admitted request's end and its upstream-reported usage, when the body carried any. */ settle(index: number, usage: EngineBrokerTurnUsage | undefined): void { const timing = this.timings[index]; if (timing === undefined || timing.endedAt !== undefined) return; + if (this.inFlight?.index === index) this.inFlight = undefined; timing.endedAt = new Date(this.now()).toISOString(); if (usage === undefined) return; timing.usage = usage; @@ -57,9 +72,13 @@ export class GrokBrokerTurnMeter { trip(reason: Exclude): void { if (this.reason !== "none") return; this.reason = reason; + this.abortInFlight(); this.onLimit(reason); } + /** Aborts the in-flight upstream call, if any (limit trip, or the broker ending the turn). */ + abortInFlight(): void { this.inFlight?.controller.abort(); } + snapshot(): GrokBrokerTurnMeterSnapshot { const measured = this.timings.flatMap((timing) => timing.usage === undefined ? [] : [timing.usage]); return { requests: this.timings.length, tokens: this.tokens, limitReason: this.reason, usage: sumEngineBrokerTurnUsage(measured), timings: this.timings.map((timing) => Object.freeze({ ...timing })) }; diff --git a/src/runtime/grokEngineBrokerTurn.ts b/src/runtime/grokEngineBrokerTurn.ts index 105ff02..22656a8 100644 --- a/src/runtime/grokEngineBrokerTurn.ts +++ b/src/runtime/grokEngineBrokerTurn.ts @@ -89,7 +89,7 @@ export async function runGrokEngineBrokerTurn(deps: GrokEngineBrokerTurnDependen await finishBrokerTurnWithUsage(deps.turns, request, failed, metering, { notionalUsd: 0, complete: false, reason, requests: requestRows(stream, snapshot), ...(stream?.sessionId === undefined ? {} : { session: stream.sessionId }) }); throw new EngineBrokerTurnFailure(code, diagnostic, accounting); } finally { - clearTimeout(timer); signal?.removeEventListener("abort", onAbort); + clearTimeout(timer); signal?.removeEventListener("abort", onAbort); meter.abortInFlight(); deps.proxy.revokeTurn(turnId); deps.proxy.revokeIsolationGuard(turnId); deps.proxy.capabilities.revoke(turnId); deps.mcp.revoke(turnId); } } diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index 408850b..6cca960 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -46,10 +46,10 @@ const untilAborted = (signal: AbortSignal): Promise => new Promise((_reso * talks to the proxy exactly as the native worker does (capability bearer, * pinned client version, lean body). Only the launcher and attestation are fakes. */ -const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId: string, worker: Worker, overrides?: Parameters[6], limits?: EngineBrokerServiceRegistration["limits"], turnStore?: string) => ReturnType; usageRows: () => Promise[]>; requestRows: () => Promise[]>; upstreamCalls: () => number }>) => Promise, usageLedgerPath?: string, upstreamDelayMs: (call: number) => number = () => 15): Promise => { +const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId: string, worker: Worker, overrides?: Parameters[6], limits?: EngineBrokerServiceRegistration["limits"], turnStore?: string) => ReturnType; usageRows: () => Promise[]>; requestRows: () => Promise[]>; upstreamCalls: () => number; upstreamAborts: () => number }>) => Promise, usageLedgerPath?: string, upstreamDelayMs: (call: number) => number = () => 15): Promise => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-usage-")); - let calls = 0; - const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; await new Promise((resolve) => setTimeout(resolve, upstreamDelayMs(calls))); return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(`data: ${JSON.stringify({ choices: [], usage: upstreamUsage })}\n\ndata: [DONE]\n\n`) }; }, undefined, 0); + let calls = 0, aborted = 0; + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async (_request, signal) => { calls++; await new Promise((resolve, reject) => { const timer = setTimeout(resolve, upstreamDelayMs(calls)); signal?.addEventListener("abort", () => { clearTimeout(timer); aborted++; reject(new Error("aborted")); }, { once: true }); }); return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(`data: ${JSON.stringify({ choices: [], usage: upstreamUsage })}\n\ndata: [DONE]\n\n`) }; }, undefined, 0); const ledger = usageLedgerPath ?? path.join(root, "usage.jsonl"); const rows = async (file: string) => (await readFile(file, "utf8").catch(() => "")).split("\n").filter((line) => line.length > 0).map((line) => JSON.parse(line) as Record); try { @@ -67,7 +67,8 @@ const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId }, usageRows: () => rows(ledger), requestRows: () => rows(path.join(path.dirname(ledger), "requests.jsonl")), - upstreamCalls: () => calls + upstreamCalls: () => calls, + upstreamAborts: () => aborted }); } finally { await proxy.close(); await rm(root, { recursive: true, force: true }); } }; @@ -128,12 +129,13 @@ test("the token ceiling stops a turn one request past the ceiling at most", asyn }); test("the wall-clock limit aborts a worker that is mid-request", async () => { - await withBroker(async ({ turn, usageRows, requestRows }) => { + await withBroker(async ({ turn, usageRows, requestRows, upstreamAborts }) => { const started = Date.now(); // Request 2 is still upstream (1.5 s) when the 1 s wall clock fires. const worker: Worker = async (send, signal) => { assert.equal(await send(), 200); void send().catch(() => undefined); return untilAborted(signal); }; await assert.rejects(turn("wake-4", worker, { timeoutMs: 1_000 }), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.code === "limit_exceeded" && error.accounting?.limitReason === "timeout" && error.accounting.requests === 2); assert.ok(Date.now() - started < 1_400, "the turn ends at the deadline, not when the in-flight request returns"); + assert.equal(upstreamAborts(), 1, "the stuck upstream call is aborted, not left running"); assert.deepEqual((await usageRows()).map((row) => [row.reason, row.limit_reason, row.total, row.calls]), [["wake_timeout", "timeout", 2_775, 2]]); // One measured row, but both admitted requests count: the killed one was sent upstream. assert.deepEqual((await requestRows()).map((row) => [row.request, row.requests]), [[0, 2]]); From 8932725781f7ba2b20592523357f1f5ceef74d83 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:39:24 +0200 Subject: [PATCH 13/21] fix: bound per-request Grok usage and charge an estimate when a response reports none --- src/contracts/runtimeContractManifest.ts | 8 ++++- src/pi/grokStreamUsage.test.ts | 7 +++++ src/pi/grokStreamUsage.ts | 7 ++++- src/runtime/grokBrokerProxy.ts | 2 +- src/runtime/grokBrokerTurnMeter.test.ts | 29 +++++++++++++++-- src/runtime/grokBrokerTurnMeter.ts | 38 +++++++++++++++++------ src/runtime/grokEngineBrokerMetering.ts | 4 +-- src/runtime/grokEngineBrokerTurn.ts | 8 ++--- src/runtime/grokEngineBrokerUsage.test.ts | 6 ++-- src/runtime/turnRequestLedger.ts | 8 ++++- src/runtime/turnUsageLedger.ts | 7 +++-- 11 files changed, 96 insertions(+), 28 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index ceef199..8aebfb1 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -78,7 +78,13 @@ export const GROK_ENGINE_BROKER = { bounds: { maxRequests: [1, GROK_WORKER_MAX_TURNS], maxTokens: [1, 10_000_000], timeoutMs: [1_000, 3_600_000] }, limitReasons: ["tokens", "requests", "timeout", "none"], wakeMayOnlyLower: true, - tokenCeilingOvershoot: "at-most-one-request" + tokenCeilingOvershoot: "at-most-one-request", + maxInFlightRequests: 1, + // A per-request usage block above this is implausible (beyond the model + // context window) and treated as invalid rather than added to any total. + requestUsageMaxTokens: 500_000, + // A request whose response carries no valid usage is charged this estimate. + missingUsageEstimate: { inputBytesPerToken: 2, outputTokens: 4_096 } }, wakeLimitEnvironment: { timeoutMs: "DAIMON_ENGINE_WAKE_TIMEOUT_MS", maxTokens: "DAIMON_ENGINE_WAKE_TOKEN_CEILING" }, projectionVersion: "noopolis.daimon.grok-broker-projection.v1", diff --git a/src/pi/grokStreamUsage.test.ts b/src/pi/grokStreamUsage.test.ts index 37942fb..8bf0f32 100644 --- a/src/pi/grokStreamUsage.test.ts +++ b/src/pi/grokStreamUsage.test.ts @@ -38,3 +38,10 @@ test("frames repeating one message id are one request, and a torn line is skippe test("the captured fixture carries no capturing machine's environment", async () => { assert.doesNotMatch(await fixture(), /\/Users\/|\/private\/|scratchpad|\/home\//u); }); + +test("a per-request stream block beyond the context-window bound is invalid, not counted", async () => { + const lines = (await fixture()).split("\n"); + const index = lines.findIndex((line) => line.includes('"msg_1"')); + lines[index] = lines[index]!.replace('"input_tokens":109', '"input_tokens":900000000'); + assert.deepEqual(decodeGrokStreamUsage(lines.join("\n")).requests, []); +}); diff --git a/src/pi/grokStreamUsage.ts b/src/pi/grokStreamUsage.ts index 79b4aef..bcb39e1 100644 --- a/src/pi/grokStreamUsage.ts +++ b/src/pi/grokStreamUsage.ts @@ -1,3 +1,5 @@ +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; + /** * Per-request token accounting read off a Grok `streaming-messages-json` * stream. @@ -27,7 +29,10 @@ const decodeUsage = (usage: unknown): Omit | undefine if (!isRecord(usage)) return undefined; const input = tokenCount(usage.input_tokens), output = tokenCount(usage.output_tokens), cacheRead = tokenCount(usage.cache_read_input_tokens), cacheWrite = tokenCount(usage.cache_creation_input_tokens); if (input === undefined || output === undefined || cacheRead === undefined || cacheWrite === undefined) return undefined; - return { input, cacheRead, cacheWrite, output, total: input + cacheRead + cacheWrite + output }; + const total = input + cacheRead + cacheWrite + output; + // Beyond the model context window one request cannot have spent it: invalid, like a malformed block. + if (total > GROK_ENGINE_BROKER.turnLimits.requestUsageMaxTokens) return undefined; + return { input, cacheRead, cacheWrite, output, total }; }; export const decodeGrokStreamUsage = (output: string): GrokStreamUsage => { diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index 57e9091..187bf7a 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -37,7 +37,7 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori const admission=turn.meter.admit(); if("refused" in admission){response.writeHead(429,{"content-type":"application/json","cache-control":"no-store"});response.end(JSON.stringify({error:"turn limit reached",limit:admission.refused}));return;} if("busy" in admission){response.writeHead(429,{"content-type":"application/json","cache-control":"no-store"});response.end('{"error":"turn request in flight"}');return;} - settle=(usage)=>{turn.meter.settle(admission.index,usage);settle=undefined;}; + settle=(usage)=>{turn.meter.settle(admission.index,usage,body.byteLength);settle=undefined;}; let result = await upstream(prepared,admission.signal); if (result.status === 401) { token = authority.refreshAfterRejection?await authority.refreshAfterRejection(rejectedDigest):await authority.accessToken(true);const refreshedDigest=createHash("sha256").update(token).digest("hex"); prepared = { ...prepared, headers: { ...prepared.headers, authorization: `Bearer ${token}` } }; token = ""; result = await upstream(prepared,admission.signal);if(result.status===401)await authority.markRejected(refreshedDigest); } settle?.(parseGrokUpstreamUsage(result.body,result.headers["content-type"])); diff --git a/src/runtime/grokBrokerTurnMeter.test.ts b/src/runtime/grokBrokerTurnMeter.test.ts index cd55082..f64911c 100644 --- a/src/runtime/grokBrokerTurnMeter.test.ts +++ b/src/runtime/grokBrokerTurnMeter.test.ts @@ -78,8 +78,11 @@ test("a request after the elapsed deadline is refused, and every admitted reques }); const snapshot = meter.snapshot(); assert.equal(snapshot.limitReason, "timeout"); - assert.equal(snapshot.usage, null, "a body without usage contributes no invented zero"); - assert.deepEqual(snapshot.timings, [{ startedAt: new Date(1_000_000).toISOString(), endedAt: new Date(1_000_000).toISOString() }]); + // A body without usage is never a zero: it is charged the conservative estimate (402-byte body). + const estimate = { input: 201, cacheRead: 0, cacheWrite: 0, output: 4_096, total: 4_297 }; + assert.deepEqual(snapshot.usage, estimate); + assert.equal(snapshot.estimatedRequests, 1); + assert.deepEqual(snapshot.timings, [{ startedAt: new Date(1_000_000).toISOString(), endedAt: new Date(1_000_000).toISOString(), usage: estimate, estimated: true }]); }); test("a turn without a registered meter is never forwarded", async () => { @@ -110,7 +113,7 @@ test("at most one upstream request is in flight per turn: an overlapping request assert.ok("index" in first); assert.deepEqual(meter.admit(), { busy: true }); assert.equal(meter.snapshot().requests, 1); - meter.settle(first.index, { input: 60, cacheRead: 0, cacheWrite: 0, output: 60, total: 120 }); + meter.settle(first.index, { input: 60, cacheRead: 0, cacheWrite: 0, output: 60, total: 120 }, 0); assert.deepEqual(meter.admit(), { refused: "tokens" }, "once settled, the next request sees the reported total"); let release!: () => void; let calls = 0; @@ -155,3 +158,23 @@ test("tripping a limit aborts the in-flight upstream call instead of letting it assert.deepEqual(meter.admit(), { refused: "timeout" }); } finally { await proxy.close(); } }); + +test("an implausible per-request usage block is never added, and missing usage still trips the token ceiling", async () => { + // Mutation guard: without the plausibility bound this adds 400 billion tokens to the total. + assert.equal(parseGrokUpstreamUsage(sse({ prompt_tokens: 400_000_000_000, completion_tokens: 1 }), "text/event-stream"), undefined); + assert.equal(parseGrokUpstreamUsage(sse({ prompt_tokens: 499_990, completion_tokens: 11 }), "text/event-stream"), undefined); + assert.equal(parseGrokUpstreamUsage(sse({ prompt_tokens: 499_990, completion_tokens: 10 }), "text/event-stream")?.total, 500_000); + const huge = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 1_000_000, timeoutMs: 60_000 }); + await withProxy({ prompt_tokens: 400_000_000_000, completion_tokens: 1 }, huge, async (send) => { assert.equal((await send()).status, 200); }); + assert.deepEqual([huge.snapshot().tokens, huge.snapshot().estimatedRequests], [4_297, 1]); + + // Mutation guard: settling a usage-less response as zero lets this turn run to maxRequests. + const blind = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 10_000, timeoutMs: 60_000 }); + await withProxy(undefined, blind, async (send, calls) => { + const statuses = []; + for (let index = 0; index < 5; index++) statuses.push((await send()).status); + assert.deepEqual(statuses, [200, 200, 200, 429, 429]); + assert.equal(calls(), 3); + }); + assert.deepEqual([blind.snapshot().limitReason, blind.snapshot().tokens, blind.snapshot().estimatedRequests], ["tokens", 12_891, 3]); +}); diff --git a/src/runtime/grokBrokerTurnMeter.ts b/src/runtime/grokBrokerTurnMeter.ts index 5136734..d5b03f4 100644 --- a/src/runtime/grokBrokerTurnMeter.ts +++ b/src/runtime/grokBrokerTurnMeter.ts @@ -1,7 +1,9 @@ +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; import { sumEngineBrokerTurnUsage, type EngineBrokerLimitReason, type EngineBrokerTurnLimits, type EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; -export type GrokBrokerRequestTiming = Readonly<{ startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage }>; -export type GrokBrokerTurnMeterSnapshot = Readonly<{ requests: number; tokens: number; limitReason: EngineBrokerLimitReason; usage: EngineBrokerTurnUsage | null; timings: readonly GrokBrokerRequestTiming[] }>; +/** `estimated` marks a request whose response carried no valid usage and was charged {@link estimateGrokRequestUsage}. */ +export type GrokBrokerRequestTiming = Readonly<{ startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage; estimated?: true }>; +export type GrokBrokerTurnMeterSnapshot = Readonly<{ requests: number; tokens: number; limitReason: EngineBrokerLimitReason; usage: EngineBrokerTurnUsage | null; estimatedRequests: number; timings: readonly GrokBrokerRequestTiming[] }>; /** * The proxy's per-turn spend gate. @@ -34,7 +36,7 @@ export type GrokBrokerTurnMeterSnapshot = Readonly<{ requests: number; tokens: n */ export class GrokBrokerTurnMeter { private readonly startedAt: number; - private readonly timings: { startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage }[] = []; + private readonly timings: { startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage; estimated?: true }[] = []; private tokens = 0; private reason: EngineBrokerLimitReason = "none"; private inFlight: { index: number; controller: AbortController } | undefined; @@ -57,15 +59,20 @@ export class GrokBrokerTurnMeter { return { index: this.inFlight.index, signal: controller.signal }; } - /** Records one admitted request's end and its upstream-reported usage, when the body carried any. */ - settle(index: number, usage: EngineBrokerTurnUsage | undefined): void { + /** + * Records one admitted request's end and its usage. A response without valid + * usage (absent, malformed, implausible, or a failed/aborted call) is charged + * a conservative estimate from the request body size, so a missing `usage` + * can never silently disable the token ceiling. + */ + settle(index: number, usage: EngineBrokerTurnUsage | undefined, requestBytes: number): void { const timing = this.timings[index]; if (timing === undefined || timing.endedAt !== undefined) return; if (this.inFlight?.index === index) this.inFlight = undefined; timing.endedAt = new Date(this.now()).toISOString(); - if (usage === undefined) return; - timing.usage = usage; - this.tokens += usage.total; + if (usage === undefined) { timing.usage = estimateGrokRequestUsage(requestBytes); timing.estimated = true; } + else timing.usage = usage; + this.tokens += timing.usage.total; } /** Trips a limit from outside the request path (the broker's wall-clock timer). */ @@ -81,10 +88,18 @@ export class GrokBrokerTurnMeter { snapshot(): GrokBrokerTurnMeterSnapshot { const measured = this.timings.flatMap((timing) => timing.usage === undefined ? [] : [timing.usage]); - return { requests: this.timings.length, tokens: this.tokens, limitReason: this.reason, usage: sumEngineBrokerTurnUsage(measured), timings: this.timings.map((timing) => Object.freeze({ ...timing })) }; + return { requests: this.timings.length, tokens: this.tokens, limitReason: this.reason, usage: sumEngineBrokerTurnUsage(measured), estimatedRequests: this.timings.filter((timing) => timing.estimated === true).length, timings: this.timings.map((timing) => Object.freeze({ ...timing })) }; } } +const { requestUsageMaxTokens, missingUsageEstimate } = GROK_ENGINE_BROKER.turnLimits; + +/** The charge for a request without valid usage: `ceil(bodyBytes / 2)` input plus a fixed output allowance. */ +export const estimateGrokRequestUsage = (requestBytes: number): EngineBrokerTurnUsage => { + const input = Math.ceil(Math.max(0, requestBytes) / missingUsageEstimate.inputBytesPerToken), output = missingUsageEstimate.outputTokens; + return { input, cacheRead: 0, cacheWrite: 0, output, total: input + output }; +}; + type JsonRecord = Record; const isRecord = (value: unknown): value is JsonRecord => value !== null && typeof value === "object" && !Array.isArray(value); const count = (value: unknown): number | undefined => typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : undefined; @@ -98,7 +113,9 @@ const count = (value: unknown): number | undefined => typeof value === "number" * include cached tokens; they are split into disjoint buckets here, and any * reasoning tokens reported outside `completion_tokens` (visible as * `total_tokens` above prompt + completion) are folded into `output` so the - * total invariant holds. A malformed block is ignored, never zero-filled. + * total invariant holds. A malformed block, or one whose total exceeds + * `GROK_ENGINE_BROKER.turnLimits.requestUsageMaxTokens`, is invalid: never + * zero-filled and never added — the meter charges an estimate instead. */ export function parseGrokUpstreamUsage(body: Uint8Array, contentType: string | undefined): EngineBrokerTurnUsage | undefined { const text = Buffer.from(body.buffer, body.byteOffset, body.byteLength).toString("utf8"); @@ -133,5 +150,6 @@ function decodeOpenAiUsage(usage: JsonRecord): EngineBrokerTurnUsage | undefined const completionDetails = isRecord(usage.completion_tokens_details) ? usage.completion_tokens_details : {}; const reasoning = count(completionDetails.reasoning_tokens); const output = total - prompt; + if (total > requestUsageMaxTokens) return undefined; return { input: prompt - cached, cacheRead: cached, cacheWrite: 0, output, total, ...(reasoning === undefined || reasoning > output ? {} : { reasoning }) }; } diff --git a/src/runtime/grokEngineBrokerMetering.ts b/src/runtime/grokEngineBrokerMetering.ts index bd4e482..12c25a4 100644 --- a/src/runtime/grokEngineBrokerMetering.ts +++ b/src/runtime/grokEngineBrokerMetering.ts @@ -9,7 +9,7 @@ export type BrokerTurnMetering = Readonly<{ agentId: string; wakeId: string; }>; -export type BrokerTurnMeteringDetail = Readonly<{ notionalUsd: number; complete: boolean; reason?: TurnUsageFailureReason; requests: readonly GrokTurnRequest[]; session?: string }>; +export type BrokerTurnMeteringDetail = Readonly<{ notionalUsd: number; complete: boolean; reason?: TurnUsageFailureReason; requests: readonly GrokTurnRequest[]; session?: string; estimatedRequests: number }>; /** * Seal a terminal turn, then meter it. The broker is the single writer. @@ -37,7 +37,7 @@ export async function finishBrokerTurnWithUsage(turns: EngineBrokerTurnRegistry, agent: metering.agentId, wake: metering.wakeId, engine: "grok", usage: { input: usage.input, output: usage.output, cacheRead: usage.cacheRead, cacheWrite: usage.cacheWrite, total: usage.total, calls: terminal.requests, notionalUsd: detail.notionalUsd, complete: detail.complete }, outcome: terminal.kind === "completed" ? { status: "completed" } : { status: "failed", reason: detail.reason ?? "unknown" }, - turn: terminal.turnId, limitReason: terminal.limitReason, model: terminal.model + turn: terminal.turnId, limitReason: terminal.limitReason, model: terminal.model, estimatedRequests: detail.estimatedRequests }); await recordGrokTurnRequests(metering.requestLedgerPath, { agent: metering.agentId, wake: metering.wakeId, turn: terminal.turnId, model: terminal.model, requests: detail.requests, requestCount: terminal.requests, ...(detail.session === undefined ? {} : { session: detail.session }) }); } diff --git a/src/runtime/grokEngineBrokerTurn.ts b/src/runtime/grokEngineBrokerTurn.ts index 22656a8..650c94b 100644 --- a/src/runtime/grokEngineBrokerTurn.ts +++ b/src/runtime/grokEngineBrokerTurn.ts @@ -76,7 +76,7 @@ export async function runGrokEngineBrokerTurn(deps: GrokEngineBrokerTurnDependen const usage = decoded.usage === undefined ? streamOrMeterUsage(stream, snapshot) : usageOf(decoded.usage); const accounting = { outcome: "completed", usage, model: declared, requests: requestCount(stream, snapshot), limitReason: "none" } as const; const completed = { version: request.version, kind: "completed", requestId: request.requestId, turnId, text: decoded.text, workerPid: result.workerPid, workerUid: result.workerUid, workerStartTime: result.startTicks.toString(), ...accounting } as const; - await finishBrokerTurnWithUsage(deps.turns, request, completed, metering, { notionalUsd: decoded.usage?.notionalUsd ?? 0, complete: decoded.usage?.complete ?? false, requests: requestRows(stream, snapshot), ...(stream.sessionId === undefined ? {} : { session: stream.sessionId }) }); + await finishBrokerTurnWithUsage(deps.turns, request, completed, metering, { notionalUsd: decoded.usage?.notionalUsd ?? 0, complete: decoded.usage?.complete ?? false, estimatedRequests: snapshot.estimatedRequests, requests: requestRows(stream, snapshot), ...(stream.sessionId === undefined ? {} : { session: stream.sessionId }) }); return { text: completed.text, workerPid: completed.workerPid, workerUid: completed.workerUid, workerStartTime: completed.workerStartTime, ...accounting }; } catch (error) { const snapshot = meter.snapshot(); @@ -86,7 +86,7 @@ export async function runGrokEngineBrokerTurn(deps: GrokEngineBrokerTurnDependen const accounting = { outcome: "failed", usage: streamOrMeterUsage(stream, snapshot), model: declared, requests: requestCount(stream, snapshot), limitReason: snapshot.limitReason } as const; const failed: EngineBrokerTerminalResponse = { version: request.version, kind: "failed", requestId: request.requestId, turnId, code, ...(diagnostic ? { diagnostic } : {}), ...accounting }; const reason = snapshot.limitReason !== "none" ? limitReasonFor[snapshot.limitReason] : rejected ? "turn_rejected" : "unknown"; - await finishBrokerTurnWithUsage(deps.turns, request, failed, metering, { notionalUsd: 0, complete: false, reason, requests: requestRows(stream, snapshot), ...(stream?.sessionId === undefined ? {} : { session: stream.sessionId }) }); + await finishBrokerTurnWithUsage(deps.turns, request, failed, metering, { notionalUsd: 0, complete: false, reason, estimatedRequests: snapshot.estimatedRequests, requests: requestRows(stream, snapshot), ...(stream?.sessionId === undefined ? {} : { session: stream.sessionId }) }); throw new EngineBrokerTurnFailure(code, diagnostic, accounting); } finally { clearTimeout(timer); signal?.removeEventListener("abort", onAbort); meter.abortInFlight(); @@ -117,8 +117,8 @@ function streamOrMeterUsage(stream: GrokStreamUsage | undefined, snapshot: GrokB function requestRows(stream: GrokStreamUsage | undefined, snapshot: GrokBrokerTurnMeterSnapshot): BrokerTurnMeteringDetail["requests"] { if (stream !== undefined && stream.requests.length > 0) { const timed = snapshot.timings.length === stream.requests.length; - return stream.requests.map((value, index) => ({ ...value, ...(timed ? clock(snapshot.timings[index]!) : {}) })); + return stream.requests.map((value, index) => ({ ...value, usageSource: "stream" as const, ...(timed ? clock(snapshot.timings[index]!) : {}) })); } - return snapshot.timings.flatMap((timing, index) => timing.usage === undefined ? [] : [{ index, ...usageOf(timing.usage), ...clock(timing) }]); + return snapshot.timings.flatMap((timing, index) => timing.usage === undefined ? [] : [{ index, ...usageOf(timing.usage), usageSource: timing.estimated === true ? "estimated" as const : "upstream" as const, ...clock(timing) }]); } const clock = (timing: GrokBrokerTurnMeterSnapshot["timings"][number]) => ({ startedAt: timing.startedAt, ...(timing.endedAt === undefined ? {} : { endedAt: timing.endedAt }) }); diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index 6cca960..9dec696 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -136,9 +136,9 @@ test("the wall-clock limit aborts a worker that is mid-request", async () => { await assert.rejects(turn("wake-4", worker, { timeoutMs: 1_000 }), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.code === "limit_exceeded" && error.accounting?.limitReason === "timeout" && error.accounting.requests === 2); assert.ok(Date.now() - started < 1_400, "the turn ends at the deadline, not when the in-flight request returns"); assert.equal(upstreamAborts(), 1, "the stuck upstream call is aborted, not left running"); - assert.deepEqual((await usageRows()).map((row) => [row.reason, row.limit_reason, row.total, row.calls]), [["wake_timeout", "timeout", 2_775, 2]]); - // One measured row, but both admitted requests count: the killed one was sent upstream. - assert.deepEqual((await requestRows()).map((row) => [row.request, row.requests]), [[0, 2]]); + // The aborted request reported nothing, so it is charged the estimate and says so. + assert.deepEqual((await usageRows()).map((row) => [row.reason, row.limit_reason, row.total, row.calls, row.estimated_requests]), [["wake_timeout", "timeout", 2_775 + 4_297, 2, 1]]); + assert.deepEqual((await requestRows()).map((row) => [row.request, row.requests, row.usage_source, row.total]), [[0, 2, "upstream", 2_775], [1, 2, "estimated", 4_297]]); }, undefined, (call) => call === 2 ? 1_500 : 15); }); diff --git a/src/runtime/turnRequestLedger.ts b/src/runtime/turnRequestLedger.ts index ef8dfca..6cd2844 100644 --- a/src/runtime/turnRequestLedger.ts +++ b/src/runtime/turnRequestLedger.ts @@ -105,7 +105,12 @@ const requestClockFields = (request: Readonly<{ startedAt?: string; endedAt?: st }); /** One Grok broker model request: usage from the worker stream, timing from the proxy. */ -export type GrokTurnRequest = Readonly<{ index: number; input: number; cacheRead: number; cacheWrite: number; output: number; total: number; startedAt?: string; endedAt?: string }>; +/** + * `usageSource`: `stream` (the worker's own per-request frame), `upstream` + * (the provider response the proxy saw), or `estimated` (no valid usage; the + * proxy's conservative charge, see `grokBrokerTurnMeter.ts`). + */ +export type GrokTurnRequest = Readonly<{ index: number; input: number; cacheRead: number; cacheWrite: number; output: number; total: number; startedAt?: string; endedAt?: string; usageSource?: "stream" | "upstream" | "estimated" }>; /** * `requestCount` is the turn's admitted request count when it exceeds the rows: * a killed turn's in-flight request was sent upstream but never reported usage, @@ -140,6 +145,7 @@ export const renderGrokTurnRequestLines = (entry: GrokTurnRequestEntry): string cache_write: request.cacheWrite, output: request.output, total: request.total, + ...(request.usageSource === undefined ? {} : { usage_source: request.usageSource }), ...requestClockFields(request) })}\n`).join(""); }; diff --git a/src/runtime/turnUsageLedger.ts b/src/runtime/turnUsageLedger.ts index 44e7621..6256caf 100644 --- a/src/runtime/turnUsageLedger.ts +++ b/src/runtime/turnUsageLedger.ts @@ -125,6 +125,8 @@ export type TurnUsageEntry = Readonly<{ turn?: string; limitReason?: EngineBrokerLimitReason; model?: GrokBrokerModel; + /** Broker rows only: how many of the turn's requests were charged an estimate because their response carried no valid usage. */ + estimatedRequests?: number; }>; /** @@ -185,10 +187,11 @@ export const renderTurnUsageLine = (entry: TurnUsageEntry): string => `${JSON.st ...brokerFields(entry) })}\n`; -const brokerFields = (entry: TurnUsageEntry): Record => ({ +const brokerFields = (entry: TurnUsageEntry): Record => ({ ...(entry.turn !== undefined && /^[a-f0-9]{64}$/u.test(entry.turn) ? { turn: entry.turn } : {}), ...(entry.limitReason !== undefined && ENGINE_BROKER_LIMIT_REASONS.includes(entry.limitReason) ? { limit_reason: entry.limitReason } : {}), - ...(entry.model !== undefined && (GROK_BROKER_MODELS as readonly string[]).includes(entry.model) ? { model: entry.model } : {}) + ...(entry.model !== undefined && (GROK_BROKER_MODELS as readonly string[]).includes(entry.model) ? { model: entry.model } : {}), + ...(entry.estimatedRequests !== undefined && Number.isSafeInteger(entry.estimatedRequests) && entry.estimatedRequests > 0 ? { estimated_requests: entry.estimatedRequests } : {}) }); /** From 9e49e924505687cbb27d2bd8a91869214d355de7 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:42:11 +0200 Subject: [PATCH 14/21] fix: seal Grok turn ledger bytes in the turn record and complete an interrupted append on replay --- src/runtime/engineBrokerTurnRegistry.test.ts | 26 ++++++- src/runtime/engineBrokerTurnRegistry.ts | 22 +++--- src/runtime/grokEngineBrokerLedger.ts | 82 ++++++++++++++++++++ src/runtime/grokEngineBrokerMetering.ts | 45 ++++++----- src/runtime/grokEngineBrokerTurn.ts | 13 ++-- src/runtime/grokEngineBrokerUsage.test.ts | 19 ++++- src/runtime/turnRequestLedger.ts | 10 +++ 7 files changed, 177 insertions(+), 40 deletions(-) create mode 100644 src/runtime/grokEngineBrokerLedger.ts diff --git a/src/runtime/engineBrokerTurnRegistry.test.ts b/src/runtime/engineBrokerTurnRegistry.test.ts index 3de4e20..249cbd6 100644 --- a/src/runtime/engineBrokerTurnRegistry.test.ts +++ b/src/runtime/engineBrokerTurnRegistry.test.ts @@ -11,14 +11,14 @@ test("turn registry replays terminal results across restart and rejects conflict try { const first = new EngineBrokerTurnRegistry(root,"boot-a"); assert.equal(await first.begin(start(),"grok-4.6"), "start"); await assert.rejects(first.begin(start(),"grok-4.6"), /already active/); - const response = { version: "noopolis.daimon.engine-broker.v2", kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 11, workerUid: 2200, workerStartTime: "123", outcome: "completed", usage: { input: 3, cacheRead: 2, cacheWrite: 0, output: 1, total: 6 }, model: "grok-4.6", requests: 1, limitReason: "none" } as const; + const response = { version: "noopolis.daimon.engine-broker.v2", kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 11, workerUid: 2200, workerStartTime: "123", outcome: "completed", usage: null, model: "grok-4.6", requests: 1, limitReason: "none" } as const; await first.finish(start(), response); - assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-b").begin(start(),"grok-4.6"), { replay: response }); + assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-b").begin(start(),"grok-4.6"), { replay: response, ledger: { usage: null, requests: "" } }); await assert.rejects(first.begin(start("different"),"grok-4.6"), /conflict/); } finally { await rm(root, { recursive: true, force: true }); } }); test("turn registry fails an orphaned active turn once after broker restart",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{assert.equal(await new EngineBrokerTurnRegistry(root,"boot-a").begin(start(),"grok-4.6"),"start");const replay=await new EngineBrokerTurnRegistry(root,"boot-b").begin(start(),"grok-4.6");assert.equal(typeof replay,"object");if(typeof replay==="object")assert.equal(replay.replay.kind,"failed");assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-c").begin(start(),"grok-4.6"),replay);}finally{await rm(root,{recursive:true,force:true});}}); -test("turn registry durably replays a sanitized pre-attestation failure",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{const registry=new EngineBrokerTurnRegistry(root,"boot-a");assert.equal(await registry.begin(start(),"grok-4.6"),"start");const response={version:"noopolis.daimon.engine-broker.v2",kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",diagnostic:{status:"worker_failed",stage:"attestation",failureClass:"profile_missing",profileApplied:false,exitCode:0,termSignal:0,workerPid:42,workerUid:2200,startTicks:"123"},outcome:"failed",usage:null,model:"grok-4.6",requests:0,limitReason:"none"} as const;await registry.finish(start(),response);assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-b").begin(start(),"grok-4.6"),{replay:response});}finally{await rm(root,{recursive:true,force:true});}}); +test("turn registry durably replays a sanitized pre-attestation failure",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{const registry=new EngineBrokerTurnRegistry(root,"boot-a");assert.equal(await registry.begin(start(),"grok-4.6"),"start");const response={version:"noopolis.daimon.engine-broker.v2",kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",diagnostic:{status:"worker_failed",stage:"attestation",failureClass:"profile_missing",profileApplied:false,exitCode:0,termSignal:0,workerPid:42,workerUid:2200,startTicks:"123"},outcome:"failed",usage:null,model:"grok-4.6",requests:0,limitReason:"none"} as const;await registry.finish(start(),response);assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-b").begin(start(),"grok-4.6"),{replay:response,ledger:{usage:null,requests:""}});}finally{await rm(root,{recursive:true,force:true});}}); test("turn registry rejects a persisted diagnostic with undeclared secret-bearing fields",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{const registry=new EngineBrokerTurnRegistry(root,"boot-a");assert.equal(await registry.begin(start(),"grok-4.6"),"start");const [name]=await readdir(root);const file=path.join(root,name!);const record=JSON.parse(await readFile(file,"utf8")) as Record;record.state="terminal";record.response={version:"noopolis.daimon.engine-broker.v2",kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",rawOutput:"secret",outcome:"failed",usage:null,model:"grok-4.6",requests:0,limitReason:"none"};await writeFile(file,JSON.stringify(record));await assert.rejects(new EngineBrokerTurnRegistry(root,"boot-b").begin(start(),"grok-4.6"),/registry unavailable/u);}finally{await rm(root,{recursive:true,force:true});}}); const withRoot = async (run: (root: string) => Promise): Promise => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-turns-")); try { await run(root); } finally { await rm(root, { recursive: true, force: true }); } }; @@ -31,7 +31,7 @@ test("a v1 record sealed before the upgrade still replays, upgraded with no usag const file = await recordFile(root); const record = JSON.parse(await readFile(file, "utf8")) as Record; const v1 = { version: "noopolis.daimon.engine-broker.v1", kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 11, workerUid: 2200, workerStartTime: "123" }; await writeFile(file, JSON.stringify({ version: "noopolis.daimon.engine-broker-turn.v1", digest: record.digest, state: "terminal", bootId: "boot-a", response: v1 })); - assert.deepEqual(await new EngineBrokerTurnRegistry(root, "boot-b").begin(start(), "grok-4.5"), { replay: { ...v1, version: "noopolis.daimon.engine-broker.v2", outcome: "completed", usage: null, model: "grok-4.5", requests: 0, limitReason: "none" } }); + assert.deepEqual(await new EngineBrokerTurnRegistry(root, "boot-b").begin(start(), "grok-4.5"), { replay: { ...v1, version: "noopolis.daimon.engine-broker.v2", outcome: "completed", usage: null, model: "grok-4.5", requests: 0, limitReason: "none" }, ledger: { usage: null, requests: "" } }); }); }); @@ -44,6 +44,7 @@ test("the v2 record parser is strict: an unknown member or a v1 frame inside a v const file = await recordFile(root); const record = JSON.parse(await readFile(file, "utf8")) as Record; assert.equal(record.version, "noopolis.daimon.engine-broker-turn.v2"); // Mutation guard: dropping the exact-member check accepts this record. + assert.deepEqual(record.ledger, { usage: null, requests: "" }); await writeFile(file, JSON.stringify({ ...record, usageRow: "extra" })); await assert.rejects(new EngineBrokerTurnRegistry(root, "boot-b").begin(start(), "grok-4.6"), /registry unavailable/u); const { outcome: _o, usage: _u, model: _m, requests: _r, limitReason: _l, ...legacy } = response; @@ -51,3 +52,20 @@ test("the v2 record parser is strict: an unknown member or a v1 frame inside a v await assert.rejects(new EngineBrokerTurnRegistry(root, "boot-b").begin(start(), "grok-4.6"), /registry unavailable/u); }); }); + +test("sealed ledger bytes must be this turn's own rows and agree with the sealed usage", async () => { + await withRoot(async (root) => { + const registry = new EngineBrokerTurnRegistry(root, "boot-a"); + assert.equal(await registry.begin(start(), "grok-4.6"), "start"); + const turnId = "turn-1"; + const usage = { input: 3, cacheRead: 2, cacheWrite: 0, output: 1, total: 6 }; + const response = { version: "noopolis.daimon.engine-broker.v2", kind: "completed", requestId: "request-1", turnId, text: "done", workerPid: 11, workerUid: 2200, workerStartTime: "123", outcome: "completed", usage, model: "grok-4.6", requests: 1, limitReason: "none" } as const; + const line = (turn: string) => `${JSON.stringify({ v: "noopolis.daimon.turn-usage.v1", turn, total: 6 })}\n`; + await registry.finish(start(), response, { usage: line(turnId), requests: "" }); + assert.deepEqual(await new EngineBrokerTurnRegistry(root, "boot-b").begin(start(), "grok-4.6"), { replay: response, ledger: { usage: line(turnId), requests: "" } }); + for (const ledger of [{ usage: line("other-turn"), requests: "" }, { usage: null, requests: "" }, { usage: line(turnId), requests: "not json\n" }, { usage: line(turnId), requests: "", extra: 1 }]) { + await registry.finish(start(), response, ledger as never); + await assert.rejects(new EngineBrokerTurnRegistry(root, "boot-c").begin(start(), "grok-4.6"), /registry unavailable/u, JSON.stringify(ledger)); + } + }); +}); diff --git a/src/runtime/engineBrokerTurnRegistry.ts b/src/runtime/engineBrokerTurnRegistry.ts index 1ddd01c..d59bab7 100644 --- a/src/runtime/engineBrokerTurnRegistry.ts +++ b/src/runtime/engineBrokerTurnRegistry.ts @@ -4,6 +4,7 @@ import { mkdir, open, readFile, rename, unlink } from "node:fs/promises"; import path from "node:path"; import { parseEngineBrokerResponse, parseEngineBrokerV1TerminalResponse, type EngineBrokerRequest, type EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; import type { GrokBrokerModel } from "./grokBrokerModelPolicy.js"; +import { EMPTY_BROKER_TURN_LEDGER, parseBrokerTurnLedgerLines, type BrokerTurnLedgerLines } from "./grokEngineBrokerLedger.js"; type Start = Extract; type Terminal = EngineBrokerTerminalResponse; @@ -13,7 +14,7 @@ export const ENGINE_BROKER_TURN_RECORD_V2 = "noopolis.daimon.engine-broker-turn. // record written before the upgrade still identifies the same turn. const digest = (request: Start): string => createHash("sha256").update(JSON.stringify([request.turnId, request.agentId, request.wakeId, request.prompt,request.mcpEndpoint])).digest("hex"); const safe = (turnId: string): string => `${createHash("sha256").update(turnId).digest("hex")}.json`; -type Observed = Readonly<{ version: typeof ENGINE_BROKER_TURN_RECORD_V1 | typeof ENGINE_BROKER_TURN_RECORD_V2; digest: string; state: "active" | "terminal"; bootId: string; response?: Terminal }>; +type Observed = Readonly<{ version: typeof ENGINE_BROKER_TURN_RECORD_V1 | typeof ENGINE_BROKER_TURN_RECORD_V2; digest: string; state: "active" | "terminal"; bootId: string; response?: Terminal; ledger?: BrokerTurnLedgerLines }>; /** * Durable per-turn state. Record v2 stores the terminal response *with* its @@ -24,28 +25,29 @@ type Observed = Readonly<{ version: typeof ENGINE_BROKER_TURN_RECORD_V1 | typeof export class EngineBrokerTurnRegistry { constructor(private readonly root: string,private readonly bootId:string=randomUUID()) {} /** `model` is the registration's declared model, used only to upgrade a v1 record on replay. */ - async begin(request: Start, model: GrokBrokerModel): Promise<"start" | { replay: Terminal }> { + async begin(request: Start, model: GrokBrokerModel): Promise<"start" | { replay: Terminal; ledger: BrokerTurnLedgerLines }> { await mkdir(this.root, { recursive: true, mode: 0o700 }); const file = path.join(this.root, safe(request.turnId)); const expected = digest(request); try { const handle = await open(file, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); try { await handle.writeFile(`${JSON.stringify({ version: ENGINE_BROKER_TURN_RECORD_V2, digest: expected, state: "active",bootId:this.bootId })}\n`); await handle.sync(); } finally { await handle.close(); } await syncDirectory(this.root); return "start"; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw new Error("broker turn registry unavailable"); } const observed = parseEngineBrokerTurnRecord(await readFile(file, "utf8"), model); if (observed.digest !== expected) throw new Error("broker turn conflict"); - if (observed.state === "terminal" && observed.response !== undefined) return { replay: observed.response }; - if(observed.state==="active"&&observed.bootId!==this.bootId){const response={version:request.version,kind:"failed",requestId:request.requestId,turnId:request.turnId,code:"engine_failed",outcome:"failed",usage:null,model,requests:0,limitReason:"none"} as const;await this.finish(request,response);return {replay:response};} + if (observed.state === "terminal" && observed.response !== undefined) return { replay: observed.response, ledger: observed.ledger ?? EMPTY_BROKER_TURN_LEDGER }; + if(observed.state==="active"&&observed.bootId!==this.bootId){const response={version:request.version,kind:"failed",requestId:request.requestId,turnId:request.turnId,code:"engine_failed",outcome:"failed",usage:null,model,requests:0,limitReason:"none"} as const;await this.finish(request,response);return {replay:response,ledger:EMPTY_BROKER_TURN_LEDGER};} throw new Error("broker turn already active"); } - async finish(request: Start, response: Terminal): Promise { + /** `ledger` is the exact ledger bytes this turn owes, sealed with it so a replay can finish an interrupted append. */ + async finish(request: Start, response: Terminal, ledger: BrokerTurnLedgerLines = EMPTY_BROKER_TURN_LEDGER): Promise { const file = path.join(this.root, safe(request.turnId)); const temporary = `${file}.${randomUUID()}.tmp`; const handle = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); - try { await handle.writeFile(`${JSON.stringify({ version: ENGINE_BROKER_TURN_RECORD_V2, digest: digest(request), state: "terminal",bootId:this.bootId, response })}\n`); await handle.sync(); } finally { await handle.close(); } + try { await handle.writeFile(`${JSON.stringify({ version: ENGINE_BROKER_TURN_RECORD_V2, digest: digest(request), state: "terminal",bootId:this.bootId, response, ledger })}\n`); await handle.sync(); } finally { await handle.close(); } try { await rename(temporary, file); await syncDirectory(this.root); } finally { await unlink(temporary).catch(() => undefined); } } } /** * Strict record parser. v2 accepts exactly `{version,digest,state,bootId}` - * plus `response` when terminal, and the response must be a v2 terminal frame. + * plus `response` and its sealed `ledger` bytes when terminal, and the response must be a v2 terminal frame. * v1 records keep their historical looser shape and are upgraded on read: no * usage (`null`), zero requests, `limitReason: "none"`, the declared model. */ @@ -65,13 +67,15 @@ export function parseEngineBrokerTurnRecord(text: string, model: GrokBrokerModel return { version: ENGINE_BROKER_TURN_RECORD_V1, ...base, response }; } if (input.version !== ENGINE_BROKER_TURN_RECORD_V2) throw new Error("broker turn conflict"); - const fields = input.state === "terminal" ? ["version", "digest", "state", "bootId", "response"] : ["version", "digest", "state", "bootId"]; + const fields = input.state === "terminal" ? ["version", "digest", "state", "bootId", "response", "ledger"] : ["version", "digest", "state", "bootId"]; if (Object.keys(input).length !== fields.length || fields.some((field) => !Object.hasOwn(input, field))) throw new Error("broker turn registry unavailable"); if (input.state === "active") return { version: ENGINE_BROKER_TURN_RECORD_V2, ...base }; let response; try { response = parseEngineBrokerResponse(input.response); } catch { throw new Error("broker turn registry unavailable"); } if (response.kind !== "completed" && response.kind !== "failed") throw new Error("broker turn registry unavailable"); - return { version: ENGINE_BROKER_TURN_RECORD_V2, ...base, response }; + const ledger = parseBrokerTurnLedgerLines(input.ledger, response.turnId); + if ((response.usage === null) !== (ledger.usage === null)) throw new Error("broker turn registry unavailable"); + return { version: ENGINE_BROKER_TURN_RECORD_V2, ...base, response, ledger }; } async function syncDirectory(directory: string): Promise { const handle = await open(directory, constants.O_RDONLY); try { await handle.sync(); } finally { await handle.close(); } } diff --git a/src/runtime/grokEngineBrokerLedger.ts b/src/runtime/grokEngineBrokerLedger.ts new file mode 100644 index 0000000..ebf6073 --- /dev/null +++ b/src/runtime/grokEngineBrokerLedger.ts @@ -0,0 +1,82 @@ +import { readFile } from "node:fs/promises"; + +import { recordLedgerLines, renderGrokTurnRequestLines, TURN_REQUEST_LEDGER_VERSION, type GrokTurnRequest } from "./turnRequestLedger.js"; +import { renderTurnUsageLine, TURN_USAGE_LEDGER_VERSION, type TurnUsageFailureReason } from "./turnUsageLedger.js"; +import type { EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; + +/** + * The exact ledger bytes a terminal broker turn owes, sealed into its turn + * record *before* they are appended. + * + * The record is published first and the ledger appended second, so a crash in + * between used to leave a sealed turn whose spend never reached the ledger — + * and a replay never metered. Now a replay re-checks: if the ledger holds no + * row for this `turn`, it appends these same bytes (same `at`, same numbers). + * That is completing the original metering, not re-metering: a replay after a + * normal append finds the row and writes nothing, and readers dedupe on `turn` + * should two replays race. + */ +export type BrokerTurnLedgerLines = Readonly<{ usage: string | null; requests: string }>; +export const EMPTY_BROKER_TURN_LEDGER: BrokerTurnLedgerLines = Object.freeze({ usage: null, requests: "" }); + +export type BrokerTurnLedgerDetail = Readonly<{ agentId: string; wakeId: string; notionalUsd: number; complete: boolean; reason?: TurnUsageFailureReason; requests: readonly GrokTurnRequest[]; session?: string; estimatedRequests: number }>; + +export function renderBrokerTurnLedger(terminal: EngineBrokerTerminalResponse, detail: BrokerTurnLedgerDetail): BrokerTurnLedgerLines { + if (terminal.usage === null) return EMPTY_BROKER_TURN_LEDGER; + const { usage } = terminal, at = new Date().toISOString(); + return { + usage: renderTurnUsageLine({ + agent: detail.agentId, wake: detail.wakeId, engine: "grok", at, + usage: { input: usage.input, output: usage.output, cacheRead: usage.cacheRead, cacheWrite: usage.cacheWrite, total: usage.total, calls: terminal.requests, notionalUsd: detail.notionalUsd, complete: detail.complete }, + outcome: terminal.kind === "completed" ? { status: "completed" } : { status: "failed", reason: detail.reason ?? "unknown" }, + turn: terminal.turnId, limitReason: terminal.limitReason, model: terminal.model, estimatedRequests: detail.estimatedRequests + }), + requests: renderGrokTurnRequestLines({ agent: detail.agentId, wake: detail.wakeId, turn: terminal.turnId, model: terminal.model, requests: detail.requests, requestCount: terminal.requests, at, ...(detail.session === undefined ? {} : { session: detail.session }) }) + }; +} + +const MAX_USAGE_LINE_BYTES = 4_096, MAX_REQUEST_LINES_BYTES = 262_144; +const rows = (text: string): Record[] => text.split("\n").filter((line) => line.length > 0).map((line) => { const value: unknown = JSON.parse(line); if (value === null || typeof value !== "object" || Array.isArray(value)) throw new Error(); return value as Record; }); + +/** Strict check of stored ledger bytes: exactly the turn's own rows, newline-terminated, bounded. */ +export function parseBrokerTurnLedgerLines(value: unknown, turnId: string): BrokerTurnLedgerLines { + const invalid = () => new Error("broker turn registry unavailable"); + if (value === null || typeof value !== "object" || Array.isArray(value) || Object.keys(value).length !== 2 || !Object.hasOwn(value, "usage") || !Object.hasOwn(value, "requests")) throw invalid(); + const { usage, requests } = value as { usage: unknown; requests: unknown }; + try { + if (usage !== null) { + if (typeof usage !== "string" || !usage.endsWith("\n") || Buffer.byteLength(usage) > MAX_USAGE_LINE_BYTES) throw invalid(); + const parsed = rows(usage); + if (parsed.length !== 1 || parsed[0]!.v !== TURN_USAGE_LEDGER_VERSION || parsed[0]!.turn !== turnId) throw invalid(); + } + if (typeof requests !== "string" || (requests.length > 0 && (usage === null || !requests.endsWith("\n"))) || Buffer.byteLength(requests) > MAX_REQUEST_LINES_BYTES) throw invalid(); + if (rows(requests).some((row) => row.v !== TURN_REQUEST_LEDGER_VERSION || row.turn !== turnId)) throw invalid(); + } catch { throw invalid(); } + return { usage: usage as string | null, requests }; +} + +/** Appends sealed lines on the first metering: no presence scan is needed, nothing was appended before the record existed. */ +export async function appendBrokerTurnLedger(lines: BrokerTurnLedgerLines, paths: Readonly<{ usageLedgerPath: string; requestLedgerPath: string }>): Promise { + if (lines.usage !== null) await recordLedgerLines(paths.usageLedgerPath, lines.usage); + await recordLedgerLines(paths.requestLedgerPath, lines.requests); +} + +/** On replay: append each stream's sealed lines only when that stream (current file or its `.1`) holds no row for this turn. Never rejects. */ +export async function ensureBrokerTurnLedgered(lines: BrokerTurnLedgerLines, turnId: string, paths: Readonly<{ usageLedgerPath: string; requestLedgerPath: string }>): Promise { + try { + if (lines.usage !== null && !await ledgerHasTurn(paths.usageLedgerPath, turnId)) await recordLedgerLines(paths.usageLedgerPath, lines.usage); + if (lines.requests.length > 0 && !await ledgerHasTurn(paths.requestLedgerPath, turnId)) await recordLedgerLines(paths.requestLedgerPath, lines.requests); + } catch { /* advisory: a replay never fails on its ledger */ } +} + +async function ledgerHasTurn(file: string, turnId: string): Promise { + for (const candidate of [`${file}.1`, file]) { + let text: string; + try { text = await readFile(candidate, "utf8"); } catch { continue; } + for (const line of text.split("\n")) { + if (!line.includes(turnId)) continue; + try { if ((JSON.parse(line) as { turn?: unknown }).turn === turnId) return true; } catch { /* a torn line is not this turn's row */ } + } + } + return false; +} diff --git a/src/runtime/grokEngineBrokerMetering.ts b/src/runtime/grokEngineBrokerMetering.ts index 12c25a4..8b8fb31 100644 --- a/src/runtime/grokEngineBrokerMetering.ts +++ b/src/runtime/grokEngineBrokerMetering.ts @@ -1,7 +1,8 @@ import type { EngineBrokerRequest, EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; import type { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; -import { recordGrokTurnRequests, type GrokTurnRequest } from "./turnRequestLedger.js"; -import { recordTurnUsage, type TurnUsageFailureReason } from "./turnUsageLedger.js"; +import { appendBrokerTurnLedger, renderBrokerTurnLedger } from "./grokEngineBrokerLedger.js"; +import type { GrokTurnRequest } from "./turnRequestLedger.js"; +import type { TurnUsageFailureReason } from "./turnUsageLedger.js"; export type BrokerTurnMetering = Readonly<{ usageLedgerPath: string; @@ -14,30 +15,32 @@ export type BrokerTurnMeteringDetail = Readonly<{ notionalUsd: number; complete: /** * Seal a terminal turn, then meter it. The broker is the single writer. * - * Order is load-bearing. `turns.finish` publishes the durable terminal record - * *with* its accounting; only after that are the advisory ledger rows appended. - * A replayed turn returns before the broker's `try` block and never reaches - * here, so a crash-recovered or repeated turn cannot double-count; every row - * also carries the turn id as `turn`, so a reader that sees one twice counts - * it once. + * Order is load-bearing. The ledger bytes are rendered first and sealed into + * the durable terminal record together with its accounting + * (`grokEngineBrokerLedger.ts`); only after `turns.finish` published that + * record are the same bytes appended. A replayed turn returns before the + * broker's `try` block and never meters again — it only completes an append a + * crash interrupted (`ensureBrokerTurnLedgered`), and every row carries the + * turn id as `turn`, so a reader that sees one twice counts it once. + * + * Remaining window, documented rather than closed: a crash before the record's + * rename (while the turn is still `active`, including mid-turn) makes the next + * boot seal that turn `failed` with `usage: null`, so its spend is unmetered. + * Closing it needs the running proxy usage checkpointed into the active record + * on every request (an fsync'd rewrite per model request); not done here. * * Both terminal kinds meter: a failed turn spent real tokens, so its partial * usage is written with `outcome: failed` and its closed `limitReason`. A turn * with no usage at all (`usage: null`) writes nothing — a zero row is * byte-identical to a measured zero. * - * `recordTurnUsage`/`recordGrokTurnRequests` never reject, so an append failure - * cannot escape into the caller's `catch` and rewrite a completed turn as failed. + * Appends never reject, so an append failure cannot escape into the caller's + * `catch` and rewrite a completed turn as failed; the caller also refuses to + * re-seal a turn this helper already sealed. */ -export async function finishBrokerTurnWithUsage(turns: EngineBrokerTurnRegistry, request: Extract, terminal: EngineBrokerTerminalResponse, metering: BrokerTurnMetering, detail: BrokerTurnMeteringDetail): Promise { - await turns.finish(request, terminal); - if (terminal.usage === null) return; - const { usage } = terminal; - await recordTurnUsage(metering.usageLedgerPath, { - agent: metering.agentId, wake: metering.wakeId, engine: "grok", - usage: { input: usage.input, output: usage.output, cacheRead: usage.cacheRead, cacheWrite: usage.cacheWrite, total: usage.total, calls: terminal.requests, notionalUsd: detail.notionalUsd, complete: detail.complete }, - outcome: terminal.kind === "completed" ? { status: "completed" } : { status: "failed", reason: detail.reason ?? "unknown" }, - turn: terminal.turnId, limitReason: terminal.limitReason, model: terminal.model, estimatedRequests: detail.estimatedRequests - }); - await recordGrokTurnRequests(metering.requestLedgerPath, { agent: metering.agentId, wake: metering.wakeId, turn: terminal.turnId, model: terminal.model, requests: detail.requests, requestCount: terminal.requests, ...(detail.session === undefined ? {} : { session: detail.session }) }); +export async function finishBrokerTurnWithUsage(turns: EngineBrokerTurnRegistry, request: Extract, terminal: EngineBrokerTerminalResponse, metering: BrokerTurnMetering, detail: BrokerTurnMeteringDetail, onSealed: () => void = () => undefined): Promise { + const lines = renderBrokerTurnLedger(terminal, { ...detail, agentId: metering.agentId, wakeId: metering.wakeId }); + await turns.finish(request, terminal, lines); + onSealed(); + await appendBrokerTurnLedger(lines, metering); } diff --git a/src/runtime/grokEngineBrokerTurn.ts b/src/runtime/grokEngineBrokerTurn.ts index 650c94b..d1ae705 100644 --- a/src/runtime/grokEngineBrokerTurn.ts +++ b/src/runtime/grokEngineBrokerTurn.ts @@ -7,6 +7,7 @@ import { lowerEngineBrokerTurnLimits, mapGrokReportedModel, type EngineBrokerTur import { NativeBrokerTurnFailure, type NativeBrokerDiagnostic, type NativeBrokerTurn, type NativeBrokerTurnResult } from "./engineBrokerNativeClient.js"; import type { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; import { engineBrokerRequestLedgerPathFor, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +import { ensureBrokerTurnLedgered } from "./grokEngineBrokerLedger.js"; import { finishBrokerTurnWithUsage, type BrokerTurnMetering, type BrokerTurnMeteringDetail } from "./grokEngineBrokerMetering.js"; import type { GrokBrokerProxyTurn } from "./grokBrokerProxy.js"; import { GrokBrokerTurnMeter, type GrokBrokerTurnMeterSnapshot } from "./grokBrokerTurnMeter.js"; @@ -51,13 +52,14 @@ export async function runGrokEngineBrokerTurn(deps: GrokEngineBrokerTurnDependen const turnId = createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"); const request = { version: ENGINE_BROKER_VERSION, kind: "start_turn", requestId: randomUUID(), turnId, agentId, wakeId, prompt, mcpEndpoint, ...(overrides === undefined ? {} : { limits: overrides }) } as const; const begun = await deps.turns.begin(request, declared); - if (begun !== "start") return replay(begun.replay); + const metering: BrokerTurnMetering = { usageLedgerPath: registration.usageLedgerPath, requestLedgerPath: engineBrokerRequestLedgerPathFor(registration.usageLedgerPath), agentId, wakeId }; + if (begun !== "start") { await ensureBrokerTurnLedgered(begun.ledger, turnId, metering); return replay(begun.replay); } const controller = new AbortController(); const meter = new GrokBrokerTurnMeter(limits, () => controller.abort()); const onAbort = () => controller.abort(); signal?.addEventListener("abort", onAbort, { once: true }); if (signal?.aborted) controller.abort(); const timer = setTimeout(() => meter.trip("timeout"), limits.timeoutMs); timer.unref?.(); - const metering: BrokerTurnMetering = { usageLedgerPath: registration.usageLedgerPath, requestLedgerPath: engineBrokerRequestLedgerPathFor(registration.usageLedgerPath), agentId, wakeId }; - let nativeDiagnostic: NativeBrokerDiagnostic | undefined, attested = false, output: string | undefined, rejected = false; + + let nativeDiagnostic: NativeBrokerDiagnostic | undefined, attested = false, output: string | undefined, rejected = false, sealed: GrokEngineBrokerTurnResult | undefined; try { const isolationGuard = await deps.prepareIsolation(registration); deps.proxy.registerIsolationGuard(turnId, isolationGuard); @@ -76,8 +78,9 @@ export async function runGrokEngineBrokerTurn(deps: GrokEngineBrokerTurnDependen const usage = decoded.usage === undefined ? streamOrMeterUsage(stream, snapshot) : usageOf(decoded.usage); const accounting = { outcome: "completed", usage, model: declared, requests: requestCount(stream, snapshot), limitReason: "none" } as const; const completed = { version: request.version, kind: "completed", requestId: request.requestId, turnId, text: decoded.text, workerPid: result.workerPid, workerUid: result.workerUid, workerStartTime: result.startTicks.toString(), ...accounting } as const; - await finishBrokerTurnWithUsage(deps.turns, request, completed, metering, { notionalUsd: decoded.usage?.notionalUsd ?? 0, complete: decoded.usage?.complete ?? false, estimatedRequests: snapshot.estimatedRequests, requests: requestRows(stream, snapshot), ...(stream.sessionId === undefined ? {} : { session: stream.sessionId }) }); - return { text: completed.text, workerPid: completed.workerPid, workerUid: completed.workerUid, workerStartTime: completed.workerStartTime, ...accounting }; + const result_ = { text: completed.text, workerPid: completed.workerPid, workerUid: completed.workerUid, workerStartTime: completed.workerStartTime, ...accounting }; + await finishBrokerTurnWithUsage(deps.turns, request, completed, metering, { notionalUsd: decoded.usage?.notionalUsd ?? 0, complete: decoded.usage?.complete ?? false, estimatedRequests: snapshot.estimatedRequests, requests: requestRows(stream, snapshot), ...(stream.sessionId === undefined ? {} : { session: stream.sessionId }) }, () => { sealed = result_; }); + return result_; } catch (error) { const snapshot = meter.snapshot(); const code: EngineBrokerTurnFailure["code"] = snapshot.limitReason !== "none" ? "limit_exceeded" : deps.credentialStale() ? "auth_stale" : controller.signal.aborted ? "cancelled" : "engine_failed"; diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index 9dec696..ce36973 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; -import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; import { request as httpRequest } from "node:http"; import os from "node:os"; import path from "node:path"; @@ -159,6 +159,23 @@ test("a turn whose stream reports an undeclared model fails as rejected but is s }); }); +test("a crash between sealing and appending is completed by the replay exactly once, with the sealed bytes", async () => { + await withBroker(async ({ root, turn, usageRows, requestRows }) => { + await turn("wake-8", twoRequests); + const [sealedUsage] = await usageRows(); const sealedRequests = await requestRows(); + // Simulate the crash window: the record is published but the append never happened. + await rm(path.join(root, "usage.jsonl")); await rm(path.join(root, "requests.jsonl")); + // Mutation guard: a replay that never ensures its ledger leaves this spend unmetered. + assert.equal((await turn("wake-8", async () => { throw new Error("a replay runs no worker"); })).outcome, "completed"); + assert.deepEqual(await usageRows(), [sealedUsage]); + assert.deepEqual(await requestRows(), sealedRequests); + // A replay after the rows exist writes nothing further. + await turn("wake-8", async () => { throw new Error("a replay runs no worker"); }); + assert.equal((await usageRows()).length, 1); + assert.equal((await requestRows()).length, 2); + }); +}); + test("an unwritable ledger leaves the turn recorded as completed, not failed", async () => { await withBroker(async ({ root, turn }) => { assert.equal((await turn("wake-7", twoRequests)).outcome, "completed"); diff --git a/src/runtime/turnRequestLedger.ts b/src/runtime/turnRequestLedger.ts index 6cd2844..bb33992 100644 --- a/src/runtime/turnRequestLedger.ts +++ b/src/runtime/turnRequestLedger.ts @@ -150,6 +150,16 @@ export const renderGrokTurnRequestLines = (entry: GrokTurnRequestEntry): string })}\n`).join(""); }; +/** + * Append already-rendered, newline-terminated ledger lines in one write, with the + * same rotation and file mode as both ledgers. Advisory: never rejects. The + * broker uses it to append the exact bytes it sealed into a turn record. + */ +export const recordLedgerLines = async (file: string, lines: string): Promise => { + if (lines.length === 0) return false; + try { await rotate(file); await appendLines(file, lines); return true; } catch { return false; } +}; + /** Advisory and never rejects, like {@link recordTurnRequests}; an empty turn writes nothing. */ export const recordGrokTurnRequests = async (file: string, entry: GrokTurnRequestEntry): Promise => { if (entry.requests.length === 0) return false; From f1f95f01bef9900a5f4ad303181febb140e30836 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:42:13 +0200 Subject: [PATCH 15/21] fix: never re-seal a completed Grok turn when metering after the seal fails --- src/runtime/grokEngineBrokerTurn.ts | 4 ++++ src/runtime/grokEngineBrokerUsage.test.ts | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/runtime/grokEngineBrokerTurn.ts b/src/runtime/grokEngineBrokerTurn.ts index d1ae705..674b572 100644 --- a/src/runtime/grokEngineBrokerTurn.ts +++ b/src/runtime/grokEngineBrokerTurn.ts @@ -82,6 +82,10 @@ export async function runGrokEngineBrokerTurn(deps: GrokEngineBrokerTurnDependen await finishBrokerTurnWithUsage(deps.turns, request, completed, metering, { notionalUsd: decoded.usage?.notionalUsd ?? 0, complete: decoded.usage?.complete ?? false, estimatedRequests: snapshot.estimatedRequests, requests: requestRows(stream, snapshot), ...(stream.sessionId === undefined ? {} : { session: stream.sessionId }) }, () => { sealed = result_; }); return result_; } catch (error) { + // Once the completed record is published it is the durable truth: anything + // failing after that (metering) must neither re-seal the turn as failed nor + // append a second row. + if (sealed !== undefined) return sealed; const snapshot = meter.snapshot(); const code: EngineBrokerTurnFailure["code"] = snapshot.limitReason !== "none" ? "limit_exceeded" : deps.credentialStale() ? "auth_stale" : controller.signal.aborted ? "cancelled" : "engine_failed"; const diagnostic = error instanceof NativeBrokerTurnFailure ? error.diagnostic : nativeDiagnostic && !attested ? { ...nativeDiagnostic, status: "worker_failed" as const, stage: "attestation" as const, failureClass: error instanceof GrokWorkerAttestationFailure ? error.failureClass : "profile_invalid" as const, profileApplied: false } : undefined; diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index ce36973..2c3fbbc 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -176,6 +176,17 @@ test("a crash between sealing and appending is completed by the replay exactly o }); }); +test("a ledger append that fails after the turn was sealed leaves it completed and appends nothing twice", async () => { + await withBroker(async ({ root, turn, usageRows }) => { + // The request stream cannot be written (its path is a directory); the usage stream can. + await mkdir(path.join(root, "requests.jsonl")); + assert.equal((await turn("wake-9", twoRequests)).outcome, "completed"); + assert.deepEqual((await usageRows()).map((row) => [row.outcome, row.turn]), [["completed", turnIdFor("foreman", "wake-9")]]); + assert.equal((await turn("wake-9", twoRequests, undefined, undefined, path.join(root, "turns"))).outcome, "completed", "the sealed record was never rewritten as failed"); + assert.equal((await usageRows()).length, 1); + }); +}); + test("an unwritable ledger leaves the turn recorded as completed, not failed", async () => { await withBroker(async ({ root, turn }) => { assert.equal((await turn("wake-7", twoRequests)).outcome, "completed"); From c9dbf3c1a18a6c0f20c8973b79c0f13bfae5d767 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:42:25 +0200 Subject: [PATCH 16/21] test: refuse a slot preflight receipt carrying a canary the projection does not deny --- .../receipt.extra-canary.json | 42 +++++++++++++++++++ src/runtime/grokSlotPreflightReceipt.test.ts | 2 + 2 files changed, 44 insertions(+) create mode 100644 src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json new file mode 100644 index 0000000..719bdc8 --- /dev/null +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json @@ -0,0 +1,42 @@ +{ + "version": "noopolis.daimon.grok-slot-preflight.v1", + "slot": 0, + "worker_uid": 2200, + "projection_sha256": "b41a6952dfe3319f014866e7468df69ca2e77cfd28d781ac39cfdf0d3d587bd8", + "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", + "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", + "canaries": [ + { + "path": "/run/paideia", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/paideia/control", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/training/inputs", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-bootstrap-auth", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-subscription-realm", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/not-projected", + "method": "sandboxed-read", + "result": "denied" + } + ], + "created_at": "2026-09-17T12:00:00.000Z" +} diff --git a/src/runtime/grokSlotPreflightReceipt.test.ts b/src/runtime/grokSlotPreflightReceipt.test.ts index 77d80c1..d6fb4cd 100644 --- a/src/runtime/grokSlotPreflightReceipt.test.ts +++ b/src/runtime/grokSlotPreflightReceipt.test.ts @@ -38,6 +38,8 @@ test("a receipt for a different projection, slot, profile or deny set is refused // Mutation guard: dropping the digest comparison accepts this fixture. await assert.rejects(async () => verifyGrokSlotPreflightReceipt(await fixture("receipt.projection-mismatch.json"), projected), /projection_sha256/u); await assert.rejects(async () => verifyGrokSlotPreflightReceipt(await fixture("receipt.missing-canary.json"), projected), /canaries/u); + // Exact match, both halves: a canary for a path the projection does not deny is as wrong as a missing one. + await assert.rejects(async () => verifyGrokSlotPreflightReceipt(await fixture("receipt.extra-canary.json"), projected), /canaries/u); assert.throws(() => verifyGrokSlotPreflightReceipt(valid, { ...projected, limits: { ...projected.limits, maxTokens: 1 } }), /projection_sha256/u); assert.throws(() => verifyGrokSlotPreflightReceipt({ ...valid, sandbox_profile_sha256: "1".repeat(64) }, projected), /sandbox_profile_sha256/u); assert.throws(() => verifyGrokSlotPreflightReceipt({ ...valid, slot: 1 }, projected), /slot/u); From a5fd91bfd2b93d22e2f23d184dd1a87db7feee12 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:43:12 +0200 Subject: [PATCH 17/21] feat: bind slot preflight receipts to the projected seccomp profile and bubblewrap runtime --- .../grok-slot-preflight/projection-input.json | 34 ++++++++++++++++--- .../receipt.extra-canary.json | 3 +- .../receipt.missing-canary.json | 3 +- .../receipt.projection-mismatch.json | 1 + .../receipt.readable-canary.json | 3 +- .../receipt.unknown-member.json | 3 +- .../grok-slot-preflight/receipt.valid.v1.json | 3 +- src/runtime/grokBrokerProjection.test.ts | 6 ++-- src/runtime/grokBrokerProjection.ts | 17 ++++++++-- src/runtime/grokSlotPreflightReceipt.test.ts | 4 +++ src/runtime/grokSlotPreflightReceipt.ts | 7 +++- 11 files changed, 69 insertions(+), 15 deletions(-) diff --git a/src/runtime/fixtures/grok-slot-preflight/projection-input.json b/src/runtime/fixtures/grok-slot-preflight/projection-input.json index b75a102..303abd0 100644 --- a/src/runtime/fixtures/grok-slot-preflight/projection-input.json +++ b/src/runtime/fixtures/grok-slot-preflight/projection-input.json @@ -1,9 +1,27 @@ { "config": { "version": "noopolis.daimon.organization-runtime.v2", - "host": { "bindHost": "127.0.0.1", "port": 19700, "controlTokenEnv": "DAIMON_CONTROL_TOKEN" }, + "host": { + "bindHost": "127.0.0.1", + "port": 19700, + "controlTokenEnv": "DAIMON_CONTROL_TOKEN" + }, "agents": [ - { "id": "foreman", "name": "Foreman", "instructions": "Fixture agent.", "workspacePath": "/var/lib/spawnfile/instance/workspace/agents/foreman", "runtimeHomePath": "/var/lib/spawnfile/instance/homes/foreman", "schedule": { "kind": "disabled" }, "engine": { "kind": "grok", "model": "grok-4.6", "reasoningEffort": "low" } } + { + "id": "foreman", + "name": "Foreman", + "instructions": "Fixture agent.", + "workspacePath": "/var/lib/spawnfile/instance/workspace/agents/foreman", + "runtimeHomePath": "/var/lib/spawnfile/instance/homes/foreman", + "schedule": { + "kind": "disabled" + }, + "engine": { + "kind": "grok", + "model": "grok-4.6", + "reasoningEffort": "low" + } + } ] }, "agentId": "foreman", @@ -13,8 +31,16 @@ "workerHomePath": "/var/lib/daimon-workers/2200", "architecture": "arm64", "usageLedgerPath": "/run/daimon-slots/0/usage/usage.jsonl", - "limits": { "maxRequests": 24, "maxTokens": 400000, "timeoutMs": 480000 }, + "limits": { + "maxRequests": 24, + "maxTokens": 400000, + "timeoutMs": 480000 + }, "acceptanceStorePath": "/run/paideia/control", - "denyPaths": ["/run/paideia", "/run/training/inputs"] + "denyPaths": [ + "/run/paideia", + "/run/training/inputs" + ], + "seccompProfileSha256": "7777777777777777777777777777777777777777777777777777777777777777" } } diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json index 719bdc8..f694edb 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json @@ -2,9 +2,10 @@ "version": "noopolis.daimon.grok-slot-preflight.v1", "slot": 0, "worker_uid": 2200, - "projection_sha256": "b41a6952dfe3319f014866e7468df69ca2e77cfd28d781ac39cfdf0d3d587bd8", + "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "sandbox_runtime": "bubblewrap", "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", "canaries": [ { diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json index 6671c48..974b321 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json @@ -2,9 +2,10 @@ "version": "noopolis.daimon.grok-slot-preflight.v1", "slot": 0, "worker_uid": 2200, - "projection_sha256": "b41a6952dfe3319f014866e7468df69ca2e77cfd28d781ac39cfdf0d3d587bd8", + "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "sandbox_runtime": "bubblewrap", "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", "canaries": [ { diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.projection-mismatch.json b/src/runtime/fixtures/grok-slot-preflight/receipt.projection-mismatch.json index cdcb916..c8729e3 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.projection-mismatch.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.projection-mismatch.json @@ -5,6 +5,7 @@ "projection_sha256": "0000000000000000000000000000000000000000000000000000000000000000", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "sandbox_runtime": "bubblewrap", "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", "canaries": [ { diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json index 34d05ff..8f5d407 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json @@ -2,9 +2,10 @@ "version": "noopolis.daimon.grok-slot-preflight.v1", "slot": 0, "worker_uid": 2200, - "projection_sha256": "b41a6952dfe3319f014866e7468df69ca2e77cfd28d781ac39cfdf0d3d587bd8", + "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "sandbox_runtime": "bubblewrap", "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", "canaries": [ { diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json b/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json index c69658a..2e50e6a 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json @@ -2,9 +2,10 @@ "version": "noopolis.daimon.grok-slot-preflight.v1", "slot": 0, "worker_uid": 2200, - "projection_sha256": "b41a6952dfe3319f014866e7468df69ca2e77cfd28d781ac39cfdf0d3d587bd8", + "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "sandbox_runtime": "bubblewrap", "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", "canaries": [ { diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v1.json b/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v1.json index 46954bc..3ccf85c 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v1.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v1.json @@ -2,9 +2,10 @@ "version": "noopolis.daimon.grok-slot-preflight.v1", "slot": 0, "worker_uid": 2200, - "projection_sha256": "b41a6952dfe3319f014866e7468df69ca2e77cfd28d781ac39cfdf0d3d587bd8", + "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "sandbox_runtime": "bubblewrap", "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", "canaries": [ { diff --git a/src/runtime/grokBrokerProjection.test.ts b/src/runtime/grokBrokerProjection.test.ts index aa8689a..10794bb 100644 --- a/src/runtime/grokBrokerProjection.test.ts +++ b/src/runtime/grokBrokerProjection.test.ts @@ -11,7 +11,7 @@ const agent = (id: string, engine: Record) => ({ id, name: id, const config = { version: "noopolis.daimon.organization-runtime.v2", host: { bindHost: "127.0.0.1", port: 19700, controlTokenEnv: "UNIT_CONTROL_TOKEN" }, agents: [agent("foreman", { kind: "grok", model: "grok-4.6", reasoningEffort: "low" }), agent("peer", { kind: "codex" })] }; const options = { slot: 0, workerUid: 2_200, workerHomePath: "/var/lib/daimon-workers/2200", architecture: "arm64", usageLedgerPath: "/run/slots/0/usage/usage.jsonl", - limits: { maxRequests: 24, maxTokens: 400_000, timeoutMs: 480_000 }, acceptanceStorePath: "/run/paideia/control", denyPaths: ["/run/paideia", "/run/training/inputs"] } as const; + limits: { maxRequests: 24, maxTokens: 400_000, timeoutMs: 480_000 }, acceptanceStorePath: "/run/paideia/control", denyPaths: ["/run/paideia", "/run/training/inputs"], seccompProfileSha256: "7".repeat(64) } as const; test("the projection is Daimon's own renderers and collectors, fully declared and deterministic", () => { const projection = resolveOrganizationGrokBrokerProjection(config, "foreman", options); @@ -23,7 +23,8 @@ test("the projection is Daimon's own renderers and collectors, fully declared an workerConfigSha256: grokBrokerWorkerConfigSha256({ model: "grok-4.6", reasoningEffort: "low" }), systemPromptSha256: GROK_ENGINE_BROKER.worker.systemPromptSha256, grokCliVersion: "1.0.34", grokExecutableSha256: GROK_ENGINE_BROKER.grokCliArtifacts.arm64.sha256, nativeAbiVersion: GROK_ENGINE_BROKER.nativeAbiVersion, model: "grok-4.6", reasoningEffort: "low", limits: options.limits, usageLedgerPath: options.usageLedgerPath, - attestation: { platform: "linux/landlock", enforced: true, restrictNetwork: true, profileName: "daimon-strict", eventsPath: "/var/lib/daimon-workers/2200/.grok/sessions/sandbox-events.jsonl" } + seccompProfileSha256: "7".repeat(64), + attestation: { platform: "linux/landlock", enforced: true, restrictNetwork: true, profileName: "daimon-strict", sandboxRuntime: "bubblewrap", eventsPath: "/var/lib/daimon-workers/2200/.grok/sessions/sandbox-events.jsonl" } }); assert.equal(grokBrokerProjectionSha256(resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, denyPaths: [...options.denyPaths].reverse() })), grokBrokerProjectionSha256(projection)); assert.match(grokBrokerProjectionSha256(projection), /^[a-f0-9]{64}$/u); @@ -32,6 +33,7 @@ test("the projection is Daimon's own renderers and collectors, fully declared an test("the projection refuses undeclared models, non-Grok agents, and a profile digest it did not render", () => { assert.throws(() => resolveOrganizationGrokBrokerProjection({ ...config, agents: [agent("foreman", { kind: "grok" })] }, "foreman", options), /declared model/u); assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "peer", options), /known Grok agent/u); + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, seccompProfileSha256: "not-a-digest" }), /seccomp profile sha256/u); assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "missing", options), /known Grok agent/u); // Mutation guard: skipping the digest comparison accepts a weaker profile's digest. assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, profileSha256: grokWorkerSandboxProfileSha256([]) }), /profile digest mismatch/u); diff --git a/src/runtime/grokBrokerProjection.ts b/src/runtime/grokBrokerProjection.ts index ac1e5f6..4052e41 100644 --- a/src/runtime/grokBrokerProjection.ts +++ b/src/runtime/grokBrokerProjection.ts @@ -39,7 +39,9 @@ export type OrganizationGrokBrokerProjection = Readonly<{ reasoningEffort: GrokBrokerReasoningEffort; limits: EngineBrokerTurnLimits; usageLedgerPath: string; - attestation: Readonly<{ platform: "linux/landlock"; enforced: true; restrictNetwork: true; profileName: typeof GROK_WORKER_SANDBOX_PROFILE; eventsPath: string }>; + /** The container seccomp profile the worker must run under (the pinned default-plus-userns profile bubblewrap needs). */ + seccompProfileSha256: string; + attestation: Readonly<{ platform: "linux/landlock"; enforced: true; restrictNetwork: true; profileName: typeof GROK_WORKER_SANDBOX_PROFILE; sandboxRuntime: "bubblewrap"; eventsPath: string }>; }>; export type OrganizationGrokBrokerProjectionOptions = Readonly<{ @@ -55,6 +57,8 @@ export type OrganizationGrokBrokerProjectionOptions = Readonly<{ acceptanceStorePath: string; /** Evaluator and host-bind paths the deployment must keep from the worker (R4). */ denyPaths?: readonly string[]; + /** sha256 of the seccomp profile bytes the deployment runs the worker under. */ + seccompProfileSha256: string; /** When the caller already holds a rendered profile digest, it must equal Daimon's. */ profileSha256?: string; }>; @@ -62,6 +66,12 @@ export type OrganizationGrokBrokerProjectionOptions = Readonly<{ /** * Resolve the public Grok broker projection for one agent. * + * Paths are taken as given, never resolved: the caller (Spawnfile provisioning) + * must supply canonical, non-symlink paths — the fixed tmpfs/workspace roots it + * creates — and its provisioning must verify they are not symlinks before a + * slot is used; the broker's own attestation re-checks the worker home at + * every turn. + * * Deterministic and I/O-free on purpose: its digest * ({@link grokBrokerProjectionSha256}) is what the slot preflight receipt * binds, so the supervisor that writes the receipt and the evaluator that reads @@ -81,6 +91,7 @@ export function resolveOrganizationGrokBrokerProjection(config: unknown, agentId for (const [label, value] of [["workerHomePath", options.workerHomePath], ["acceptanceStorePath", options.acceptanceStorePath]] as const) { if (!path.posix.isAbsolute(value) || path.posix.normalize(value) !== value || value === "/" || value.endsWith("/")) throw new Error(`Grok broker projection requires a canonical absolute ${label}`); } + if (!/^[a-f0-9]{64}$/u.test(options.seccompProfileSha256)) throw new Error("Grok broker projection requires a seccomp profile sha256"); const model = agent.engine.model as GrokBrokerModel, reasoningEffort = agent.engine.reasoningEffort as GrokBrokerReasoningEffort; const denyPaths = [...new Set([...grokSandboxProtectedPaths(agent.id, parsed.agents, [options.acceptanceStorePath]), ...(options.denyPaths ?? [])])].sort(); renderGrokWorkerSandboxProfile(denyPaths); @@ -96,8 +107,8 @@ export function resolveOrganizationGrokBrokerProjection(config: unknown, agentId version: GROK_BROKER_PROJECTION_VERSION, agentId, workspacePath: agent.workspacePath, runtimeHomePath: agent.runtimeHomePath, workerUid: options.workerUid, slot: options.slot, profilePath, profileSha256, denyPaths, workerConfigSha256, systemPromptSha256: GROK_ENGINE_BROKER.worker.systemPromptSha256, grokCliVersion: GROK_ENGINE_BROKER.grokCliVersion, grokExecutableSha256: artifact.sha256, - nativeAbiVersion: GROK_ENGINE_BROKER.nativeAbiVersion, model, reasoningEffort, limits: parseEngineBrokerTurnLimits(options.limits), usageLedgerPath: options.usageLedgerPath, - attestation: { platform: "linux/landlock", enforced: true, restrictNetwork: true, profileName: GROK_WORKER_SANDBOX_PROFILE, eventsPath: grokWorkerEventsPathFor(profilePath) } + nativeAbiVersion: GROK_ENGINE_BROKER.nativeAbiVersion, model, reasoningEffort, limits: parseEngineBrokerTurnLimits(options.limits), usageLedgerPath: options.usageLedgerPath, seccompProfileSha256: options.seccompProfileSha256, + attestation: { platform: "linux/landlock", enforced: true, restrictNetwork: true, profileName: GROK_WORKER_SANDBOX_PROFILE, sandboxRuntime: "bubblewrap", eventsPath: grokWorkerEventsPathFor(profilePath) } }; // The registration this projection implies must itself be a valid v2 service.json entry. grokBrokerServiceRegistrationFor(projection); diff --git a/src/runtime/grokSlotPreflightReceipt.test.ts b/src/runtime/grokSlotPreflightReceipt.test.ts index d6fb4cd..b141060 100644 --- a/src/runtime/grokSlotPreflightReceipt.test.ts +++ b/src/runtime/grokSlotPreflightReceipt.test.ts @@ -43,4 +43,8 @@ test("a receipt for a different projection, slot, profile or deny set is refused assert.throws(() => verifyGrokSlotPreflightReceipt(valid, { ...projected, limits: { ...projected.limits, maxTokens: 1 } }), /projection_sha256/u); assert.throws(() => verifyGrokSlotPreflightReceipt({ ...valid, sandbox_profile_sha256: "1".repeat(64) }, projected), /sandbox_profile_sha256/u); assert.throws(() => verifyGrokSlotPreflightReceipt({ ...valid, slot: 1 }, projected), /slot/u); + // Mutation guard: never comparing the seccomp digest accepts a receipt taken under another profile. + assert.throws(() => verifyGrokSlotPreflightReceipt({ ...valid, seccomp_profile_sha256: "8".repeat(64) }, projected), /seccomp_profile_sha256/u); + assert.throws(() => parseGrokSlotPreflightReceipt({ ...valid, sandbox_runtime: "none" }), /invalid Grok slot preflight receipt/u); + assert.throws(() => parseGrokSlotPreflightReceipt((({ sandbox_runtime: _omit, ...rest }) => rest)(valid)), /invalid Grok slot preflight receipt/u); }); diff --git a/src/runtime/grokSlotPreflightReceipt.ts b/src/runtime/grokSlotPreflightReceipt.ts index 3bed48d..7658581 100644 --- a/src/runtime/grokSlotPreflightReceipt.ts +++ b/src/runtime/grokSlotPreflightReceipt.ts @@ -38,6 +38,8 @@ export const grokSlotPreflightReceiptSchema = z.strictObject({ sandbox_profile_sha256: sha256, /** The container seccomp profile the worker ran under. */ seccomp_profile_sha256: sha256, + /** Grok 1.0.34 runs every profile inside bubblewrap; the supervisor observed it present and working. */ + sandbox_runtime: z.literal("bubblewrap"), grok_executable_sha256: sha256, canaries: z.array(grokSlotPreflightCanarySchema).min(1).max(256), created_at: z.string().regex(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/u).refine((value) => !Number.isNaN(Date.parse(value)) && new Date(value).toISOString() === value, "exact RFC3339 timestamp") @@ -58,7 +60,8 @@ export function parseGrokSlotPreflightReceipt(value: unknown): GrokSlotPreflight /** * Parse a receipt and require that it proves *this* projection's slot: same * digest, slot, worker uid, profile and executable, and a denied canary for - * exactly every projected deny path (no more, no fewer). + * exactly every projected deny path (no more, no fewer), under the projected + * seccomp profile and sandbox runtime. */ export function verifyGrokSlotPreflightReceipt(value: unknown, projection: OrganizationGrokBrokerProjection): GrokSlotPreflightReceipt { const receipt = parseGrokSlotPreflightReceipt(value); @@ -68,6 +71,8 @@ export function verifyGrokSlotPreflightReceipt(value: unknown, projection: Organ if (receipt.worker_uid !== projection.workerUid) mismatch("worker_uid"); if (receipt.sandbox_profile_sha256 !== projection.profileSha256) mismatch("sandbox_profile_sha256"); if (receipt.grok_executable_sha256 !== projection.grokExecutableSha256) mismatch("grok_executable_sha256"); + if (receipt.seccomp_profile_sha256 !== projection.seccompProfileSha256) mismatch("seccomp_profile_sha256"); + if (receipt.sandbox_runtime !== projection.attestation.sandboxRuntime) mismatch("sandbox_runtime"); const denied = receipt.canaries.map((canary) => canary.path).sort(); if (denied.length !== projection.denyPaths.length || denied.some((entry, index) => entry !== projection.denyPaths[index])) mismatch("canaries"); return receipt; From 3a1fac813d879012a21460e2933fd68d8eec8cc6 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:43:12 +0200 Subject: [PATCH 18/21] docs: record the Grok in-flight gate, usage estimates, sealed ledger bytes and projection path contract --- src/runtime/AGENTS.md | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index ca8eb55..472352f 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -37,9 +37,14 @@ upstream-reported running total (prompt tokens *including* cached, plus completion) has reached `maxTokens` — HTTP 429, and the tripped limit aborts the worker through the ordinary cancel/kill path. The token ceiling is checked between requests, so a turn overshoots it by at most the last admitted -request; if an upstream body carries no `usage`, only `maxRequests` and -`timeoutMs` bound that turn mid-flight. A broker timer also trips `timeout` -for a worker that is mid-request. Limits come from `service.json` v2 +request. That bound holds only because a turn has at most one upstream request +in flight: an overlapping request is refused (429, uncounted), and Grok's loop +is sequential in every live capture. A per-request usage block above +`turnLimits.requestUsageMaxTokens` (500k) is invalid, and a response without +valid usage is charged `ceil(bodyBytes/2) + 4096` tokens (rows say +`usage_source: "estimated"`, usage rows `estimated_requests`), so a missing +`usage` never disables the ceiling. A broker timer also trips `timeout` for a +worker that is mid-request, and any trip aborts the in-flight upstream call. Limits come from `service.json` v2 (`engineBrokerServiceConfig.ts`; v1 gets `GROK_ENGINE_BROKER.turnLimits.v1Defaults`) and a wake may only lower them: a raise is refused as `invalid_request`, never clamped. @@ -49,9 +54,13 @@ seals every terminal turn — completed, failed, limit, cancelled — through `finishBrokerTurnWithUsage` (`grokEngineBrokerMetering.ts`): the turn registry record v2 stores the control-protocol v2 terminal response *with* its numeric-only accounting (`usage`, `outcome`, declared `model`, `requests`, -closed `limitReason`), and only then are ledger rows appended. A replay -returns the sealed accounting and never meters again; v1 records still replay -(upgraded with `usage: null`). Completed usage is the terminal `result.usage`; +closed `limitReason`) *and the exact ledger bytes it owes*, and only then are +those bytes appended. A replay returns the sealed accounting and never meters +again; it only appends the sealed bytes when the ledger has no row for that +`turn` (a crash between seal and append). The window not closed: a crash +before the record's rename seals the turn `failed` with `usage: null` on the +next boot. Once a completed record is sealed, nothing after it can re-seal the +turn as failed. v1 records still replay (upgraded with `usage: null`). Completed usage is the terminal `result.usage`; a failed turn's partial usage is its per-request stream frames (`../pi/grokStreamUsage.ts`) when output arrived, else the upstream usage the proxy saw. Usage rows carry `turn` (the idempotency key readers dedupe on — @@ -67,7 +76,10 @@ Grok agent's slot (`noopolis.daimon.grok-broker-projection.v1`): Daimon's own deny collectors plus the caller's evaluator paths, profile/config/prompt digests, pinned executable, model, limits and ledger. A Grok agent must declare `model` and `reasoningEffort` for it; nothing is defaulted, and a supplied -profile digest that differs is refused. `grokSlotPreflightReceipt.ts` is the +profile digest that differs is refused. Paths are never resolved: Spawnfile +must supply canonical non-symlink paths (its fixed tmpfs and workspace roots) +and verify that during provisioning. The projection also carries the seccomp +profile digest and the `bubblewrap` sandbox runtime a receipt must match. `grokSlotPreflightReceipt.ts` is the zod schema a root slot supervisor's receipt must satisfy (`noopolis.daimon.grok-slot-preflight.v1`, fixtures under `fixtures/grok-slot-preflight/`); `verifyGrokSlotPreflightReceipt` binds it to From 97784b40d82e367a74d32ac73adb8f88f99c4539 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:51:23 +0200 Subject: [PATCH 19/21] fix: treat a Grok turn record as sealed once renamed and report a failed directory sync instead of rejecting --- src/runtime/engineBrokerTurnRegistry.ts | 21 ++++++++++++++++----- src/runtime/grokEngineBrokerUsage.test.ts | 23 ++++++++++++++++++++--- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/src/runtime/engineBrokerTurnRegistry.ts b/src/runtime/engineBrokerTurnRegistry.ts index d59bab7..43ff351 100644 --- a/src/runtime/engineBrokerTurnRegistry.ts +++ b/src/runtime/engineBrokerTurnRegistry.ts @@ -23,12 +23,13 @@ type Observed = Readonly<{ version: typeof ENGINE_BROKER_TURN_RECORD_V1 | typeof * only on the path that returned `"start"`. */ export class EngineBrokerTurnRegistry { - constructor(private readonly root: string,private readonly bootId:string=randomUUID()) {} + /** `syncDirectoryOf` is injectable only so the post-publish failure path can be exercised under test. */ + constructor(private readonly root: string,private readonly bootId:string=randomUUID(),private readonly syncDirectoryOf:(directory:string)=>Promise=syncDirectory) {} /** `model` is the registration's declared model, used only to upgrade a v1 record on replay. */ async begin(request: Start, model: GrokBrokerModel): Promise<"start" | { replay: Terminal; ledger: BrokerTurnLedgerLines }> { await mkdir(this.root, { recursive: true, mode: 0o700 }); const file = path.join(this.root, safe(request.turnId)); const expected = digest(request); - try { const handle = await open(file, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); try { await handle.writeFile(`${JSON.stringify({ version: ENGINE_BROKER_TURN_RECORD_V2, digest: expected, state: "active",bootId:this.bootId })}\n`); await handle.sync(); } finally { await handle.close(); } await syncDirectory(this.root); return "start"; } + try { const handle = await open(file, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); try { await handle.writeFile(`${JSON.stringify({ version: ENGINE_BROKER_TURN_RECORD_V2, digest: expected, state: "active",bootId:this.bootId })}\n`); await handle.sync(); } finally { await handle.close(); } await this.syncDirectoryOf(this.root); return "start"; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw new Error("broker turn registry unavailable"); } const observed = parseEngineBrokerTurnRecord(await readFile(file, "utf8"), model); if (observed.digest !== expected) throw new Error("broker turn conflict"); @@ -36,12 +37,22 @@ export class EngineBrokerTurnRegistry { if(observed.state==="active"&&observed.bootId!==this.bootId){const response={version:request.version,kind:"failed",requestId:request.requestId,turnId:request.turnId,code:"engine_failed",outcome:"failed",usage:null,model,requests:0,limitReason:"none"} as const;await this.finish(request,response);return {replay:response,ledger:EMPTY_BROKER_TURN_LEDGER};} throw new Error("broker turn already active"); } - /** `ledger` is the exact ledger bytes this turn owes, sealed with it so a replay can finish an interrupted append. */ - async finish(request: Start, response: Terminal, ledger: BrokerTurnLedgerLines = EMPTY_BROKER_TURN_LEDGER): Promise { + /** + * `ledger` is the exact ledger bytes this turn owes, sealed with it so a replay can finish an interrupted append. + * + * The rename is the publish point. A failure before it rejects (nothing was + * published); a failure after it — the directory fsync — must not, because + * the terminal record is already the visible truth and a caller that saw a + * rejection would believe the turn unsealed and write a contradicting + * record over it. That durability gap is reported as `directorySynced: false` + * instead: the record is published but may not survive a power loss. + */ + async finish(request: Start, response: Terminal, ledger: BrokerTurnLedgerLines = EMPTY_BROKER_TURN_LEDGER): Promise> { const file = path.join(this.root, safe(request.turnId)); const temporary = `${file}.${randomUUID()}.tmp`; const handle = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); try { await handle.writeFile(`${JSON.stringify({ version: ENGINE_BROKER_TURN_RECORD_V2, digest: digest(request), state: "terminal",bootId:this.bootId, response, ledger })}\n`); await handle.sync(); } finally { await handle.close(); } - try { await rename(temporary, file); await syncDirectory(this.root); } finally { await unlink(temporary).catch(() => undefined); } + try { await rename(temporary, file); } finally { await unlink(temporary).catch(() => undefined); } + try { await this.syncDirectoryOf(this.root); return { directorySynced: true }; } catch { return { directorySynced: false }; } } } diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index 2c3fbbc..b850069 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -46,7 +46,7 @@ const untilAborted = (signal: AbortSignal): Promise => new Promise((_reso * talks to the proxy exactly as the native worker does (capability bearer, * pinned client version, lean body). Only the launcher and attestation are fakes. */ -const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId: string, worker: Worker, overrides?: Parameters[6], limits?: EngineBrokerServiceRegistration["limits"], turnStore?: string) => ReturnType; usageRows: () => Promise[]>; requestRows: () => Promise[]>; upstreamCalls: () => number; upstreamAborts: () => number }>) => Promise, usageLedgerPath?: string, upstreamDelayMs: (call: number) => number = () => 15): Promise => { +const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId: string, worker: Worker, overrides?: Parameters[6], limits?: EngineBrokerServiceRegistration["limits"], turnStore?: string, syncDirectory?: (directory: string) => Promise) => ReturnType; usageRows: () => Promise[]>; requestRows: () => Promise[]>; upstreamCalls: () => number; upstreamAborts: () => number }>) => Promise, usageLedgerPath?: string, upstreamDelayMs: (call: number) => number = () => 15): Promise => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-usage-")); let calls = 0, aborted = 0; const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async (_request, signal) => { calls++; await new Promise((resolve, reject) => { const timer = setTimeout(resolve, upstreamDelayMs(calls)); signal?.addEventListener("abort", () => { clearTimeout(timer); aborted++; reject(new Error("aborted")); }, { once: true }); }); return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(`data: ${JSON.stringify({ choices: [], usage: upstreamUsage })}\n\ndata: [DONE]\n\n`) }; }, undefined, 0); @@ -55,10 +55,10 @@ const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId try { await body({ root, - turn: (wakeId, worker, overrides, limits = { maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }, turnStore = path.join(root, "turns")) => { + turn: (wakeId, worker, overrides, limits = { maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }, turnStore = path.join(root, "turns"), syncDirectory = undefined) => { const registration: EngineBrokerServiceRegistration = { agentId: "foreman", slot: 0, workerUid: 2_200, workspace: "/workspace", profilePath: "/workers/0/.grok/sandbox.toml", eventsPath: "/workers/0/.grok/sessions/sandbox-events.jsonl", profileSha256: "a".repeat(64), usageLedgerPath: ledger, limits, model: { model: "grok-4.6", reasoningEffort: "low" } }; const deps: GrokEngineBrokerTurnDependencies = { - turns: new EngineBrokerTurnRegistry(turnStore), proxy, credentialStale: () => false, + turns: syncDirectory === undefined ? new EngineBrokerTurnRegistry(turnStore) : new EngineBrokerTurnRegistry(turnStore, undefined, syncDirectory), proxy, credentialStale: () => false, mcp: { register: () => "mcp-capability-0123456789abcdef", revoke: () => undefined }, prepareIsolation: async () => async () => undefined, runNative: async (input: NativeBrokerTurn, signal: AbortSignal) => nativeResult(await worker(() => post(proxy.port, input.providerCapability), signal)) @@ -187,6 +187,23 @@ test("a ledger append that fails after the turn was sealed leaves it completed a }); }); +test("a directory-sync failure after the completed record is published never re-seals the turn as failed", async () => { + await withBroker(async ({ root, turn, usageRows, requestRows }) => { + let syncs = 0; + // Sync 1 is begin()'s active record; sync 2 follows the completed record's rename. + const failAfterPublish = async (): Promise => { syncs += 1; if (syncs === 2) throw new Error("EIO"); }; + // Mutation guard: letting the post-rename failure reject makes the turn's catch write `failed` over the published record. + const result = await turn("wake-10", twoRequests, undefined, undefined, path.join(root, "turns"), failAfterPublish); + assert.equal(result.outcome, "completed"); + assert.equal(syncs, 2); + assert.deepEqual((await usageRows()).map((row) => [row.outcome, row.turn]), [["completed", turnIdFor("foreman", "wake-10")]]); + assert.equal((await requestRows()).length, 2); + const replayed = await turn("wake-10", async () => { throw new Error("a replay runs no worker"); }); + assert.deepEqual(replayed, result); + assert.equal((await usageRows()).length, 1); + }); +}); + test("an unwritable ledger leaves the turn recorded as completed, not failed", async () => { await withBroker(async ({ root, turn }) => { assert.equal((await turn("wake-7", twoRequests)).outcome, "completed"); From 3ab4e4a7d513094cbf6ffd72bd1b0f0ea1d45a87 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:51:23 +0200 Subject: [PATCH 20/21] test: prove concurrent replays of one Grok turn are counted once and require readers to dedupe on turn --- src/runtime/AGENTS.md | 8 ++++++- src/runtime/grokEngineBrokerUsage.test.ts | 29 +++++++++++++++++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 472352f..182a3fa 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -57,7 +57,13 @@ numeric-only accounting (`usage`, `outcome`, declared `model`, `requests`, closed `limitReason`) *and the exact ledger bytes it owes*, and only then are those bytes appended. A replay returns the sealed accounting and never meters again; it only appends the sealed bytes when the ledger has no row for that -`turn` (a crash between seal and append). The window not closed: a crash +`turn` (a crash between seal and append). Two replays of one sealed turn in +the same broker may both append those identical bytes (a second broker cannot +exist: the realm lease is an exclusive lock), so **every ledger consumer — +`wakeFuse.ts`, Spawnfile's reader (P3), Paideia's evidence reader (P4) — MUST +dedupe usage rows by `turn`** (`dedupeTurnUsageRows`). The turn record's rename +is its publish point: a directory-sync failure after it is reported, never +raised, so a published completed turn is never re-sealed as failed. The window not closed: a crash before the record's rename seals the turn `failed` with `usage: null` on the next boot. Once a completed record is sealed, nothing after it can re-seal the turn as failed. v1 records still replay (upgraded with `usage: null`). Completed usage is the terminal `result.usage`; diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index b850069..9964bd9 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; -import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { request as httpRequest } from "node:http"; import os from "node:os"; import path from "node:path"; @@ -13,7 +13,8 @@ import { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; import { EngineBrokerTurnFailure, runGrokEngineBrokerTurn, type GrokEngineBrokerTurnDependencies } from "./grokEngineBrokerTurn.js"; import { TURN_REQUEST_LEDGER_VERSION } from "./turnRequestLedger.js"; -import { TURN_USAGE_LEDGER_VERSION } from "./turnUsageLedger.js"; +import { dedupeTurnUsageRows, TURN_USAGE_LEDGER_VERSION } from "./turnUsageLedger.js"; +import { WakeFuse } from "./wakeFuse.js"; const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); const leanBody = JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: lean }); @@ -204,6 +205,30 @@ test("a directory-sync failure after the completed record is published never re- }); }); +test("two concurrent replays of one sealed turn may both append, and every reader still counts the turn once", async () => { + await withBroker(async ({ root, turn, usageRows }) => { + await turn("wake-11", twoRequests); + const [sealed] = await usageRows(); + await rm(path.join(root, "usage.jsonl")); await rm(path.join(root, "requests.jsonl")); + const noWorker = async (): Promise => { throw new Error("a replay runs no worker"); }; + const replays = await Promise.all([turn("wake-11", noWorker), turn("wake-11", noWorker), turn("wake-11", noWorker)]); + assert.ok(replays.every((replayed) => replayed.outcome === "completed")); + const rows = await usageRows(); + assert.ok(rows.length >= 1 && rows.every((row) => row.turn === sealed!.turn && row.total === sealed!.total), "duplicates, if any, are byte-equal sealed rows"); + // Readers dedupe on `turn`: the ledger helper and the wake fuse's sum both count it once. + assert.deepEqual(dedupeTurnUsageRows(rows).map((row) => row.total), [sealed!.total]); + const fuseDirectory = path.join(root, "fuse"); await mkdir(fuseDirectory); + const fuse = await WakeFuse.open({ organizationKey: "org", now: () => new Date(Date.parse(String(sealed!.at)) - 1), environment: { + DAIMON_WAKE_FUSE_DIRECTORY: fuseDirectory, DAIMON_WAKE_FUSE_EPOCH: "replay", DAIMON_WAKE_FUSE_MAX_WAKES: "10", + DAIMON_WAKE_FUSE_MAX_TOKENS: String(Number(sealed!.total) + 1), DAIMON_TURN_USAGE_LEDGER_PATH: path.join(root, "usage.jsonl") + } }); + // Counted once the turn is below the ceiling by one token; counted twice it would trip. + const concurrentRows = [...rows, ...rows]; + await writeFile(path.join(root, "usage.jsonl"), concurrentRows.map((row) => JSON.stringify(row)).join("\n") + "\n"); + assert.deepEqual(await fuse.admit("foreman", "next"), { state: "admitted" }); + }); +}); + test("an unwritable ledger leaves the turn recorded as completed, not failed", async () => { await withBroker(async ({ root, turn }) => { assert.equal((await turn("wake-7", twoRequests)).outcome, "completed"); From 3174716be9b3f5a5005e57f070aaf3727852d791 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:51:37 +0200 Subject: [PATCH 21/21] fix: charge the estimate when a Grok response's final usage block is invalid --- src/runtime/grokBrokerTurnMeter.test.ts | 3 +++ src/runtime/grokBrokerTurnMeter.ts | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/runtime/grokBrokerTurnMeter.test.ts b/src/runtime/grokBrokerTurnMeter.test.ts index f64911c..0f28a88 100644 --- a/src/runtime/grokBrokerTurnMeter.test.ts +++ b/src/runtime/grokBrokerTurnMeter.test.ts @@ -103,6 +103,9 @@ test("upstream usage parsing takes the last usage block and never zero-fills", ( assert.equal(parseGrokUpstreamUsage(sse({ prompt_tokens: "4", completion_tokens: 1 }), "text/event-stream"), undefined); assert.equal(parseGrokUpstreamUsage(sse({ prompt_tokens: 4, completion_tokens: 1, prompt_tokens_details: { cached_tokens: 9 } }), "text/event-stream"), undefined); assert.equal(parseGrokUpstreamUsage(Buffer.from("not json"), "application/json"), undefined); + // Mutation guard: keeping the earlier valid block under-reports a request whose final usage is implausible. + const twoBlocks = Buffer.from([{ choices: [], usage: { prompt_tokens: 5, completion_tokens: 1 } }, { choices: [], usage: { prompt_tokens: 900_000, completion_tokens: 1 } }].map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("")); + assert.equal(parseGrokUpstreamUsage(twoBlocks, "text/event-stream"), undefined); }); test("at most one upstream request is in flight per turn: an overlapping request is refused, uncounted", async () => { diff --git a/src/runtime/grokBrokerTurnMeter.ts b/src/runtime/grokBrokerTurnMeter.ts index d5b03f4..3c14b7c 100644 --- a/src/runtime/grokBrokerTurnMeter.ts +++ b/src/runtime/grokBrokerTurnMeter.ts @@ -133,8 +133,10 @@ export function parseGrokUpstreamUsage(body: Uint8Array, contentType: string | u let found: EngineBrokerTurnUsage | undefined; for (const candidate of candidates) { if (!isRecord(candidate) || !isRecord(candidate.usage)) continue; - const decoded = decodeOpenAiUsage(candidate.usage); - if (decoded !== undefined) found = decoded; + // Last usage block wins even when invalid: an implausible final report + // must not fall back to an earlier, smaller block (the request is then + // charged the estimate instead). + found = decodeOpenAiUsage(candidate.usage); } return found; }