From e3400f04dff2c16ae10206b95c221252f653e706 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Sun, 13 Sep 2026 21:59:58 +0200 Subject: [PATCH 001/124] fix: preserve effective codex sandbox denials --- README.md | 16 +++ src/pi/AGENTS.md | 11 ++ src/pi/CLAUDE.md | 1 + src/pi/cliEngineSpawn.ts | 11 +- src/pi/codexFilesystemRules.test.ts | 68 ++++++++++++ src/pi/codexFilesystemRules.ts | 29 ++++++ src/runtime/codexSandbox.linux.test.ts | 116 +++++++++++++++++++++ src/runtime/codexSandboxProjection.test.ts | 40 +++++++ src/runtime/codexSandboxProjection.ts | 50 +++++++++ src/runtime/index.ts | 2 + 10 files changed, 339 insertions(+), 5 deletions(-) create mode 100644 src/pi/AGENTS.md create mode 120000 src/pi/CLAUDE.md create mode 100644 src/pi/codexFilesystemRules.test.ts create mode 100644 src/pi/codexFilesystemRules.ts create mode 100644 src/runtime/codexSandbox.linux.test.ts create mode 100644 src/runtime/codexSandboxProjection.test.ts create mode 100644 src/runtime/codexSandboxProjection.ts diff --git a/README.md b/README.md index 60cf74b..ce35a4e 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,22 @@ import { PiHarnessAdapter } from "@noopolis/daimon/pi"; ## Organization-runtime contract +For a strict Codex agent in an already prepared runtime, the public +`resolveOrganizationCodexSandboxProjection(config, agentId, { acceptanceStorePath })` +API returns `noopolis.daimon.codex-sandbox-projection.v1`: canonical workspace/home +paths, the resolved executable, and `sandboxArgs` containing the exact permission +profile rendered for production. Append only `--` and a mechanical command when +checking whether that policy can execute in the caller's container. + +The resolver opens and verifies caller-owned path identities and executes the +Codex version probe. It does not import/read authentication, create an agent, +accept a wake or invoke a model. The acceptance-store path must be the same one +given to the control host. This API projects command permissions, not the complete +cognition invocation: Codex's `sandbox` subcommand lacks the strict configuration +flags accepted by `exec`. A caller must therefore use a fresh empty `HOME` and +`CODEX_HOME` and a restricted environment for its mechanical probe. Do not infer +successful tool use or completed agent work from this projection alone. + `@noopolis/daimon/runtime` exports a standard JSON Schema for structural validation plus the strict, side-effect-free semantic `validateOrganizationRuntimeConfig` / `parseOrganizationRuntimeConfig` API and diff --git a/src/pi/AGENTS.md b/src/pi/AGENTS.md new file mode 100644 index 0000000..eb26811 --- /dev/null +++ b/src/pi/AGENTS.md @@ -0,0 +1,11 @@ +# Pi runtime implementation + +This directory owns the Pi harness and native CLI adaptation. Keep public +runtime contracts independent of Pi types. Files stay below 400 lines and +tests live beside their implementations. + +Codex permission rendering must preserve the effective path permissions while +removing redundant nested denies that cannot be mounted by its Linux sandbox. +An intervening readable path or workspace root makes a deeper deny necessary. +Verify changes with generated production arguments and real local sandbox +commands; a model call is neither required nor permitted for this check. diff --git a/src/pi/CLAUDE.md b/src/pi/CLAUDE.md new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/src/pi/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/src/pi/cliEngineSpawn.ts b/src/pi/cliEngineSpawn.ts index 861cc58..3b4ef99 100644 --- a/src/pi/cliEngineSpawn.ts +++ b/src/pi/cliEngineSpawn.ts @@ -2,6 +2,7 @@ import { spawn, type ChildProcess } from "node:child_process"; import { trackCliChild } from "./cliProcess.js"; import { cliChildEnvironment } from "./cliEnvironment.js"; +import { codexFilesystemRules } from "./codexFilesystemRules.js"; import type { CliEngineOptions, CliSessionInput } from "./cliSession.js"; export const GROK_STRICT_SANDBOX_PROFILE = "strict"; @@ -49,7 +50,7 @@ export const renderCodexArgs = ( "--strict-config", "-c", "web_search=\"disabled\"", "-c", `default_permissions=${JSON.stringify(profileName)}`, - "-c", renderCodexPermissionProfile(profileName, options.codexSandboxProtectedPaths ?? [], options.codexSandboxReadablePaths ?? []), + "-c", renderCodexPermissionProfile(profileName, options.codexSandboxProtectedPaths ?? [], options.codexSandboxReadablePaths ?? [], cwd), "-c", "approval_policy=\"never\"", "-c", `mcp_servers={daimon={url=\"${endpoint}\",enabled=true,default_tools_approval_mode=\"approve\"}}` ]; @@ -66,13 +67,13 @@ export const renderCodexArgs = ( export const renderCodexPermissionProfile = ( profileName: string, protectedPaths: readonly string[], - readablePaths: readonly string[] = [] + readablePaths: readonly string[] = [], + workspacePath?: string ): string => { const filesystem: Record> = { - ":workspace_roots": { ".": "write" } + ":workspace_roots": { ".": "write" }, + ...codexFilesystemRules(protectedPaths, readablePaths, workspacePath) }; - for (const readablePath of readablePaths) filesystem[readablePath] = "read"; - for (const protectedPath of protectedPaths) filesystem[protectedPath] = "deny"; return `permissions=${tomlInline({ [profileName]: { extends: ":workspace", filesystem, diff --git a/src/pi/codexFilesystemRules.test.ts b/src/pi/codexFilesystemRules.test.ts new file mode 100644 index 0000000..dd3dca0 --- /dev/null +++ b/src/pi/codexFilesystemRules.test.ts @@ -0,0 +1,68 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { renderCodexArgs } from "./cliEngineSpawn.js"; +import { codexFilesystemRules } from "./codexFilesystemRules.js"; + +test("prunes redundant nested denies independently of declaration order", () => { + for (const denies of [["/run", "/run/control", "/run/control/acceptance"], ["/run/control/acceptance", "/run/control", "/run"]]) { + assert.deepEqual(codexFilesystemRules(denies, []), { "/run": "deny" }); + } + assert.deepEqual(codexFilesystemRules(["/run", "/runner", "/other/control"], []), { + "/run": "deny", "/runner": "deny", "/other/control": "deny" + }); +}); + +test("preserves a deny below an intervening readable exception", () => { + assert.deepEqual(codexFilesystemRules([ + "/vault", "/vault/shared/auth", "/vault/shared/auth/deeper", "/vault/secret" + ], ["/vault/shared"]), { + "/vault/shared": "read", "/vault": "deny", "/vault/shared/auth": "deny" + }); +}); + +test("normalizes aliases before testing ancestry, including a shared path that escapes /run", () => { + assert.deepEqual(codexFilesystemRules(["/run/", "/run//control/", "/run/../var/lib/private/control/", "/var/lib/private/control"], []), { + "/run": "deny", "/var/lib/private/control": "deny" + }); + assert.deepEqual(codexFilesystemRules(["/vault/", "/vault/open//secret/"], ["/vault/unused/../open/"]), { + "/vault/open": "read", "/vault": "deny", "/vault/open/secret": "deny" + }); + assert.deepEqual(codexFilesystemRules(["/vault", "/vault/work/secret"], [], "/vault//work/"), { + "/vault": "deny", "/vault/work/secret": "deny" + }); +}); + +test("same-path deny wins over a read and does not open a descendant", () => { + assert.deepEqual(codexFilesystemRules(["/vault", "/vault/auth"], ["/vault"]), { "/vault": "deny" }); +}); + +test("implicit workspace write is an intervening exception, including in production argv", () => { + const denies = ["/vault", "/vault/workspace/secret", "/vault/workspace/secret/deeper"]; + assert.deepEqual(codexFilesystemRules(denies, [], "/vault/workspace"), { + "/vault": "deny", "/vault/workspace/secret": "deny" + }); + const args = renderCodexArgs({ codexSandbox: { mode: "workspace-write", networkAccess: false, webSearch: "disabled" }, + codexSandboxProtectedPaths: denies }, "/vault/workspace", "http://127.0.0.1:1/mcp"); + const config = args.find(arg => arg.startsWith("permissions="))!; + assert.ok(config.includes('"/vault/workspace/secret"="deny"')); + assert.ok(!config.includes('"/vault/workspace/secret/deeper"')); +}); + +test("root deny subsumes descendants and a same-path workspace deny remains authoritative", () => { + assert.deepEqual(codexFilesystemRules(["/", "/run"], []), { "/": "deny" }); + assert.deepEqual(codexFilesystemRules(["/workspace", "/workspace/secret"], [], "/workspace"), { "/workspace": "deny" }); + assert.deepEqual(codexFilesystemRules([], [], "/workspace"), {}); +}); + +test("pruning preserves the most specific permission across alternating rules", () => { + const deny = ["/v", "/v/a", "/v/a/read/secret", "/v/a/read/secret/deeper", "/v/work/secret"]; + const read = ["/v/a/read", "/v/a/read/secret/open"]; + const before = { "/v/work": "write", ...Object.fromEntries(read.map(p => [p, "read"])), ...Object.fromEntries(deny.map(p => [p, "deny"])) }; + const after = { "/v/work": "write", ...codexFilesystemRules(deny, read, "/v/work") }; + const permission = (rules: Record, target: string): string | undefined => Object.entries(rules) + .filter(([root]) => target === root || target.startsWith(`${root}/`)).sort(([a], [b]) => b.length - a.length)[0]?.[1]; + for (const entry of [...deny, ...read, "/v/work", "/v/work/secret", "/v/ab", "/other"]) { + for (const target of [entry, `${entry}/file`]) assert.equal(permission(after, target), permission(before, target), target); + } +}); diff --git a/src/pi/codexFilesystemRules.ts b/src/pi/codexFilesystemRules.ts new file mode 100644 index 0000000..510ff78 --- /dev/null +++ b/src/pi/codexFilesystemRules.ts @@ -0,0 +1,29 @@ +import path from "node:path"; + +type Permission = "read" | "write" | "deny"; + +/** Codex mounts deny roots read-only; a second deny below one cannot create parents. */ +export function codexFilesystemRules( + protectedPaths: readonly string[], readablePaths: readonly string[], workspacePath?: string +): Record { + const explicit = new Map(); + // Agent roots are canonical, but shared acceptance paths may contain `..` + // or trailing separators. Compare the same absolute locations Codex uses. + for (const readablePath of readablePaths) explicit.set(path.resolve(readablePath), "read"); + for (const protectedPath of protectedPaths) explicit.set(path.resolve(protectedPath), "deny"); + const effective = new Map([ + ...(workspacePath === undefined ? [] : [[path.resolve(workspacePath), "write"] as const]), + ...explicit + ]); + return Object.fromEntries([...explicit].filter(([target, permission]) => { + if (permission !== "deny") return true; + let ancestor = path.dirname(target); + while (ancestor !== target) { + const inherited = effective.get(ancestor); + if (inherited !== undefined) return inherited !== "deny"; + target = ancestor; + ancestor = path.dirname(target); + } + return true; + })); +} diff --git a/src/runtime/codexSandbox.linux.test.ts b/src/runtime/codexSandbox.linux.test.ts new file mode 100644 index 0000000..e9f3afd --- /dev/null +++ b/src/runtime/codexSandbox.linux.test.ts @@ -0,0 +1,116 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import test from "node:test"; +import { promisify } from "node:util"; + +import { renderCodexArgs } from "../pi/cliEngineSpawn.js"; +import { codexSandboxProtectedPaths, codexSandboxReadablePaths } from "./engineDispatcher.js"; +import type { OrganizationRuntimeAgentConfig } from "./organizationRuntime.js"; + +const image = process.env.DAIMON_CODEX_SANDBOX_TEST_IMAGE; +const script = ` +const fs=require('node:fs'),path=require('node:path'),cp=require('node:child_process'),net=require('node:net'); +const input=JSON.parse(process.argv[1]); +for(const p of [input.workspace,input.home,...input.denied.filter(p=>p!=='/proc'&&p!=='/run').map(p=>p.endsWith('/auth.json')?path.dirname(p):p),...input.readable])fs.mkdirSync(p,{recursive:true,mode:448}); +const canaries=input.denied.filter(p=>p!=='/proc').map(p=>p.endsWith('/auth.json')?p:path.join(p,'canary')); +for(const p of canaries)fs.writeFileSync(p,'non-secret-denied-canary',{mode:384}); +for(const p of canaries)if(fs.readFileSync(p,'utf8')!=='non-secret-denied-canary')throw Error('outside-sandbox canary missing'); +for(const p of input.readable)fs.writeFileSync(path.join(p,'canary'),'readable-canary'); +fs.writeFileSync(path.join(input.workspace,'input'),'workspace-read'); +const server=net.createServer(s=>s.destroy()); +server.listen(0,'127.0.0.1',()=>{ +const command=input.workspaceDenied?\`const fs=require('node:fs'); +for(const p of \${JSON.stringify([input.workspace+'/input',input.workspace+'/secret/canary'])}){let denied=false;try{fs.readFileSync(p);}catch{denied=true;}if(!denied)throw Error('workspace deny lost to implicit write');} +let denied=false;try{fs.writeFileSync(\${JSON.stringify(input.workspace+'/output')},'forbidden');}catch{denied=true;}if(!denied)throw Error('denied workspace writable'); +process.stdout.write('EXACT_WORKSPACE_DENIED');\`:\`const fs=require('node:fs'),net=require('node:net'); +if(fs.readFileSync('input','utf8')!=='workspace-read')throw Error('workspace read failed'); +fs.writeFileSync('output','workspace-write'); +for(const p of \${JSON.stringify(canaries)}){let denied=false;try{fs.readFileSync(p);}catch{denied=true;}if(!denied)throw Error('canary became readable: '+p);} +for(const p of \${JSON.stringify(input.readable)})if(fs.readFileSync(p+'/canary','utf8')!=='readable-canary')throw Error('readable exception missing'); +const socket=net.connect({host:'127.0.0.1',port:\${server.address().port}});let done=false; +socket.once('connect',()=>{done=true;socket.destroy();throw Error('network reached outside sandbox');}); +socket.once('error',()=>{if(!done){done=true;fs.writeFileSync('complete','CODEX_SANDBOX_ENFORCED');}}); +socket.setTimeout(1500,()=>{socket.destroy();if(!done){done=true;throw Error('network probe timed out');}});\`; +const env={PATH:process.env.PATH,HOME:input.home,CODEX_HOME:input.home+'/.codex',TMPDIR:'/tmp',LANG:'C.UTF-8'}; +const executable=input.workspaceDenied?['/bin/sh','-c','cd /tmp && exec /usr/local/bin/node -e "$1"','probe',command]:['/usr/local/bin/node','-e',command]; +cp.execFile('codex',['sandbox','-P','daimon-strict','-C',input.workspace,'-c',input.profile,'--',...executable],{env,timeout:15000,maxBuffer:65536},(error,stdout,stderr)=>{ +server.close(); +if(error){process.stderr.write(stderr);process.exitCode=1;return;} +if(input.workspaceDenied){if(stdout!=='EXACT_WORKSPACE_DENIED'||fs.existsSync(path.join(input.workspace,'output')))throw Error('workspace deny not enforced: '+JSON.stringify({stdout,stderr,outputExists:fs.existsSync(path.join(input.workspace,'output'))}));process.stdout.write('CODEX_SANDBOX_ENFORCED');return;} +if(fs.readFileSync(path.join(input.workspace,'complete'),'utf8')!=='CODEX_SANDBOX_ENFORCED'||fs.readFileSync(path.join(input.workspace,'output'),'utf8')!=='workspace-write')throw Error('missing sandbox side effect'); +process.stdout.write('CODEX_SANDBOX_ENFORCED'); +});});`; + +async function runSandbox(input: { workspace: string; home: string; denied: readonly string[]; readable: readonly string[]; profile: string; workspaceDenied?: boolean }): Promise { + return (await promisify(execFile)("docker", [ + "run", "--rm", "--network", "none", "--read-only", "--user", "501:20", "--cap-drop", "ALL", + "--security-opt", "no-new-privileges", "--security-opt", "seccomp=unconfined", "--security-opt", "apparmor=unconfined", + ...["/tmp", "/run", "/var/lib/daimon", "/var/lib/spawnfile"].flatMap(target => ["--tmpfs", `${target}:rw,uid=501,gid=20,mode=0700`]), + "--entrypoint", "/usr/local/bin/node", image!, "-e", script, JSON.stringify(input) + ], { timeout: 25000, maxBuffer: 131072 })).stdout; +} + +test("actual Codex sandbox executes the production policy with overlapping control denies", { skip: !image, timeout: 60000 }, async () => { + const configFile = process.env.DAIMON_CODEX_SANDBOX_TEST_CONFIG; + const agent: OrganizationRuntimeAgentConfig = configFile + ? (JSON.parse(await readFile(configFile, "utf8")) as { agents: OrganizationRuntimeAgentConfig[] }).agents[0]! + : { id: "agent:current", name: "Current", instructions: "Unused in this mechanical probe.", + workspacePath: "/var/lib/daimon/workspace", runtimeHomePath: "/var/lib/daimon/home", + engine: { kind: "codex", codexSandbox: { mode: "workspace-write", networkAccess: false, webSearch: "disabled" } } }; + const control = "/run/paideia/control"; + const denied = codexSandboxProtectedPaths(agent.id, agent, path.join(agent.runtimeHomePath, ".codex"), [agent], [control]); + const readable = codexSandboxReadablePaths(agent); + const profile = renderCodexArgs({ codexSandbox: agent.engine.codexSandbox, + codexSandboxProtectedPaths: denied, codexSandboxReadablePaths: readable }, agent.workspacePath, "http://127.0.0.1:1/mcp") + .find(arg => arg.startsWith("permissions="))!; + assert.ok(denied.includes("/run") && denied.includes(control)); + assert.ok(!profile.includes(`"${control}"="deny"`)); + const run = (permissionConfig: string): Promise => runSandbox({ workspace: agent.workspacePath, + home: agent.runtimeHomePath, denied, readable, profile: permissionConfig }); + assert.equal(await run(profile), "CODEX_SANDBOX_ENFORCED"); + // Restore the exact redundant deny emitted before the fix. The real sandbox + // must reproduce the setup failure instead of merely executing an empty shell. + await assert.rejects(run(profile.replace('"/run"="deny"', `"/run"="deny","${control}"="deny"`)), /Can't mkdir parents.*Read-only file system/u); +}); + +test("a denied workspace fails readiness when Codex exits zero without executing the command", { skip: !image, timeout: 30000 }, async () => { + const workspace = "/var/lib/daimon/workspace", home = "/var/lib/daimon/home"; + const denied = [workspace, `${workspace}/secret`, "/run", "/proc", `${home}/.codex/auth.json`]; + const profile = renderCodexArgs({ codexSandbox: { mode: "workspace-write", networkAccess: false, webSearch: "disabled" }, + codexSandboxProtectedPaths: denied }, workspace, "http://127.0.0.1:1/mcp").find(arg => arg.startsWith("permissions="))!; + assert.ok(!profile.includes(`"${workspace}/secret"="deny"`)); + // Codex 0.142.3 executes neither Node nor the shell wrapper in this geometry. + // Exit zero is not readiness: the missing sentinel must reject it. + await assert.rejects(runSandbox({ workspace, home, denied, readable: [], profile, workspaceDenied: true }), + /workspace deny not enforced: \{"stdout":"","stderr":"","outputExists":false\}/u); + await assert.rejects(runSandbox({ workspace, home, denied, readable: [], workspaceDenied: true, + profile: profile.replace(`,"${workspace}"="deny"`, "") }), /workspace deny lost to implicit write/u); +}); + +test("a shared protected path containing .. cannot inherit the wrong ancestor deny", { skip: !image, timeout: 30000 }, async () => { + const workspace = "/var/lib/daimon/workspace", home = "/var/lib/daimon/home"; + const denied = ["/run", "/run/../var/lib/daimon/control/", "/proc", `${home}/.codex/auth.json`]; + const profile = renderCodexArgs({ codexSandbox: { mode: "workspace-write", networkAccess: false, webSearch: "disabled" }, + codexSandboxProtectedPaths: denied }, workspace, "http://127.0.0.1:1/mcp").find(arg => arg.startsWith("permissions="))!; + assert.ok(profile.includes('"/var/lib/daimon/control"="deny"')); + assert.equal(await runSandbox({ workspace, home, denied, readable: [], profile }), "CODEX_SANDBOX_ENFORCED"); + await assert.rejects(runSandbox({ workspace, home, denied, readable: [], + profile: profile.replace(',"/var/lib/daimon/control"="deny"', "") }), /canary became readable/u); +}); + +test("unsupported nested exception geometry fails closed instead of dropping its protected descendant", { skip: !image, timeout: 30000 }, async () => { + const workspace = "/var/lib/daimon/workspace", home = "/var/lib/daimon/home", vault = "/var/lib/daimon/vault"; + const denied = ["/run", "/proc", `${home}/.codex/auth.json`, `${home}/.daimon-inbound`, vault, `${vault}/open/secret`]; + const readable = [`${vault}/open`]; + const profile = renderCodexArgs({ codexSandbox: { mode: "workspace-write", networkAccess: false, webSearch: "disabled" }, + codexSandboxProtectedPaths: denied, codexSandboxReadablePaths: readable }, workspace, "http://127.0.0.1:1/mcp") + .find(arg => arg.startsWith("permissions="))!; + assert.ok(profile.includes(`"${vault}/open/secret"="deny"`)); + // Codex 0.142.3 cannot mount this nonredundant geometry. Keep the deny and + // require the actual-profile readiness probe to refuse it before cognition. + await assert.rejects(runSandbox({ workspace, home, denied, readable, profile }), /Can't mkdir parents.*Read-only file system/u); + await assert.rejects(runSandbox({ workspace, home, denied, readable, + profile: profile.replace(`,"${vault}/open/secret"="deny"`, "") }), /canary became readable/u); +}); diff --git a/src/runtime/codexSandboxProjection.test.ts b/src/runtime/codexSandboxProjection.test.ts new file mode 100644 index 0000000..b3ba256 --- /dev/null +++ b/src/runtime/codexSandboxProjection.test.ts @@ -0,0 +1,40 @@ +import assert from "node:assert/strict"; +import { chmod, mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { resolveOrganizationCodexSandboxProjection } from "./codexSandboxProjection.js"; +import { renderCodexArgs } from "../pi/cliEngineSpawn.js"; +import { codexSandboxProtectedPaths, codexSandboxReadablePaths } from "./engineDispatcher.js"; + +test("public projection uses actual canonical policy and never reads auth or runs cognition", async () => { + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "daimon-projection-"))); + const priorPath = process.env.PATH; + const agent = { id: "agent:writer", name: "Writer", instructions: "Unused", workspacePath: path.join(root, "workspace"), + runtimeHomePath: path.join(root, "home"), schedule: { kind: "disabled" }, engine: { kind: "codex", codexSandbox: { mode: "workspace-write", networkAccess: false, webSearch: "disabled" } } } as const; + const config = { version: "noopolis.daimon.organization-runtime.v2", host: { bindHost: "127.0.0.1", port: 19700, controlTokenEnv: "UNIT_CONTROL_TOKEN" }, agents: [agent] }; + try { + for (const directory of [agent.workspacePath, agent.runtimeHomePath]) await mkdir(directory, { mode: 0o700 }); + const executable = path.join(root, "codex"), log = path.join(root, "calls.jsonl"); + await writeFile(executable, `#!${process.execPath}\nrequire('node:fs').appendFileSync(${JSON.stringify(log)},JSON.stringify(process.argv.slice(2))+'\\n');if(process.argv[2]!=='--version')process.exit(9);process.stdout.write('unit-codex-version');`, { mode: 0o700 }); + process.env.PATH = `${root}${path.delimiter}${priorPath ?? ""}`; + const projection = await resolveOrganizationCodexSandboxProjection(config, agent.id, { acceptanceStorePath: "/run/paideia/control" }); + const expected = renderCodexArgs({ codexSandbox: agent.engine.codexSandbox, + codexSandboxProtectedPaths: codexSandboxProtectedPaths(agent.id, agent, path.join(agent.runtimeHomePath, ".codex"), [agent], ["/run/paideia/control"]), + codexSandboxReadablePaths: codexSandboxReadablePaths(agent) }, agent.workspacePath, undefined).find(arg => arg.startsWith("permissions=")); + assert.equal(projection.permissionConfig, expected); + assert.equal(projection.executablePath, executable); + assert.equal(projection.engineHomePath, path.join(agent.runtimeHomePath, ".codex")); + assert.deepEqual(projection.sandboxArgs, ["sandbox", "-P", "daimon-strict", "-C", agent.workspacePath, "-c", expected]); + assert.ok(!projection.permissionConfig.includes('"/run/paideia/control"="deny"')); + assert.deepEqual((await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line)), [["--version"], ["--version"]]); + await assert.rejects(readFile(path.join(agent.runtimeHomePath, ".codex", "auth.json")), /ENOENT/); + await assert.rejects(resolveOrganizationCodexSandboxProjection(config, "missing", { acceptanceStorePath: "/run/control" }), /known strict/); + await assert.rejects(resolveOrganizationCodexSandboxProjection({ ...config, agents: [{ ...agent, engine: { kind: "grok" } }] }, agent.id, { acceptanceStorePath: "/run/control" }), /known strict/); + await assert.rejects(resolveOrganizationCodexSandboxProjection({ ...config, agents: [{ ...agent, engine: { kind: "codex" } }] }, agent.id, { acceptanceStorePath: "/run/control" }), /known strict/); + await assert.rejects(resolveOrganizationCodexSandboxProjection(config, agent.id, { acceptanceStorePath: "relative" }), /absolute/); + await chmod(executable, 0o600); + process.env.PATH = root; + await assert.rejects(resolveOrganizationCodexSandboxProjection(config, agent.id, { acceptanceStorePath: "/run/control" }), /unavailable/); + } finally { if (priorPath === undefined) delete process.env.PATH; else process.env.PATH = priorPath; await rm(root, { recursive: true, force: true }); } +}); diff --git a/src/runtime/codexSandboxProjection.ts b/src/runtime/codexSandboxProjection.ts new file mode 100644 index 0000000..b8d62b8 --- /dev/null +++ b/src/runtime/codexSandboxProjection.ts @@ -0,0 +1,50 @@ +import path from "node:path"; +import { renderCodexArgs } from "../pi/cliEngineSpawn.js"; +import { codexSandboxProtectedPaths, codexSandboxReadablePaths } from "./engineDispatcher.js"; +import { engineHomeName, prepareEngineExecutable } from "./engineReadiness.js"; +import { parseOrganizationRuntimeConfig } from "./organizationRuntime.js"; +import { prepareOrganizationRuntimePaths } from "./physicalReadiness.js"; + +export const CODEX_SANDBOX_PROJECTION_VERSION = "noopolis.daimon.codex-sandbox-projection.v1"; +export type OrganizationCodexSandboxProjection = Readonly<{ + version: typeof CODEX_SANDBOX_PROJECTION_VERSION; + agentId: string; + workspacePath: string; + runtimeHomePath: string; + engineHomePath: string; + executablePath: string; + profileName: string; + permissionConfig: string; + sandboxArgs: readonly string[]; +}>; + +/** Resolves the production command policy without importing auth or accepting a wake. */ +export async function resolveOrganizationCodexSandboxProjection(config: unknown, agentId: string, + options: Readonly<{ acceptanceStorePath: string }>): Promise { + const parsed = parseOrganizationRuntimeConfig(config); + const selected = parsed.agents.find(agent => agent.id === agentId); + if (!selected || selected.engine.kind !== "codex" || selected.engine.codexSandbox === undefined) { + throw new Error("Sandbox projection requires a known strict Codex agent"); + } + if (!path.isAbsolute(options.acceptanceStorePath)) throw new Error("Sandbox projection requires an absolute acceptance store path"); + const authority = await prepareOrganizationRuntimePaths(parsed.agents); + try { + const paths = authority.forAgent(selected); + await paths.verify(); + const agent = { ...selected, workspacePath: paths.workspacePath, runtimeHomePath: paths.runtimeHomePath }; + const engineHomePath = path.join(agent.runtimeHomePath, engineHomeName("codex")); + const executable = await prepareEngineExecutable(agent.id, "codex"); + // These are the same collectors, canonical roots and renderer used by + // startOrganizationRuntimeEngine and its strict production invocation. + const args = renderCodexArgs({ codexSandbox: agent.engine.codexSandbox, + codexSandboxProtectedPaths: codexSandboxProtectedPaths(agent.id, agent, engineHomePath, parsed.agents, [options.acceptanceStorePath]), + codexSandboxReadablePaths: codexSandboxReadablePaths(agent) }, agent.workspacePath, undefined); + const permissionConfig = args.find(arg => arg.startsWith("permissions="))!; + const profileName = JSON.parse(args.find(arg => arg.startsWith("default_permissions="))!.slice("default_permissions=".length)) as string; + await paths.verify(); + await executable.verify(); + return { version: CODEX_SANDBOX_PROJECTION_VERSION, agentId, workspacePath: agent.workspacePath, + runtimeHomePath: agent.runtimeHomePath, engineHomePath, executablePath: executable.executablePath, profileName, permissionConfig, + sandboxArgs: ["sandbox", "-P", profileName, "-C", agent.workspacePath, "-c", permissionConfig] }; + } finally { await authority.close(); } +} diff --git a/src/runtime/index.ts b/src/runtime/index.ts index 3111163..b72ee90 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -3,6 +3,8 @@ export * from "./contractManifest.js"; export * from "./agySubscriptionRealm.js"; export { createOrganizationRuntimeHost } from "./organizationRuntimeHost.js"; export { createOrganizationRuntimeControlHost } from "./organizationRuntimeControl.js"; +export { CODEX_SANDBOX_PROJECTION_VERSION, resolveOrganizationCodexSandboxProjection, + type OrganizationCodexSandboxProjection } from "./codexSandboxProjection.js"; export { WakeTransitionLockBlockedError } from "./wakeAcceptanceStore.js"; export { OFFLINE_RECONCILIATION_BLOCKED_CODE, From 43fe48cbf2cc3865c9499e0e9562727aebf54587 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Sun, 13 Sep 2026 22:40:27 +0200 Subject: [PATCH 002/124] fix: validate and bound credential reads from file descriptors --- .../portableCredentialBoundary.test.ts | 51 +++++ .../portableCredentialFailures.test.ts | 58 ++++++ .../portableCredentialMaterial.linux.test.ts | 55 ++++++ src/runtime/portableCredentialMaterial.ts | 39 +++- src/runtime/portableCredentialRead.test.ts | 181 ++++++++++++++++++ 5 files changed, 374 insertions(+), 10 deletions(-) create mode 100644 src/runtime/portableCredentialBoundary.test.ts create mode 100644 src/runtime/portableCredentialFailures.test.ts create mode 100644 src/runtime/portableCredentialMaterial.linux.test.ts create mode 100644 src/runtime/portableCredentialRead.test.ts diff --git a/src/runtime/portableCredentialBoundary.test.ts b/src/runtime/portableCredentialBoundary.test.ts new file mode 100644 index 0000000..8d00742 --- /dev/null +++ b/src/runtime/portableCredentialBoundary.test.ts @@ -0,0 +1,51 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { chmod, link, mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +import { materializePortableCredential } from "./portableCredentialMaterial.js"; +import type { OrganizationRuntimeAgentConfig } from "./organizationRuntime.js"; + +const execute = promisify(execFile); +const reader = fileURLToPath(new URL("./portableCredentialMaterial.ts", import.meta.url)); +const child = `const {materializePortableCredential}=await import(process.argv[1]);const home=process.argv[2];try{await materializePortableCredential({id:'agent:codex',name:'Codex',instructions:'Unused',workspacePath:home+'/workspace',runtimeHomePath:home,engine:{kind:'codex'}},home);process.stdout.write('UNEXPECTED_IMPORT');}catch{process.stdout.write('MATERIALIZATION_REFUSED');}`; + +for (const kind of ["mode", "hardlink", "directory", "empty", "oversize", "symlink"] as const) { + test(`real filesystem rejects unsafe credential ${kind}`, async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-credential-boundary-")); + const source = `${root}/.daimon-inbound/codex-auth`; + const agent: OrganizationRuntimeAgentConfig = { id: "agent:codex", name: "Codex", instructions: "Unused", workspacePath: `${root}/workspace`, runtimeHomePath: root, engine: { kind: "codex" } }; + try { + await mkdir(path.dirname(source), { mode: 0o700 }); + if (kind === "directory") await mkdir(source, { mode: 0o600 }); + else if (kind === "symlink") { await writeFile(`${root}/target`, "dummy", { mode: 0o600 }); await symlink(`${root}/target`, source); } + else { + await writeFile(source, kind === "empty" ? "" : kind === "oversize" ? "x".repeat(65537) : "dummy", { mode: 0o600 }); + if (kind === "mode") await chmod(source, 0o644); + if (kind === "hardlink") await link(source, `${root}/second-link`); + } + await assert.rejects(materializePortableCredential(agent, root), /credential materialization failed/u); + await assert.rejects(readFile(`${root}/.codex/auth.json`), { code: "ENOENT" }); + } finally { await rm(root, { recursive: true, force: true }); } + }); +} + +test("a real FIFO is rejected promptly; deleting nonblocking open makes the subprocess hang", { skip: process.platform === "win32", timeout: 15000 }, async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-credential-fifo-")); + try { + await mkdir(`${root}/.daimon-inbound`, { mode: 0o700 }); + await execute("mkfifo", ["-m", "600", `${root}/.daimon-inbound/codex-auth`]); + const run = (modulePath: string) => execute(process.execPath, ["--import", "tsx", "--input-type=module", "-e", child, modulePath, root], { timeout: 3000, killSignal: "SIGKILL" }); + assert.equal((await run(reader)).stdout, "MATERIALIZATION_REFUSED"); + const original = await readFile(reader, "utf8"); + const mutated = original.replace(" | constants.O_NONBLOCK", "").replace('"./contractManifest.js"', JSON.stringify(fileURLToPath(new URL("./contractManifest.ts", import.meta.url)))); + assert.notEqual(mutated, original); + await writeFile(`${root}/package.json`, '{"type":"module"}'); + const mutant = `${root}/mutant.ts`; await writeFile(mutant, mutated); + await assert.rejects(run(mutant), (error: NodeJS.ErrnoException & { killed?: boolean; signal?: string }) => error.killed === true && error.signal === "SIGKILL"); + } finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/src/runtime/portableCredentialFailures.test.ts b/src/runtime/portableCredentialFailures.test.ts new file mode 100644 index 0000000..475e801 --- /dev/null +++ b/src/runtime/portableCredentialFailures.test.ts @@ -0,0 +1,58 @@ +import assert from "node:assert/strict"; +import fs from "node:fs/promises"; +import { syncBuiltinESMExports } from "node:module"; +import os from "node:os"; +import path from "node:path"; +import test, { mock } from "node:test"; + +import { materializePortableCredential } from "./portableCredentialMaterial.js"; +import type { OrganizationRuntimeAgentConfig } from "./organizationRuntime.js"; + +test("keeps directory and existing-credential protections and refuses undeclared engines", async () => { + for (const mode of ["engine", "directory-mode", "directory-link", "destination-mode", "destination-directory", "repair-destination"] as const) { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "daimon-credential-guards-")); + const config: OrganizationRuntimeAgentConfig = { id: "agent:codex", name: "Codex", instructions: "Unused", workspacePath: `${root}/workspace`, runtimeHomePath: root, engine: { kind: mode === "engine" ? "grok" : "codex" } }; + try { + await fs.mkdir(`${root}/.daimon-inbound`, { mode: 0o700 }); + await fs.writeFile(`${root}/.daimon-inbound/codex-auth`, "dummy", { mode: 0o600 }); + if (mode === "directory-mode") await fs.chmod(`${root}/.daimon-inbound`, 0o755); + if (mode === "directory-link") { await fs.rename(`${root}/.daimon-inbound`, `${root}/source`); await fs.symlink(`${root}/source`, `${root}/.daimon-inbound`); } + if (mode.startsWith("destination") || mode === "repair-destination") await fs.mkdir(`${root}/.codex`, { mode: mode === "repair-destination" ? 0o755 : 0o700 }); + if (mode === "destination-mode") await fs.writeFile(`${root}/.codex/auth.json`, "existing", { mode: 0o644 }); + if (mode === "destination-directory") await fs.mkdir(`${root}/.codex/auth.json`, { mode: 0o600 }); + if (mode === "repair-destination") { + assert.equal(await materializePortableCredential(config, root), "created"); + assert.equal((await fs.stat(`${root}/.codex`)).mode & 0o777, 0o700); + } else await assert.rejects(materializePortableCredential(config, root), /credential material/u); + } finally { await fs.rm(root, { recursive: true, force: true }); } + } +}); + +for (const failure of ["mkdir", "rename", "source-lstat", "existing-lstat", "temporary-close"] as const) test(`does not install credentials after ${failure} failure`, async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "daimon-credential-failure-")); + const config: OrganizationRuntimeAgentConfig = { id: "agent:codex", name: "Codex", instructions: "Unused", workspacePath: `${root}/workspace`, runtimeHomePath: root, engine: { kind: "codex" } }; + const source = `${root}/.daimon-inbound/codex-auth`, destination = `${root}/.codex/auth.json`; + const fail = (): never => { throw Object.assign(new Error("private-path-and-contents-must-not-leak"), { code: "EIO" }); }; + try { + await fs.mkdir(path.dirname(source), { mode: 0o700 }); await fs.writeFile(source, "dummy", { mode: 0o600 }); + if (failure === "mkdir") mock.method(fs, "mkdir", fail); + if (failure === "rename") mock.method(fs, "rename", fail); + if (failure.endsWith("lstat")) { + const original = fs.lstat; + mock.method(fs, "lstat", (...args: Parameters) => args[0] === (failure === "source-lstat" ? source : destination) ? fail() : original(...args)); + } + if (failure === "temporary-close") { + const original = fs.open; + mock.method(fs, "open", async (...args: Parameters) => { + const handle = await original(...args); + if (String(args[0]).endsWith(".tmp")) { const close = handle.close.bind(handle); mock.method(handle, "close", async () => { await close(); fail(); }); } + return handle; + }); + } + syncBuiltinESMExports(); + await assert.rejects(materializePortableCredential(config, root), (error: Error) => error.message === "agent agent:codex codex credential materialization failed"); + await assert.rejects(fs.readFile(destination), { code: "ENOENT" }); + const files = await fs.readdir(`${root}/.codex`).catch(() => []); + assert.equal(files.some(file => file.endsWith(".tmp")), false); + } finally { mock.restoreAll(); syncBuiltinESMExports(); await fs.rm(root, { recursive: true, force: true }); } +}); diff --git a/src/runtime/portableCredentialMaterial.linux.test.ts b/src/runtime/portableCredentialMaterial.linux.test.ts new file mode 100644 index 0000000..70d1ed3 --- /dev/null +++ b/src/runtime/portableCredentialMaterial.linux.test.ts @@ -0,0 +1,55 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +const image = process.env.DAIMON_CREDENTIAL_BIND_TEST_IMAGE; +const script = `import fs from 'node:fs/promises'; +const home='/native-home',source=home+'/.daimon-inbound/codex-auth'; +const before=await fs.lstat(source); +const {materializePortableCredential}=await import('/opt/paideia/credential-test/reader.ts'); +const result=await materializePortableCredential({id:'agent:codex',name:'Codex',instructions:'Unused',workspacePath:'/tmp/workspace',runtimeHomePath:home,engine:{kind:'codex'}},home); +if(result!=='created'||await fs.readFile(home+'/.codex/auth.json','utf8')!=='non-secret-dummy-credential')throw Error('dummy credential was not materialized'); +const after=await fs.lstat(source);process.stdout.write(JSON.stringify({result,uid:process.getuid(),beforeUid:before.uid,afterUid:after.uid,descriptorRefreshObserved:before.uid!==after.uid,verified:true}));`; + +test("actual read-only Docker bind materializes a fresh dummy credential without metadata prewarming", { skip: !image, timeout: 30000 }, async t => { + const parent = fileURLToPath(new URL("../../.runtime/", import.meta.url)); + await mkdir(parent, { recursive: true, mode: 0o700 }); + const root = await mkdtemp(path.join(parent, "daimon-dummy-credential-bind-")); + try { + const home = `${root}/home`, code = `${root}/code`, source = `${root}/dummy-credential`; + await mkdir(`${home}/.daimon-inbound`, { recursive: true, mode: 0o700 }); await mkdir(code); + await writeFile(source, "non-secret-dummy-credential", { mode: 0o600 }); + // Execute the edited reader without building dist; its unchanged contract + // constant comes from the image's public runtime export, never private code. + const bytes = (await readFile(fileURLToPath(new URL("./portableCredentialMaterial.ts", import.meta.url)), "utf8")) + .replace('"./contractManifest.js"', '"@noopolis/daimon/runtime"'); + await writeFile(`${code}/reader.ts`, bytes); + const result = await promisify(execFile)("docker", ["run", "--rm", "--network", "none", "--read-only", + "--user", `${process.getuid?.() ?? 501}:${process.getgid?.() ?? 20}`, "--cap-drop", "ALL", "--security-opt", "no-new-privileges", + "--tmpfs", "/tmp:rw,nosuid,nodev", "--mount", `type=bind,source=${home},target=/native-home`, + "--mount", `type=bind,source=${source},target=/native-home/.daimon-inbound/codex-auth,readonly`, + "--mount", `type=bind,source=${code},target=/opt/paideia/credential-test,readonly`, + "--entrypoint", "/usr/local/bin/node", image!, "--experimental-strip-types", "--input-type=module", "-e", script + ], { timeout: 20000, maxBuffer: 16384 }); + const proof = JSON.parse(result.stdout) as { result: string; uid: number; beforeUid: number; afterUid: number; verified: boolean; descriptorRefreshObserved: boolean }; + assert.equal(proof.result, "created"); assert.equal(proof.verified, true); assert.equal(proof.afterUid, proof.uid); + t.diagnostic(JSON.stringify(proof)); + // Root can read the dummy file via DAC_OVERRIDE; the reader must still + // reject its genuinely different owner rather than relying on open EACCES. + const unsafeOwner = `import fs from 'node:fs/promises'; +const home='/native-home',source=home+'/.daimon-inbound/codex-auth';await fs.mkdir(home+'/.daimon-inbound',{mode:448});await fs.writeFile(source,'dummy',{mode:384});await fs.chown(source,501,20); +const opened=await fs.open(source,'r');if((await opened.stat()).uid===process.getuid())throw Error('owner fixture invalid');await opened.close(); +const {materializePortableCredential}=await import('/opt/paideia/credential-test/reader.ts');let refused=false;try{await materializePortableCredential({id:'agent:codex',name:'Codex',instructions:'Unused',workspacePath:'/tmp/workspace',runtimeHomePath:home,engine:{kind:'codex'}},home);}catch{refused=true;} +if(!refused)throw Error('unsafe owner imported');try{await fs.access(home+'/.codex/auth.json');throw Error('destination exists');}catch(e){if(e.code!=='ENOENT')throw e;}process.stdout.write('UNSAFE_OWNER_REFUSED');`; + const negative = await promisify(execFile)("docker", ["run", "--rm", "--network", "none", "--read-only", "--user", "0:0", + "--cap-drop", "ALL", "--cap-add", "CHOWN", "--cap-add", "DAC_OVERRIDE", "--security-opt", "no-new-privileges", + "--tmpfs", "/tmp:rw,nosuid,nodev", "--tmpfs", "/native-home:rw,mode=0700", + "--mount", `type=bind,source=${code},target=/opt/paideia/credential-test,readonly`, "--entrypoint", "/usr/local/bin/node", + image!, "--experimental-strip-types", "--input-type=module", "-e", unsafeOwner], { timeout: 10000, maxBuffer: 16384 }); + assert.equal(negative.stdout, "UNSAFE_OWNER_REFUSED"); + } finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/src/runtime/portableCredentialMaterial.ts b/src/runtime/portableCredentialMaterial.ts index 0dc0a71..3d729fd 100644 --- a/src/runtime/portableCredentialMaterial.ts +++ b/src/runtime/portableCredentialMaterial.ts @@ -1,4 +1,4 @@ -import { constants } from "node:fs"; +import { constants, type Stats } from "node:fs"; import { chmod, lstat, mkdir, open, rename, unlink } from "node:fs/promises"; import { randomUUID } from "node:crypto"; import path from "node:path"; @@ -15,6 +15,7 @@ type FileIdentity = Readonly<{ mtimeMs: number; size: number; uid: number; + nlink: number; }>; /** @@ -72,13 +73,25 @@ async function readCredential( ): Promise<{ bytes: Buffer; identity: FileIdentity }> { let handle: Awaited> | undefined; try { - const before = await assertCredential(filePath, mode, agent); - handle = await open(filePath, constants.O_RDONLY | noFollow()); - const opened = identity(await handle.stat()); - if (!sameIdentity(before, opened)) throw new Error("credential changed during import"); - const bytes = await handle.readFile(); - const after = identity(await handle.stat()); - if (!sameIdentity(before, after) || bytes.length !== before.size) { + // Opening a read-only bind can refresh its metadata. Validate the actual + // descriptor before consulting the refreshed path or reading any bytes. + // Nonblocking open lets us reject a FIFO by type instead of hanging here. + handle = await open(filePath, constants.O_RDONLY | noFollow() | constants.O_NONBLOCK); + const before = assertCredentialEntry(await handle.stat(), mode, agent); + if (!sameIdentity(before, await assertCredential(filePath, mode, agent))) throw new Error("credential changed during import"); + // One extra byte detects growth without allowing a changing file to drive + // an unbounded read/allocation. Partial reads advance within this buffer. + const buffer = Buffer.alloc(before.size + 1); + let length = 0; + while (length < buffer.length) { + const result = await handle.read(buffer, length, buffer.length - length, length); + if (result.bytesRead === 0) break; + length += result.bytesRead; + } + const bytes = buffer.subarray(0, length); + const after = assertCredentialEntry(await handle.stat(), mode, agent); + const afterPath = await assertCredential(filePath, mode, agent); + if (!sameIdentity(before, after) || !sameIdentity(before, afterPath) || bytes.length !== before.size) { throw new Error("credential changed during import"); } return { bytes, identity: before }; @@ -147,6 +160,10 @@ async function assertCredential( if ((error as NodeJS.ErrnoException).code === "ENOENT") throw error; throw unavailable(agent, error); } + return assertCredentialEntry(entry, mode, agent); +} + +function assertCredentialEntry(entry: Stats, mode: number, agent: PortableAgent): FileIdentity { if (!entry.isFile() || entry.isSymbolicLink() || entry.uid !== process.getuid?.() || entry.nlink !== 1 || (entry.mode & 0o777) !== mode || entry.size === 0 || entry.size > MAX_CREDENTIAL_BYTES) { @@ -172,6 +189,7 @@ function identity(value: Awaited>): FileIdentity { mtimeMs: number; size: number; uid: number; + nlink: number; }; return { dev: numeric.dev, @@ -179,12 +197,13 @@ function identity(value: Awaited>): FileIdentity { mode: numeric.mode & 0o7777, mtimeMs: numeric.mtimeMs, size: numeric.size, - uid: numeric.uid + uid: numeric.uid, + nlink: numeric.nlink }; } function sameIdentity(left: FileIdentity, right: FileIdentity): boolean { return left.dev === right.dev && left.ino === right.ino && left.mode === right.mode - && left.mtimeMs === right.mtimeMs && left.size === right.size && left.uid === right.uid; + && left.mtimeMs === right.mtimeMs && left.size === right.size && left.uid === right.uid && left.nlink === right.nlink; } function noFollow(): number { return (constants as typeof constants & { O_NOFOLLOW?: number }).O_NOFOLLOW ?? 0; diff --git a/src/runtime/portableCredentialRead.test.ts b/src/runtime/portableCredentialRead.test.ts new file mode 100644 index 0000000..9245de0 --- /dev/null +++ b/src/runtime/portableCredentialRead.test.ts @@ -0,0 +1,181 @@ +import assert from "node:assert/strict"; +import { constants, type Stats } from "node:fs"; +import fs from "node:fs/promises"; +import { syncBuiltinESMExports } from "node:module"; +import os from "node:os"; +import path from "node:path"; +import test, { mock } from "node:test"; + +import { materializePortableCredential } from "./portableCredentialMaterial.js"; +import type { OrganizationRuntimeAgentConfig } from "./organizationRuntime.js"; + +const altered = (entry: Stats, patch: Partial): Stats => Object.assign(Object.create(Object.getPrototypeOf(entry)), entry, patch); +async function fixture(run: (agent: OrganizationRuntimeAgentConfig, source: string, destination: string) => Promise): Promise { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "daimon-credential-descriptor-")); + const agent: OrganizationRuntimeAgentConfig = { id: "agent:codex", name: "Codex", instructions: "Unused.", workspacePath: `${root}/workspace`, runtimeHomePath: `${root}/home`, engine: { kind: "codex" } }; + const source = `${agent.runtimeHomePath}/.daimon-inbound/codex-auth`, destination = `${agent.runtimeHomePath}/.codex/auth.json`; + await fs.mkdir(path.dirname(source), { recursive: true, mode: 0o700 }); + await fs.writeFile(source, "dummy-credential", { mode: 0o600 }); + try { await run(agent, source, destination); } + finally { mock.restoreAll(); syncBuiltinESMExports(); await fs.rm(root, { recursive: true, force: true }); } +} + +test("opens before validation when a bind's pathname metadata refreshes on open", async () => { + await fixture(async (agent, source, destination) => { + const originalOpen = fs.open, originalLstat = fs.lstat; + const order: string[] = []; + let opened = false; + mock.method(fs, "lstat", async (...args: Parameters) => { + const entry = await originalLstat(...args); + if (args[0] !== source) return entry; + order.push("path"); + return opened ? entry : altered(entry as Stats, { uid: (process.getuid?.() ?? 0) + 1 }); + }); + mock.method(fs, "open", async (...args: Parameters) => { + const handle = await originalOpen(...args); + if (args[0] === source) { + order.push("open"); opened = true; + assert.ok((Number(args[1]) & constants.O_NOFOLLOW) !== 0); + assert.ok((Number(args[1]) & constants.O_NONBLOCK) !== 0); + const stat = handle.stat.bind(handle), read = handle.read.bind(handle); + mock.method(handle, "stat", async () => { order.push("descriptor"); return stat(); }); + mock.method(handle, "read", (buffer: Buffer, offset: number, length: number, position: number) => { order.push("read"); return read(buffer, offset, length, position); }); + } + return handle; + }); + syncBuiltinESMExports(); + assert.equal(await materializePortableCredential(agent, agent.runtimeHomePath), "created"); + assert.deepEqual(order, ["open", "descriptor", "path", "read", "read", "descriptor", "path"]); + assert.equal(await fs.readFile(destination, "utf8"), "dummy-credential"); + }); +}); + +for (const [name, patch] of [ + ["owner", { uid: (process.getuid?.() ?? 0) + 1 }], ["mode", { mode: 0o100644 }], + ["link count", { nlink: 2 }], ["empty", { size: 0 }], ["oversize", { size: 65537 }], + ["directory", { mode: 0o040600 }] +] as const) test(`rejects unsafe descriptor ${name} before reading`, async () => { + await fixture(async (agent, source, destination) => { + const originalOpen = fs.open; + let reads = 0; + mock.method(fs, "open", async (...args: Parameters) => { + const handle = await originalOpen(...args); + if (args[0] === source) { + const entry = await handle.stat(), read = handle.read.bind(handle); + mock.method(handle, "stat", async () => altered(entry, patch)); + mock.method(handle, "read", (buffer: Buffer, offset: number, length: number, position: number) => { reads++; return read(buffer, offset, length, position); }); + } + return handle; + }); + syncBuiltinESMExports(); + await assert.rejects(materializePortableCredential(agent, agent.runtimeHomePath), /credential materialization failed/u); + assert.equal(reads, 0); + await assert.rejects(fs.stat(destination), { code: "ENOENT" }); + }); +}); + +for (const phase of ["before", "after"] as const) test(`rejects a pathname replaced ${phase} the read`, async () => { + await fixture(async (agent, source, destination) => { + const originalLstat = fs.lstat; + let paths = 0; + mock.method(fs, "lstat", async (...args: Parameters) => { + const entry = await originalLstat(...args); + if (args[0] !== source) return entry; + paths++; + return paths >= (phase === "before" ? 1 : 2) ? altered(entry as Stats, { ino: Number(entry.ino) + 1 }) : entry; + }); + syncBuiltinESMExports(); + await assert.rejects(materializePortableCredential(agent, agent.runtimeHomePath), /credential materialization failed/u); + await assert.rejects(fs.stat(destination), { code: "ENOENT" }); + }); +}); + +test("rejects descriptor metadata and byte-length changes during the read", async () => { + for (const changed of ["mtime", "bytes"] as const) await fixture(async (agent, source, destination) => { + const originalOpen = fs.open; + mock.method(fs, "open", async (...args: Parameters) => { + const handle = await originalOpen(...args); + if (args[0] === source) { + const entry = await handle.stat(); let stats = 0; + mock.method(handle, "stat", async () => ++stats === 1 || changed === "bytes" ? entry : altered(entry, { mtimeMs: entry.mtimeMs + 1 })); + if (changed === "bytes") mock.method(handle, "read", async (buffer: Buffer) => ({ bytesRead: 0, buffer })); + } + return handle; + }); + syncBuiltinESMExports(); + await assert.rejects(materializePortableCredential(agent, agent.runtimeHomePath), /credential materialization failed/u); + await assert.rejects(fs.stat(destination), { code: "ENOENT" }); + }); +}); + +test("assembles partial descriptor reads within one expected-size-plus-one buffer", async () => { + await fixture(async (agent, source, destination) => { + const originalOpen = fs.open, expected = await fs.readFile(source); + const buffers = new Set(); let calls = 0; + mock.method(fs, "open", async (...args: Parameters) => { + const handle = await originalOpen(...args); + if (args[0] === source) { + const read = handle.read.bind(handle); + mock.method(handle, "readFile", () => { throw Error("unbounded read forbidden"); }); + mock.method(handle, "read", async (buffer: Buffer, offset: number, length: number, position: number) => { + calls++; buffers.add(buffer); + assert.equal(buffer.length, expected.length + 1); + assert.equal(position, offset); assert.ok(offset + length <= buffer.length); + return read(buffer, offset, Math.min(length, 3), position); + }); + } + return handle; + }); + syncBuiltinESMExports(); + assert.equal(await materializePortableCredential(agent, agent.runtimeHomePath), "created"); + assert.deepEqual(await fs.readFile(destination), expected); + assert.equal(buffers.size, 1); assert.equal(calls, Math.ceil(expected.length / 3) + 1); + }); +}); + +for (const partial of [false, true]) test(`growth after safe metadata remains byte-bounded (${partial ? "partial" : "full"} reads)`, async () => { + await fixture(async (agent, source, destination) => { + const originalOpen = fs.open, expectedSize = (await fs.stat(source)).size; + const buffers = new Set(); let calls = 0, bytes = 0; + mock.method(fs, "open", async (...args: Parameters) => { + const handle = await originalOpen(...args); + if (args[0] === source) { + const read = handle.read.bind(handle); + mock.method(handle, "readFile", () => { throw Error("unbounded read forbidden"); }); + mock.method(handle, "read", async (buffer: Buffer, offset: number, length: number, position: number) => { + if (calls++ === 0) await fs.truncate(source, 16 * 1024 * 1024); + buffers.add(buffer); assert.equal(buffer.length, expectedSize + 1); + assert.ok(length > 0 && offset + length <= buffer.length); + const result = await read(buffer, offset, partial ? 1 : length, position); bytes += result.bytesRead; return result; + }); + } + return handle; + }); + syncBuiltinESMExports(); + await assert.rejects(materializePortableCredential(agent, agent.runtimeHomePath), /credential materialization failed/u); + assert.equal(bytes, expectedSize + 1); assert.equal(buffers.size, 1); + assert.equal(calls, partial ? expectedSize + 1 : 1); + await assert.rejects(fs.stat(destination), { code: "ENOENT" }); + }); +}); + +for (const size of [0, 5]) test(`rejects truncation to ${size} bytes after safe metadata`, async () => { + await fixture(async (agent, source, destination) => { + const originalOpen = fs.open; let calls = 0; + mock.method(fs, "open", async (...args: Parameters) => { + const handle = await originalOpen(...args); + if (args[0] === source) { + const read = handle.read.bind(handle); + mock.method(handle, "read", async (buffer: Buffer, offset: number, length: number, position: number) => { + if (calls++ === 0) await fs.truncate(source, size); + return read(buffer, offset, length, position); + }); + } + return handle; + }); + syncBuiltinESMExports(); + await assert.rejects(materializePortableCredential(agent, agent.runtimeHomePath), /credential materialization failed/u); + assert.equal(calls, size === 0 ? 1 : 2); + await assert.rejects(fs.stat(destination), { code: "ENOENT" }); + }); +}); From 551f5961507cc27ada6962460676d8d0aff0dc8b Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 04:46:18 +0200 Subject: [PATCH 003/124] feat: pin Grok CLI 1.0.34 lean worker contract and single config renderer --- scripts/liveGrokBrokerSession.ts | 16 +-- src/contracts/grokWorkerContract.ts | 42 ++++++++ src/contracts/runtimeContractManifest.ts | 33 ++++++ src/runtime/grokBrokerModelPolicy.ts | 32 ++++++ src/runtime/grokBrokerWorkerConfig.test.ts | 65 +++++++++++- src/runtime/grokBrokerWorkerConfig.ts | 115 +++++++++++++++++++-- 6 files changed, 284 insertions(+), 19 deletions(-) create mode 100644 src/contracts/grokWorkerContract.ts create mode 100644 src/runtime/grokBrokerModelPolicy.ts diff --git a/scripts/liveGrokBrokerSession.ts b/scripts/liveGrokBrokerSession.ts index 092bb9b..0e0663e 100644 --- a/scripts/liveGrokBrokerSession.ts +++ b/scripts/liveGrokBrokerSession.ts @@ -9,7 +9,8 @@ import { terminateChild, trackCliChild } from "../src/pi/cliProcess.ts"; import { decodeGrokHeadlessTurn } from "../src/pi/grokHeadlessResult.ts"; import { readGrokBrokerCredential } from "../src/runtime/grokBrokerCredentialReader.ts"; import { startGrokBrokerProxy } from "../src/runtime/grokBrokerProxy.ts"; -import { renderGrokBrokerWorkerConfig } from "../src/runtime/grokBrokerWorkerConfig.ts"; +import { DEFAULT_GROK_BROKER_MODEL_POLICY } from "../src/runtime/grokBrokerModelPolicy.ts"; +import { renderGrokBrokerWorkerArgs, renderGrokBrokerWorkerConfigWith } from "../src/runtime/grokBrokerWorkerConfig.ts"; // Explicit live auth/transport check, not the Linux native worker/isolation E2E. // Read the operator credential only in this process; never stage or rotate it. @@ -40,15 +41,16 @@ try { const helper = path.join(home, "auth-helper"); await writeFile(helper, `#!/bin/sh\nprintf '{"access_token":"${capability}","expires_in":600}\\n'\n`, { mode: 0o700 }); // No MCP tools are needed for this exact-reply authentication probe. - await writeFile(path.join(home, "config.toml"), renderGrokBrokerWorkerConfig(helper, proxy.port).split("[mcp_servers.daimon]")[0]); + await writeFile(path.join(home, "config.toml"), renderGrokBrokerWorkerConfigWith(DEFAULT_GROK_BROKER_MODEL_POLICY, { helperPath: helper, proxyPort: proxy.port, mcpUrl: "http://127.0.0.1:43124/mcp" }).split("[mcp_servers.daimon]")[0]); const prompt = path.join(home, "prompt.txt"); await writeFile(prompt, `Reply exactly ${sentinel}. Do not use tools.`); stage = `model turn ${round}`; - const child = trackCliChild(spawn("grok", [ - "--sandbox", "strict", "--prompt-file", prompt, "--no-memory", "--no-subagents", - "--disable-web-search", "--max-turns", "1", "--permission-mode", "dontAsk", - "--model", "daimon-broker-grok", "--output-format", "streaming-messages-json", - ], { + // The proxy only forwards the lean worker request shape (pinned client + // version, exact tool set, declared effort), so the probe uses the same + // argv as the native launcher with the built-in strict profile. + const args = [...renderGrokBrokerWorkerArgs(prompt, home)].map((value) => value === "daimon-strict" ? "strict" : value); + args[args.indexOf("--max-turns") + 1] = "1"; + const child = trackCliChild(spawn("grok", args, { cwd: home, detached: process.platform !== "win32", env: { PATH: process.env.PATH, HOME: home, GROK_HOME: home, LANG: "C", LC_ALL: "C", TZ: "UTC" }, stdio: ["ignore", "pipe", "pipe"], diff --git a/src/contracts/grokWorkerContract.ts b/src/contracts/grokWorkerContract.ts new file mode 100644 index 0000000..c0cc666 --- /dev/null +++ b/src/contracts/grokWorkerContract.ts @@ -0,0 +1,42 @@ +/** + * Fixed operating contract every broker-launched Grok worker receives through + * `--system-prompt-override`. + * + * The override replaces Grok's ~12k-token coding-agent system prompt and stops + * cwd `AGENTS.md` injection; the agent's identity and instructions still arrive + * in the prompt file. It is compiled into the native launcher byte-for-byte + * (`DBL_GROK_SYSTEM_PROMPT`), pinned by sha256 in the runtime contract + * manifest, and must stay ASCII without quotes or backslashes so the C literal + * needs no escaping. + * + * Daimon's per-wake tools reach Grok as deferred MCP tools named + * `daimon__` (server `[mcp_servers.daimon]`). Naming them lets `use_tool` + * run directly and saves one `search_tool` round trip per tool (P0: 3 → 2 + * requests). + */ +export const DAIMON_GROK_SYSTEM_PROMPT = [ + "You are a headless Daimon agent; no human is present.", + "Your identity, instructions and wake event are in the user prompt.", + "Daimon tools are MCP tools on server daimon: call a known one directly with use_tool (tool_name daimon__moltnet_read, daimon__moltnet_send, daimon__memory_search, daimon__memory_register, or another daimon__ name you were given); use search_tool only for a name you do not know.", + "If a tool result says output was saved to a file, read that path with read_file.", + "If a tool fails, do not retry it in a loop: stop and report the failure.", + "Your final answer is a private note to the runtime: one line, or empty." +].join(" "); + +/** Closed declared-model vocabulary; defaults are `grok-4.6` at `low`. */ +export const GROK_BROKER_MODELS = Object.freeze(["grok-4.6", "grok-4.5", "grok-build"] as const); +export const GROK_BROKER_REASONING_EFFORTS = Object.freeze(["low", "medium", "high"] as const); + +/** `--tools` input ids. These are NOT the model-visible names (see below). */ +export const GROK_WORKER_TOOL_IDS = Object.freeze(["run_terminal_cmd", "read_file", "grep", "list_dir", "search_tool", "use_tool"] as const); + +/** + * The exact tool names a lean worker request body must carry. + * + * Grok 1.0.34 fails open on an unmappable `--tools` entry and ships all 19 + * tools, so the proxy compares every upstream body against this set. + */ +export const GROK_WORKER_VISIBLE_TOOLS = Object.freeze(["grep", "list_dir", "read_file", "run_terminal_command", "search_tool", "use_tool"] as const); + +/** `--max-turns` backstop compiled into the launcher; per-wake ceilings belong to the broker. */ +export const GROK_WORKER_MAX_TURNS = 48 as const; diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index b462d54..561e781 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -1,4 +1,5 @@ import { WORK_AVAILABILITY_SCHEMA, WORK_BLOCKED_SCHEMA } from "./attentionContract.js"; +import { GROK_BROKER_MODELS, GROK_BROKER_REASONING_EFFORTS, GROK_WORKER_MAX_TURNS, GROK_WORKER_TOOL_IDS, GROK_WORKER_VISIBLE_TOOLS } from "./grokWorkerContract.js"; import { ORGANIZATION_RUNTIME_CONFIG_SCHEMA, ORGANIZATION_RUNTIME_CONFIG_V2_SCHEMA, @@ -34,6 +35,38 @@ export const GROK_ENGINE_BROKER = { providerProxy: { host: "127.0.0.1", port: 43_123 }, mcpFacade: { host: "127.0.0.1", port: 43_124, path: "/mcp" }, identities: { organizationUid: 2_000, brokerUid: 2_100, firstWorkerUid: 2_200 }, + grokCliVersion: "1.0.34", + grokCliBuild: "3736acbc8658", + grokCliArtifacts: { + arm64: { url: "https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-aarch64", sha256: "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", bytes: 136_090_504 }, + x64: { url: "https://storage.googleapis.com/grok-build-public-artifacts/cli/grok-1.0.34-linux-x86_64", sha256: "be5905e107d2b8b5f3c142d21ecfe4c8fd32a913d2fd551b788707930c4dc80d", bytes: 163_035_648 } + }, + worker: { + modelId: "daimon-broker-grok", + models: GROK_BROKER_MODELS, + reasoningEfforts: GROK_BROKER_REASONING_EFFORTS, + defaultModel: "grok-4.6", + defaultReasoningEffort: "low", + toolIds: GROK_WORKER_TOOL_IDS, + visibleTools: GROK_WORKER_VISIBLE_TOOLS, + maxTurns: GROK_WORKER_MAX_TURNS, + systemPromptSha256: "2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8", + // sha256 of `renderGrokBrokerWorkerConfig({ model, reasoningEffort })`, the only accepted config.toml bytes. + configSha256: { + "grok-4.6": { low: "e09c127363094a7cad560586d89e994a9e6117b9c548feaffb1b5b01705cf363", medium: "b90807d3c73651a1a3f0bf89eacb0c73360e7cfc34d10e0185a381a27b40a718", high: "045171e44ba44b09522589ba85770f7c09fb8cab3e3f76fbb88cb534dcc01018" }, + "grok-4.5": { low: "848df2f71d0a88cf1185728cd6e0b7e15238b3a7536bf1538467f8c4868b0647", medium: "3e84795e834cf7b5857c24070d9f91b951c9c5f806823dd29c92cf88635a9318", high: "792f90bb7ae5154d6e002419f5b308e2ea975fc55ae7c324952f928329238f93" }, + "grok-build": { low: "1be3438799f9023b6acdaec991c139133bc791877f86a8b139129ef4b7b8386c", medium: "cadef8a2fcca778b515fcc10bfa8ab2ed8425fa46f37dc3a64095b477beb914a", high: "cb3ef9f71eefa913517dc775e5d72c49cbf718cf6c43ccc33d55b22401ef2e2f" } + }, + // Worker `GROK_HOME` layout the broker attests before every turn. The home and + // its `sessions/` directory are root-owned, worker-group writable and sticky so + // Grok can create its own state but never replace a root-owned file. + home: { + directory: { uid: 0, group: "worker", mode: 0o1771 }, + sessionsDirectory: { relativePath: "sessions", uid: 0, group: "worker", mode: 0o1771 }, + readOnlyFiles: { names: ["config.toml", "managed_config.toml", "requirements.toml", "sandbox.toml", "trusted_folders.toml"], uid: 0, gid: 0, mode: 0o444 }, + sandboxEvents: { relativePath: "sessions/sandbox-events.jsonl", owner: "worker", group: "broker", mode: 0o640 } + } + }, bounds: { promptBytes: 65_536, capabilityBytes: 4_096, capabilityBundleBytes: 8_196, outputBytes: 65_536 }, artifacts: { sourceSha256: "bdcab1e12dcc531ed8e56f890263ca23a9ee7bac468191dd598e143df4ff8c58", diff --git a/src/runtime/grokBrokerModelPolicy.ts b/src/runtime/grokBrokerModelPolicy.ts new file mode 100644 index 0000000..266a1d7 --- /dev/null +++ b/src/runtime/grokBrokerModelPolicy.ts @@ -0,0 +1,32 @@ +/** + * The closed model and reasoning-effort vocabulary a Grok broker worker may be + * declared with. + * + * Both halves of one declaration are consumed from this single parser: the + * worker `config.toml` renderer (`grokBrokerWorkerConfig.ts`) writes them into + * the worker's only custom model, and the provider proxy + * (`grokBrokerProxyRequest.ts`) refuses any request body that does not carry + * exactly them. Nothing is inherited: Grok 1.0.34 silently drops an effort for + * a model that does not declare effort support, and its embedded catalog + * default for `grok-4.6` is `high`. + */ +import { GROK_BROKER_MODELS, GROK_BROKER_REASONING_EFFORTS } from "../contracts/grokWorkerContract.js"; + +export { GROK_BROKER_MODELS, GROK_BROKER_REASONING_EFFORTS }; +export type GrokBrokerModel = (typeof GROK_BROKER_MODELS)[number]; +export type GrokBrokerReasoningEffort = (typeof GROK_BROKER_REASONING_EFFORTS)[number]; +export type GrokBrokerModelPolicy = Readonly<{ model: GrokBrokerModel; reasoningEffort: GrokBrokerReasoningEffort }>; + +export const DEFAULT_GROK_BROKER_MODEL_POLICY: GrokBrokerModelPolicy = Object.freeze({ model: "grok-4.6", reasoningEffort: "low" }); + +export function parseGrokBrokerModelPolicy(value: unknown = {}): GrokBrokerModelPolicy { + if (value === null || typeof value !== "object" || Array.isArray(value)) throw new TypeError("invalid Grok broker model policy"); + const input = value as Record; + if (Object.keys(input).some((key) => key !== "model" && key !== "reasoningEffort")) throw new TypeError("invalid Grok broker model policy"); + const model = input.model ?? DEFAULT_GROK_BROKER_MODEL_POLICY.model; + const reasoningEffort = input.reasoningEffort ?? DEFAULT_GROK_BROKER_MODEL_POLICY.reasoningEffort; + if (!(GROK_BROKER_MODELS as readonly unknown[]).includes(model) || !(GROK_BROKER_REASONING_EFFORTS as readonly unknown[]).includes(reasoningEffort)) { + throw new TypeError("invalid Grok broker model policy"); + } + return Object.freeze({ model: model as GrokBrokerModel, reasoningEffort: reasoningEffort as GrokBrokerReasoningEffort }); +} diff --git a/src/runtime/grokBrokerWorkerConfig.test.ts b/src/runtime/grokBrokerWorkerConfig.test.ts index 0be5567..af63d80 100644 --- a/src/runtime/grokBrokerWorkerConfig.test.ts +++ b/src/runtime/grokBrokerWorkerConfig.test.ts @@ -1,8 +1,63 @@ import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; import test from "node:test"; -import { renderGrokBrokerWorkerArgs, renderGrokBrokerWorkerConfig } from "./grokBrokerWorkerConfig.js"; -test("worker config uses only named in-memory auth and fixed loopback proxy", () => { - const config = renderGrokBrokerWorkerConfig("/opt/daimon/bin/grok-broker-auth", 43123); - assert.match(config, /auth_provider\.daimon/u); assert.match(config, /args = \["--auth-provider"\]/u);assert.match(config, /127\.0\.0\.1:43123/u);assert.match(config,/127\.0\.0\.1:43124\/mcp/u);assert.match(config,/DAIMON_MCP_CAPABILITY/u); assert.doesNotMatch(config, /access_token|refresh_token|auth\.json/u); - const args = renderGrokBrokerWorkerArgs("/run/worker/prompt", "/workspace"); assert.equal(args.includes("--prompt-file"), true); assert.equal(args.includes("--single"), false); + +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import { GROK_BROKER_MODELS, GROK_BROKER_REASONING_EFFORTS } from "./grokBrokerModelPolicy.js"; +import { GROK_1_0_34_BUNDLED_SKILLS, grokBrokerWorkerConfigSha256, renderGrokBrokerWorkerArgs, renderGrokBrokerWorkerConfig, renderGrokBrokerWorkerConfigWith } from "./grokBrokerWorkerConfig.js"; + +const section = (config: string, header: string): string => { + const start = config.indexOf(`${header}\n`); + assert.notEqual(start, -1, `missing ${header}`); + const end = config.indexOf("\n[", start + header.length); + return config.slice(start, end === -1 ? undefined : end); +}; + +test("worker config uses only named in-memory auth, the fixed loopback proxy, and the capability-scoped MCP facade", () => { + const config = renderGrokBrokerWorkerConfig(); + assert.match(section(config, "[auth_provider.daimon]"), /command = "\/opt\/daimon\/bin\/daimon-engine-broker"\nargs = \["--auth-provider"\]/u); + assert.match(section(config, "[model.daimon-broker-grok]"), /base_url = "http:\/\/127\.0\.0\.1:43123\/v1"\nauth_provider = "daimon"/u); + assert.equal(section(config, "[mcp_servers.daimon]"), '[mcp_servers.daimon]\nurl = "http://127.0.0.1:43124/mcp"\nbearer_token_env_var = "DAIMON_MCP_CAPABILITY"\n'); + assert.doesNotMatch(config, /access_token|refresh_token|auth\.json/u); +}); + +test("worker config disables every bundled 1.0.34 skill, workflows, and the per-turn session title request", () => { + const config = renderGrokBrokerWorkerConfig(); + assert.equal(GROK_1_0_34_BUNDLED_SKILLS.length, 25); + assert.equal(section(config, "[skills]"), `[skills]\ndisabled = [${GROK_1_0_34_BUNDLED_SKILLS.map((name) => JSON.stringify(name)).join(", ")}]\n`); + assert.equal(section(config, "[workflows]"), "[workflows]\nenabled = false\n"); + assert.match(section(config, "[models]"), /\nsession_summary = "daimon-session-title-disabled"\n/u); + assert.equal(section(config, "[model.daimon-session-title-disabled]"), '[model.daimon-session-title-disabled]\nmodel = "disabled"\nbase_url = "http://127.0.0.1:9/v1"\napi_key = "session-title-disabled"\nmax_retries = 0\nhidden = true\n'); + for (const toggle of ["title_refresh", "telemetry", "session_recap", "turn_summary", "backend_tools", "ask_user_question"]) assert.match(section(config, "[features]"), new RegExp(`\\n${toggle} = false\\n`, "u")); + assert.match(section(config, "[cli]"), /auto_update = false\nuse_leader = false/u); +}); + +test("the declared model and effort reach the worker's only model as its sole allowed effort", () => { + const config = renderGrokBrokerWorkerConfig({ model: "grok-build", reasoningEffort: "medium" }); + assert.match(section(config, "[model.daimon-broker-grok]"), /\nmodel = "grok-build"\n/u); + assert.match(section(config, "[models]"), /\ndefault = "daimon-broker-grok"\ndefault_reasoning_effort = "medium"\n/u); + assert.equal(section(config, "[[model.daimon-broker-grok.reasoning_efforts]]"), '[[model.daimon-broker-grok.reasoning_efforts]]\nvalue = "medium"\nlabel = "Medium"\ndefault = true\n'); + assert.equal(config.match(/reasoning_efforts\]\]/gu)?.length, 1); + assert.match(renderGrokBrokerWorkerConfig(), /\nmodel = "grok-4\.6"\n[\s\S]*value = "low"/u); + for (const invalid of [{ model: "grok-3" }, { reasoningEffort: "xhigh" }, { model: "grok-4.6", reasoningEffort: "low", extra: true }]) { + assert.throws(() => renderGrokBrokerWorkerConfig(invalid as never), /model policy/u); + } +}); + +test("the manifest pins the sha256 of every renderable worker config", () => { + for (const model of GROK_BROKER_MODELS) { + for (const reasoningEffort of GROK_BROKER_REASONING_EFFORTS) { + const digest = createHash("sha256").update(renderGrokBrokerWorkerConfig({ model, reasoningEffort })).digest("hex"); + assert.equal(grokBrokerWorkerConfigSha256({ model, reasoningEffort }), digest); + assert.equal(GROK_ENGINE_BROKER.worker.configSha256[model][reasoningEffort], digest, `${model}/${reasoningEffort}`); + } + } +}); + +test("the probe-only renderer refuses non-loopback endpoints and injected helper paths", () => { + const policy = { model: "grok-4.6", reasoningEffort: "low" } as const; + assert.throws(() => renderGrokBrokerWorkerConfigWith(policy, { helperPath: "/x\"\n[evil]", proxyPort: 1, mcpUrl: "http://127.0.0.1:1/mcp" }), /invalid/u); + assert.throws(() => renderGrokBrokerWorkerConfigWith(policy, { helperPath: "/x", proxyPort: 1, mcpUrl: "http://example.com/mcp" }), /invalid/u); + const args = renderGrokBrokerWorkerArgs("/run/worker/prompt", "/workspace"); + assert.equal(args.includes("--prompt-file"), true); assert.equal(args.includes("--single"), false); }); diff --git a/src/runtime/grokBrokerWorkerConfig.ts b/src/runtime/grokBrokerWorkerConfig.ts index 4223490..da08c87 100644 --- a/src/runtime/grokBrokerWorkerConfig.ts +++ b/src/runtime/grokBrokerWorkerConfig.ts @@ -1,17 +1,118 @@ +import { createHash } from "node:crypto"; import path from "node:path"; -export function renderGrokBrokerWorkerConfig(helperPath: string, proxyPort: number): string { - if (!path.posix.isAbsolute(helperPath) || /[\r\n"']/u.test(helperPath) || !Number.isInteger(proxyPort) || proxyPort < 1 || proxyPort > 65_535) throw new TypeError("invalid Grok broker worker configuration"); +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import { DAIMON_GROK_SYSTEM_PROMPT, GROK_WORKER_MAX_TURNS, GROK_WORKER_TOOL_IDS } from "../contracts/grokWorkerContract.js"; +import { parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; + +export { DAIMON_GROK_SYSTEM_PROMPT, GROK_WORKER_MAX_TURNS, GROK_WORKER_TOOL_IDS, GROK_WORKER_VISIBLE_TOOLS } from "../contracts/grokWorkerContract.js"; + +/** + * Bundled skills shipped by Grok CLI 1.0.34. Names are version-specific: + * `[skills] disabled` removed all ~2.1k skill tokens in the P0 matrix, and a + * CLI bump must re-derive this list rather than inherit it. + */ +export const GROK_1_0_34_BUNDLED_SKILLS = Object.freeze([ + "build-with-ai", "code-review", "create-skill", "create-workflow", "design", "docx", "execute-plan", + "game-animation-frames", "game-asset-core", "game-character-consistency", "game-tilesets", "game-ui-icons", + "imagine", "implement", "learn", "long-running-background-tasks", "pdf", "pptx", "pr-babysit", + "resume-claude", "resume-codex", "resume-cursor", "review", "skill-design-principles", "statusline" +] as const); + +/** The worker's only model id; the argv selects it and the proxy never sees another. */ +export const GROK_BROKER_WORKER_MODEL_ID = "daimon-broker-grok" as const; + +/** + * Grok 1.0.34 sends a `session_title` model request before every headless + * turn, and no config key or environment variable disables it + * (`features.title_refresh` governs only the later refresh; verified against a + * loopback stub). `[models] session_summary` does select the model it uses, so + * the title goes to a hidden model whose endpoint is a closed privileged + * loopback port: the connection is refused locally, Grok falls back to the + * truncated prompt as the title, and neither the proxy nor the provider sees a + * request. The placeholder `api_key` is not a credential; it only stops Grok + * from looking for one. + */ +export const GROK_SESSION_TITLE_SINK_MODEL_ID = "daimon-session-title-disabled" as const; +const renderSessionTitleSink = (): readonly string[] => [ + `[model.${GROK_SESSION_TITLE_SINK_MODEL_ID}]`, 'model = "disabled"', 'base_url = "http://127.0.0.1:9/v1"', 'api_key = "session-title-disabled"', + "max_retries = 0", "hidden = true", "" +]; + +type WorkerEndpoints =Readonly<{ helperPath: string; proxyPort: number; mcpUrl: string }>; +const PRODUCTION_ENDPOINTS: WorkerEndpoints = Object.freeze({ + helperPath: GROK_ENGINE_BROKER.nativeExecutablePath, + proxyPort: GROK_ENGINE_BROKER.providerProxy.port, + mcpUrl: `http://${GROK_ENGINE_BROKER.mcpFacade.host}:${GROK_ENGINE_BROKER.mcpFacade.port}${GROK_ENGINE_BROKER.mcpFacade.path}` +}); + +/** + * Sections shared by every Grok worker Daimon configures, broker or direct. + * + * Each toggle is either measured (skills −2.1k tokens, workflows −314, the + * per-turn `session_title` model request) or behavioural hardening that P0 + * showed to be token-neutral and warning-free on 1.0.34. + */ +export const renderGrokLeanBaseConfig = (): string => [ + "[cli]", "auto_update = false", "use_leader = false", "show_tips = false", "", + "[features]", "telemetry = false", "title_refresh = false", "session_recap = false", "turn_summary = false", + "repo_status_in_system_prompt = false", "codebase_indexing = false", "backend_tools = false", "ask_user_question = false", + "image_gen = false", "video_gen = false", "web_fetch = false", "campaigns = false", "managed_config = false", "", + "[managed_mcps]", "enabled = false", "", + "[skills]", `disabled = [${GROK_1_0_34_BUNDLED_SKILLS.map((name) => JSON.stringify(name)).join(", ")}]`, "", + "[workflows]", "enabled = false", "" +].join("\n"); + +/** + * The only source of broker worker `config.toml` bytes. + * + * Effort is declared here, not on the compiled launcher argv: the argv is one + * constant for every registration while effort is declared per deployment, and + * Grok 1.0.34 drops both `--reasoning-effort` and `[models] + * default_reasoning_effort` unless the model advertises effort support. A + * one-entry `reasoning_efforts` table makes the declared effort the model's + * default *and* its closed enum, so it reaches the request body and nothing + * else can be selected; the proxy then re-verifies it on every body. + */ +export function renderGrokBrokerWorkerConfig(policy: Partial = {}): string { + return renderGrokBrokerWorkerConfigWith(parseGrokBrokerModelPolicy(policy), PRODUCTION_ENDPOINTS); +} + +/** Explicit-endpoint variant for the local live probe; production bytes come only from the function above. */ +export function renderGrokBrokerWorkerConfigWith(policy: GrokBrokerModelPolicy, endpoints: WorkerEndpoints): string { + const declared = parseGrokBrokerModelPolicy(policy); + const { helperPath, proxyPort, mcpUrl } = endpoints; + if (!path.posix.isAbsolute(helperPath) || /[\r\n"'\\]/u.test(helperPath) || !Number.isInteger(proxyPort) || proxyPort < 1 || proxyPort > 65_535 || !/^http:\/\/127\.0\.0\.1:\d{1,5}\/mcp$/u.test(mcpUrl)) { + throw new TypeError("invalid Grok broker worker configuration"); + } + const label = `${declared.reasoningEffort[0]!.toUpperCase()}${declared.reasoningEffort.slice(1)}`; return [ - "[cli]", "auto_update = false", "use_leader = false", "", - "[features]", "telemetry = false", "", + renderGrokLeanBaseConfig(), + "[models]", `default = "${GROK_BROKER_WORKER_MODEL_ID}"`, `default_reasoning_effort = "${declared.reasoningEffort}"`, `session_summary = "${GROK_SESSION_TITLE_SINK_MODEL_ID}"`, "", + ...renderSessionTitleSink(), "[auth_provider.daimon]", `command = ${JSON.stringify(helperPath)}`, 'args = ["--auth-provider"]', "timeout_secs = 5", "token_ttl_secs = 600", "", - "[model.daimon-broker-grok]", 'model = "grok-build"', `base_url = "http://127.0.0.1:${proxyPort}/v1"`, 'auth_provider = "daimon"', "context_window = 131072", "supports_backend_search = false", "", - "[mcp_servers.daimon]", 'url = "http://127.0.0.1:43124/mcp"', 'headers = { Authorization = "Bearer ${DAIMON_MCP_CAPABILITY}" }', "" + `[model.${GROK_BROKER_WORKER_MODEL_ID}]`, `model = "${declared.model}"`, `base_url = "http://127.0.0.1:${proxyPort}/v1"`, 'auth_provider = "daimon"', + 'api_backend = "chat_completions"', "context_window = 131072", "supports_backend_search = false", "", + `[[model.${GROK_BROKER_WORKER_MODEL_ID}.reasoning_efforts]]`, `value = "${declared.reasoningEffort}"`, `label = "${label}"`, "default = true", "", + "[mcp_servers.daimon]", `url = "${mcpUrl}"`, 'bearer_token_env_var = "DAIMON_MCP_CAPABILITY"', "" ].join("\n"); } +export const grokBrokerWorkerConfigSha256 = (policy: Partial = {}): string => + createHash("sha256").update(renderGrokBrokerWorkerConfig(policy)).digest("hex"); + +/** + * TypeScript mirror of the argv compiled into `native/engineBrokerLauncherCore.inc`. + * `grokWorkerArgv.test.ts` parses the C source and fails on any divergence. + */ export const renderGrokBrokerWorkerArgs = (promptFile: string, cwd: string): readonly string[] => { if (!path.posix.isAbsolute(promptFile) || !path.posix.isAbsolute(cwd)) throw new TypeError("invalid Grok broker worker path"); - return ["--sandbox", "daimon-strict", "--always-approve", "--no-subagents", "--prompt-file", promptFile, "--no-memory", "--disable-web-search", "--cwd", cwd, "--output-format", "streaming-messages-json", "--model", "daimon-broker-grok"]; + return [ + "--sandbox", "daimon-strict", "--always-approve", "--no-subagents", "--prompt-file", promptFile, + "--no-memory", "--disable-web-search", "--no-plan", "--verbatim", + "--system-prompt-override", DAIMON_GROK_SYSTEM_PROMPT, + "--tools", GROK_WORKER_TOOL_IDS.join(","), + "--max-turns", String(GROK_WORKER_MAX_TURNS), + "--cwd", cwd, "--output-format", "streaming-messages-json", "--model", GROK_BROKER_WORKER_MODEL_ID + ]; }; From 18b9d4897b6591309b12fbe87b803bc6c72f1a5f Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 04:46:18 +0200 Subject: [PATCH 004/124] feat: refuse Grok broker requests outside the lean tool set and declared model policy --- src/runtime/grokBrokerProxy.test.ts | 27 ++++++++-- src/runtime/grokBrokerProxy.ts | 12 +++-- src/runtime/grokBrokerProxyRequest.test.ts | 57 +++++++++++++++------- src/runtime/grokBrokerProxyRequest.ts | 40 ++++++++++++--- 4 files changed, 103 insertions(+), 33 deletions(-) diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index 5f2a362..0d09eb8 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -2,6 +2,9 @@ import assert from "node:assert/strict"; import test from "node:test"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; +const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); +const leanBody = (overrides: Record = {}): string => JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: lean, ...overrides }); + test("proxy retries one 401 with refreshed broker bearer and shuts down", async () => { const calls: string[] = []; let refreshes = 0; const proxy = await startGrokBrokerProxy({ accessToken: async (force) => force ? "second" : "first", markRejected: async () => { refreshes += 1; } }, async (request) => { @@ -9,19 +12,35 @@ test("proxy retries one 401 with refreshed broker bearer and shuts down", async }); const token = proxy.capabilities.issue("agent", "turn"); proxy.registerIsolationGuard("turn", async () => undefined); - const result = await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`, { method: "POST", headers: { authorization: `Bearer ${token}`, "content-type": "application/json", "x-grok-client-version": "1.0.25" }, body: JSON.stringify({ stream: true, messages: [] }) }); + const result = await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`, { method: "POST", headers: { authorization: `Bearer ${token}`, "content-type": "application/json", "x-grok-client-version": "1.0.34" }, body: leanBody() }); assert.equal(result.status, 200); assert.equal(await result.text(), "data: done\n\n"); assert.deepEqual(calls, ["Bearer first", "Bearer second"]); assert.equal(refreshes, 0); await proxy.close(); await assert.rejects(fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`)); }); -test("proxy stale-fences a refreshed credential rejected by upstream",async()=>{let rejected=0;const proxy=await startGrokBrokerProxy({accessToken:async(force)=>force?"second":"first",markRejected:async()=>{rejected++;throw new Error("stale");}},async()=>({status:401,headers:{"content-type":"application/json"},body:new Uint8Array()}));const token=proxy.capabilities.issue("agent","turn");proxy.registerIsolationGuard("turn",async()=>undefined);const response=await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`,{method:"POST",headers:{authorization:`Bearer ${token}`,"x-grok-client-version":"1.0.25"},body:JSON.stringify({stream:true,messages:[]})});assert.equal(response.status,503);assert.equal(rejected,1);await proxy.close();}); +test("proxy stale-fences a refreshed credential rejected by upstream",async()=>{let rejected=0;const proxy=await startGrokBrokerProxy({accessToken:async(force)=>force?"second":"first",markRejected:async()=>{rejected++;throw new Error("stale");}},async()=>({status:401,headers:{"content-type":"application/json"},body:new Uint8Array()}));const token=proxy.capabilities.issue("agent","turn");proxy.registerIsolationGuard("turn",async()=>undefined);const response=await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`,{method:"POST",headers:{authorization:`Bearer ${token}`,"x-grok-client-version":"1.0.34"},body:leanBody()});assert.equal(response.status,503);assert.equal(rejected,1);await proxy.close();}); test("proxy failures expose only a fixed diagnostic", async () => { const proxy = await startGrokBrokerProxy({ accessToken: async () => { throw new Error("secret-token"); }, markRejected: async () => undefined }, async () => { throw new Error("unreachable"); }); const token = proxy.capabilities.issue("agent", "turn"); proxy.registerIsolationGuard("turn", async () => undefined); - const result = await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`, { method: "POST", headers: { authorization: `Bearer ${token}`, "x-grok-client-version": "1.0.25" }, body: JSON.stringify({ stream: true, messages: [] }) }); + const result = await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`, { method: "POST", headers: { authorization: `Bearer ${token}`, "x-grok-client-version": "1.0.34" }, body: leanBody() }); assert.equal(result.status, 503); const body = await result.text(); assert.equal(body, '{"error":"broker unavailable"}'); assert.doesNotMatch(body, /secret/u); await proxy.close(); }); -test("one turn capability supports multiple guarded cognition requests",async()=>{let guarded=0,calls=0;const proxy=await startGrokBrokerProxy({accessToken:async()=>"provider-token",markRejected:async()=>undefined},async()=>{calls++;return{status:200,headers:{"content-type":"application/json"},body:Buffer.from("{}")};});try{const token=proxy.capabilities.issue("agent","turn");proxy.registerIsolationGuard("turn",async()=>{guarded++;});for(let index=0;index<2;index++){const response=await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`,{method:"POST",headers:{authorization:`Bearer ${token}`,"x-grok-client-version":"1.0.25"},body:JSON.stringify({stream:true,messages:[]})});assert.equal(response.status,200);}assert.equal(calls,2);assert.equal(guarded,2);}finally{await proxy.close();}}); +test("one turn capability supports multiple guarded cognition requests",async()=>{let guarded=0,calls=0;const proxy=await startGrokBrokerProxy({accessToken:async()=>"provider-token",markRejected:async()=>undefined},async()=>{calls++;return{status:200,headers:{"content-type":"application/json"},body:Buffer.from("{}")};});try{const token=proxy.capabilities.issue("agent","turn");proxy.registerIsolationGuard("turn",async()=>{guarded++;});for(let index=0;index<2;index++){const response=await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`,{method:"POST",headers:{authorization:`Bearer ${token}`,"x-grok-client-version":"1.0.34"},body:leanBody()});assert.equal(response.status,200);}assert.equal(calls,2);assert.equal(guarded,2);}finally{await proxy.close();}}); + +test("proxy refuses a fail-open tool set or an undeclared effort without calling upstream", async () => { + let calls = 0; let accessed = 0; + const proxy = await startGrokBrokerProxy({ accessToken: async () => { accessed++; return "provider-token"; }, markRejected: async () => undefined }, async () => { calls++; return { status: 200, headers: { "content-type": "application/json" }, body: Buffer.from("{}") }; }, { model: "grok-4.6", reasoningEffort: "low" }); + try { + const token = proxy.capabilities.issue("agent", "turn"); proxy.registerIsolationGuard("turn", async () => undefined); + const full = [...lean, ...["search_replace", "todo_write", "write", "monitor"].map((name) => ({ type: "function", function: { name } }))]; + for (const payload of [leanBody({ tools: full }), leanBody({ tools: [{ type: "function", function: { name: "session_title" } }] }), leanBody({ reasoning_effort: "high" }), leanBody({ reasoning_effort: undefined })]) { + const response = await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`, { method: "POST", headers: { authorization: `Bearer ${token}`, "x-grok-client-version": "1.0.34" }, body: payload }); + assert.equal(response.status, 503); await response.text(); + } + assert.equal(calls, 0); + const accepted = await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`, { method: "POST", headers: { authorization: `Bearer ${token}`, "x-grok-client-version": "1.0.34" }, body: leanBody() }); + assert.equal(accepted.status, 200); await accepted.text(); assert.equal(calls, 1); assert.ok(accessed >= 1); + } finally { await proxy.close(); } +}); diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index 4dee112..023a474 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -2,24 +2,26 @@ import { createHash } from "node:crypto"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import type { AddressInfo } from "node:net"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; +import { DEFAULT_GROK_BROKER_MODEL_POLICY, parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; import { authorizeGrokBrokerProxyRequest } from "./grokBrokerProxyRequest.js"; export type GrokBrokerCredentialAuthority = Readonly<{ accessToken(forceRefresh: boolean): Promise; refreshAfterRejection?(rejectedTokenDigest:string):Promise; markRejected(rejectedTokenDigest?:string): Promise }>; export type GrokBrokerUpstream = (request: ReturnType) => Promise>; body: Uint8Array }>>; -export async function startGrokBrokerProxy(authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream = defaultUpstream): PromisePromise):void; revokeIsolationGuard(turnId:string):void; close(): Promise }>> { - const capabilities = new EngineBrokerCapabilities(); - const guards=new MapPromise>();const server = createServer((request, response) => { void serve(request, response, authority, upstream, capabilities,guards); }); +/** `policy` is the declared model/effort every forwarded body must carry (closed list; defaults grok-4.6/low). */ +export async function startGrokBrokerProxy(authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream = defaultUpstream, policy: GrokBrokerModelPolicy = DEFAULT_GROK_BROKER_MODEL_POLICY): PromisePromise):void; revokeIsolationGuard(turnId:string):void; close(): Promise }>> { + const declared = parseGrokBrokerModelPolicy(policy); const capabilities = new EngineBrokerCapabilities(); + const guards=new MapPromise>();const server = createServer((request, response) => { void serve(request, response, authority, upstream, capabilities,guards,declared); }); await new Promise((resolve, reject) => { server.once("error", reject); server.listen(43_123, "127.0.0.1", () => { server.off("error", reject); resolve(); }); }); const address = server.address() as AddressInfo; return { port: address.port, capabilities,registerIsolationGuard(turnId,guard){guards.set(turnId,guard);},revokeIsolationGuard(turnId){guards.delete(turnId);}, close: () => new Promise((resolve, reject) => server.close((error) => error === undefined ? resolve() : reject(error))) }; } -async function serve(request: IncomingMessage, response: ServerResponse, authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream, capabilities: EngineBrokerCapabilities,guards:MapPromise>): Promise { +async function serve(request: IncomingMessage, response: ServerResponse, authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream, capabilities: EngineBrokerCapabilities,guards:MapPromise>,policy:GrokBrokerModelPolicy): Promise { try { const body = await readBody(request); const headers = Object.fromEntries(Object.entries(request.headers).map(([key, value]) => [key, Array.isArray(value) ? value[0] : value])); const match=headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u),scope=match?capabilities.inspectToken(match[1]!):undefined;if(!scope)throw new Error();const guard=guards.get(scope.turnId);if(!guard)throw new Error();await guard(); - let token = await authority.accessToken(false);const rejectedDigest=createHash("sha256").update(token).digest("hex"); let prepared = authorizeGrokBrokerProxyRequest({ method: request.method ?? "", pathname: new URL(request.url ?? "/", "http://127.0.0.1").pathname, headers, body }, capabilities, token); token = ""; + let token = await authority.accessToken(false);const rejectedDigest=createHash("sha256").update(token).digest("hex"); let prepared = authorizeGrokBrokerProxyRequest({ method: request.method ?? "", pathname: new URL(request.url ?? "/", "http://127.0.0.1").pathname, headers, body }, capabilities, token, policy); token = ""; let result = await upstream(prepared); if (result.status === 401) { token = authority.refreshAfterRejection?await authority.refreshAfterRejection(rejectedDigest):await authority.accessToken(true);const refreshedDigest=createHash("sha256").update(token).digest("hex"); prepared = { ...prepared, headers: { ...prepared.headers, authorization: `Bearer ${token}` } }; token = ""; result = await upstream(prepared);if(result.status===401)await authority.markRejected(refreshedDigest); } response.writeHead(result.status, { "content-type": result.headers["content-type"] ?? "application/json", "cache-control": "no-store" }); response.end(result.body); diff --git a/src/runtime/grokBrokerProxyRequest.test.ts b/src/runtime/grokBrokerProxyRequest.test.ts index 462c140..fb4752c 100644 --- a/src/runtime/grokBrokerProxyRequest.test.ts +++ b/src/runtime/grokBrokerProxyRequest.test.ts @@ -3,28 +3,49 @@ import test from "node:test"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; import { authorizeGrokBrokerProxyRequest } from "./grokBrokerProxyRequest.js"; -const body = Buffer.from(JSON.stringify({ stream: true, messages: [] })); - -test("proxy substitutes broker bearer, forwards bounded Grok client version, and rejects arbitrary routes and headers", () => { +const leanTools = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"]; +const tool = (name: string) => ({ type: "function", function: { name, parameters: { type: "object" } } }); +const leanBody = (overrides: Record = {}): Buffer => Buffer.from(JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: leanTools.map(tool), ...overrides })); +const request = (body: Uint8Array, headers: Record = {}) => { const caps = new EngineBrokerCapabilities(); const opaque = caps.issue("a", "t"); - const request = authorizeGrokBrokerProxyRequest({ method: "POST", pathname: "/v1/chat/completions", headers: { authorization: `Bearer ${opaque}`, cookie: "forbidden", "x-grok-client-version": "1.0.25", "x-grok-client-identifier": "attacker" }, body, agentId: "a", turnId: "t" }, caps, "real-bearer"); - assert.equal(request.url, "https://cli-chat-proxy.grok.com/v1/chat/completions"); - assert.equal(request.headers.authorization, "Bearer real-bearer"); - assert.equal(request.headers["x-grok-client-version"], "1.0.25"); - assert.equal(request.headers["x-grok-client-identifier"], "grok-shell"); - assert.equal("cookie" in request.headers, false); - assert.throws(() => authorizeGrokBrokerProxyRequest({ method: "GET", pathname: "/", headers: { authorization: `Bearer ${opaque}`, "x-grok-client-version": "1.0.25" }, body, agentId: "a", turnId: "t" }, caps, "real-bearer"), /rejected/); + return { caps, input: { method: "POST", pathname: "/v1/chat/completions", headers: { authorization: `Bearer ${opaque}`, "x-grok-client-version": "1.0.34", ...headers }, body, agentId: "a", turnId: "t" } }; +}; + +test("proxy substitutes broker bearer, forwards the pinned client version, and rejects arbitrary routes and headers", () => { + const { caps, input } = request(leanBody(), { cookie: "forbidden", "x-grok-client-identifier": "attacker", "x-grok-model-override": "grok-build" }); + const upstream = authorizeGrokBrokerProxyRequest(input, caps, "real-bearer"); + assert.equal(upstream.url, "https://cli-chat-proxy.grok.com/v1/chat/completions"); + assert.equal(upstream.headers.authorization, "Bearer real-bearer"); + assert.equal(upstream.headers["x-grok-client-version"], "1.0.34"); + assert.equal(upstream.headers["x-grok-client-identifier"], "grok-shell"); + assert.equal(upstream.headers["x-grok-model-override"], "grok-4.6"); + assert.equal("cookie" in upstream.headers, false); + assert.throws(() => authorizeGrokBrokerProxyRequest({ ...input, method: "GET", pathname: "/" }, caps, "real-bearer"), /rejected/); }); -test("proxy fails closed when the worker omits or malforms the Grok client version", () => { - for (const version of [undefined, "", "1", "1.0", "v1.0.25", "1.0.25\nInjected: yes", "1.0.25+build", `1.2.3-${"a".repeat(65)}`] as const) { - const caps = new EngineBrokerCapabilities(); const opaque = caps.issue("a", "t"); - assert.throws(() => authorizeGrokBrokerProxyRequest({ method: "POST", pathname: "/v1/chat/completions", headers: { authorization: `Bearer ${opaque}`, ...(version === undefined ? {} : { "x-grok-client-version": version }) }, body, agentId: "a", turnId: "t" }, caps, "real-bearer"), /rejected/); +test("proxy fails closed on any client version other than the pinned Grok CLI", () => { + for (const version of [undefined, "", "1", "1.0", "v1.0.34", "1.0.13", "1.0.25", "1.0.33", "1.0.35", "1.0.34-beta.1", "1.0.34\nInjected: yes", "1.0.34+build"] as const) { + const { caps, input } = request(leanBody(), { "x-grok-client-version": version }); + assert.throws(() => authorizeGrokBrokerProxyRequest(input, caps, "real-bearer"), /rejected/, String(version)); } }); -test("proxy accepts prerelease Grok client versions", () => { - const caps = new EngineBrokerCapabilities(); const opaque = caps.issue("a", "t"); - const request = authorizeGrokBrokerProxyRequest({ method: "POST", pathname: "/v1/chat/completions", headers: { authorization: `Bearer ${opaque}`, "x-grok-client-version": "1.0.25-beta.1" }, body, agentId: "a", turnId: "t" }, caps, "real-bearer"); - assert.equal(request.headers["x-grok-client-version"], "1.0.25-beta.1"); +test("proxy refuses a request carrying Grok's fail-open full tool set before any upstream call", () => { + const full = [...leanTools, "search_replace", "kill_command_or_subagent", "todo_write", "get_command_or_subagent_output", "spawn_subagent", "scheduler_create", "scheduler_delete", "scheduler_list", "monitor", "workflow", "enter_plan_mode", "exit_plan_mode", "write"]; + for (const tools of [full.map(tool), [tool("session_title")], leanTools.slice(1).map(tool), [...leanTools, "use_tool"].map(tool), [...leanTools.slice(1), "run_terminal_cmd"].map(tool), undefined, [], leanTools.map((name) => ({ type: "custom", function: { name } }))]) { + const { caps, input } = request(leanBody({ tools })); + assert.throws(() => authorizeGrokBrokerProxyRequest(input, caps, "real-bearer"), /rejected/); + } +}); + +test("proxy refuses a reasoning effort or model other than the declared policy", () => { + for (const overrides of [{ reasoning_effort: "high" }, { reasoning_effort: undefined }, { reasoning_effort: "xhigh" }, { model: "grok-build" }, { model: "daimon-broker-grok" }]) { + const { caps, input } = request(leanBody(overrides)); + assert.throws(() => authorizeGrokBrokerProxyRequest(input, caps, "real-bearer"), /rejected/, JSON.stringify(overrides)); + } + const { caps, input } = request(leanBody({ model: "grok-build", reasoning_effort: "medium" })); + const upstream = authorizeGrokBrokerProxyRequest(input, caps, "real-bearer", { model: "grok-build", reasoningEffort: "medium" }); + assert.equal(upstream.headers["x-grok-model-override"], "grok-build"); + const other = request(leanBody()); + assert.throws(() => authorizeGrokBrokerProxyRequest(other.input, other.caps, "real-bearer", { model: "grok-3" as never, reasoningEffort: "low" }), /model policy/); }); diff --git a/src/runtime/grokBrokerProxyRequest.ts b/src/runtime/grokBrokerProxyRequest.ts index 15f79d4..309ad7e 100644 --- a/src/runtime/grokBrokerProxyRequest.ts +++ b/src/runtime/grokBrokerProxyRequest.ts @@ -1,12 +1,26 @@ +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import { GROK_WORKER_VISIBLE_TOOLS } from "../contracts/grokWorkerContract.js"; import type { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; +import { DEFAULT_GROK_BROKER_MODEL_POLICY, parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; const MAX_BODY = 2 * 1024 * 1024; -const MAX_CLIENT_VERSION = 64; -const CLIENT_VERSION = /^(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-[0-9A-Za-z.-]+)?$/u; export type GrokBrokerProxyInput = Readonly<{ method: string; pathname: string; headers: Readonly>; body: Uint8Array; agentId?: string; turnId?: string }>; export type GrokBrokerUpstreamRequest = Readonly<{ url: "https://cli-chat-proxy.grok.com/v1/chat/completions"; headers: Readonly>; body: Uint8Array }>; -export function authorizeGrokBrokerProxyRequest(input: GrokBrokerProxyInput, capabilities: EngineBrokerCapabilities, bearer: string): GrokBrokerUpstreamRequest { +/** + * Authorizes one worker request and rebuilds it for the provider. + * + * Everything that decides spend is checked here, before a bearer is attached + * and before any upstream call: + * - the client version is exactly the pinned Grok CLI (`GROK_ENGINE_BROKER.grokCliVersion`); + * - the body model and `reasoning_effort` are exactly the declared policy, and + * the model override header follows that declaration instead of a constant; + * - the offered tool names are exactly the lean visible set. Grok 1.0.34 turns + * an unmappable `--tools` entry into its full 19-tool set, and its per-turn + * `session_title` request carries a single forced tool; both are refused. + */ +export function authorizeGrokBrokerProxyRequest(input: GrokBrokerProxyInput, capabilities: EngineBrokerCapabilities, bearer: string, policy: GrokBrokerModelPolicy = DEFAULT_GROK_BROKER_MODEL_POLICY): GrokBrokerUpstreamRequest { + const declared = parseGrokBrokerModelPolicy(policy); if (input.method !== "POST" || input.pathname !== "/v1/chat/completions" || input.body.byteLength < 2 || input.body.byteLength > MAX_BODY) throw new Error("broker proxy request rejected"); const authorization = input.headers.authorization; const match = authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u); if (match === null || match === undefined) throw new Error("broker proxy request rejected"); @@ -14,7 +28,21 @@ export function authorizeGrokBrokerProxyRequest(input: GrokBrokerProxyInput, cap if (scope === undefined || (input.agentId !== undefined && scope.agentId !== input.agentId) || (input.turnId !== undefined && scope.turnId !== input.turnId)) throw new Error("broker proxy request rejected"); if (!bearer || /[\r\n]/u.test(bearer)) throw new Error("broker credential authority unavailable"); const clientVersion = input.headers["x-grok-client-version"]; - if (clientVersion === undefined || clientVersion.length > MAX_CLIENT_VERSION || !CLIENT_VERSION.test(clientVersion)) throw new Error("broker proxy request rejected"); - try { const parsed = JSON.parse(Buffer.from(input.body).toString("utf8")) as Record; if (parsed.stream !== true || !Array.isArray(parsed.messages)) throw new Error(); } catch { throw new Error("broker proxy request rejected"); } - return { url: "https://cli-chat-proxy.grok.com/v1/chat/completions", headers: { authorization: `Bearer ${bearer}`, "content-type": "application/json", "x-xai-token-auth": "xai-grok-cli", "x-grok-model-override": "grok-build", "x-grok-client-version": clientVersion, "x-grok-client-identifier": "grok-shell" }, body: input.body }; + if (clientVersion !== GROK_ENGINE_BROKER.grokCliVersion) throw new Error("broker proxy request rejected"); + let parsed: Record; + try { parsed = JSON.parse(Buffer.from(input.body).toString("utf8")) as Record; } catch { throw new Error("broker proxy request rejected"); } + if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed) || parsed.stream !== true || !Array.isArray(parsed.messages)) throw new Error("broker proxy request rejected"); + if (parsed.model !== declared.model || parsed.reasoning_effort !== declared.reasoningEffort) throw new Error("broker proxy request rejected"); + if (!exactLeanTools(parsed.tools)) throw new Error("broker proxy request rejected"); + return { url: "https://cli-chat-proxy.grok.com/v1/chat/completions", headers: { authorization: `Bearer ${bearer}`, "content-type": "application/json", "x-xai-token-auth": "xai-grok-cli", "x-grok-model-override": declared.model, "x-grok-client-version": clientVersion, "x-grok-client-identifier": "grok-shell" }, body: input.body }; +} + +export function exactLeanTools(tools: unknown): boolean { + if (!Array.isArray(tools) || tools.length !== GROK_WORKER_VISIBLE_TOOLS.length) return false; + const names = tools.map((tool) => { + if (tool === null || typeof tool !== "object" || (tool as { type?: unknown }).type !== "function") return undefined; + const fn = (tool as { function?: unknown }).function; + return fn !== null && typeof fn === "object" ? (fn as { name?: unknown }).name : undefined; + }); + return JSON.stringify([...names].sort()) === JSON.stringify(GROK_WORKER_VISIBLE_TOOLS); } From 8dbd8c04a9d7ec055a166f446dcea3234e2745f9 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 04:46:18 +0200 Subject: [PATCH 005/124] feat: attest Grok 1.0.34 sandbox events, deny lists and read-only worker home --- src/pi/grokSandbox.test.ts | 7 +- src/pi/grokSandbox.ts | 17 ++--- src/runtime/engineBrokerServiceCli.test.ts | 5 +- src/runtime/engineBrokerServiceCli.ts | 3 +- .../fixtures/grok-1.0.34-sandbox-events.jsonl | 2 + src/runtime/grokEngineBroker.ts | 12 ++-- src/runtime/grokWorkerAttestation.test.ts | 32 ++++++++- src/runtime/grokWorkerAttestation.ts | 37 +++++++---- src/runtime/grokWorkerHomeAttestation.test.ts | 62 ++++++++++++++++++ src/runtime/grokWorkerHomeAttestation.ts | 61 +++++++++++++++++ src/runtime/grokWorkerSandboxProfile.test.ts | 26 ++++++++ src/runtime/grokWorkerSandboxProfile.ts | Bin 0 -> 2154 bytes 12 files changed, 230 insertions(+), 34 deletions(-) create mode 100644 src/runtime/fixtures/grok-1.0.34-sandbox-events.jsonl create mode 100644 src/runtime/grokWorkerHomeAttestation.test.ts create mode 100644 src/runtime/grokWorkerHomeAttestation.ts create mode 100644 src/runtime/grokWorkerSandboxProfile.test.ts create mode 100644 src/runtime/grokWorkerSandboxProfile.ts diff --git a/src/pi/grokSandbox.test.ts b/src/pi/grokSandbox.test.ts index d8bd5d0..b34f194 100644 --- a/src/pi/grokSandbox.test.ts +++ b/src/pi/grokSandbox.test.ts @@ -45,7 +45,8 @@ test("rejects a protected root overlapping the selected agent workspace", async test("rotates its private enforcement receipt before the bounded log is exhausted", async () => { const fixture = await createFixture(); try { - const events = path.join(fixture.engineHomePath, "sandbox-events.jsonl"); + const events = path.join(fixture.engineHomePath, "sessions", "sandbox-events.jsonl"); + await mkdir(path.dirname(events), { mode: 0o700 }); await writeFile(events, "x".repeat(8 * 1024 * 1024), { mode: 0o600 }); await prepareAndVerifyGrokSandbox(fixture.authority); assert.ok((await readFile(events)).byteLength < 64 * 1024); @@ -75,8 +76,8 @@ const profile=fs.readFileSync(path.join(home,"sandbox.toml"),"utf8"); const deny=JSON.parse(profile.split("\\n").find((line)=>line.startsWith("deny = ")).slice(7)); const observed=${JSON.stringify(mode)}==="drop-deny"?deny.slice(1):deny; const event={event_type:"ProfileApplied",profile:"${GROK_DAIMON_SANDBOX_PROFILE}",workspace:fs.realpathSync(args[args.indexOf("--cwd")+1]),platform:"linux/landlock",enforced:${JSON.stringify(mode)}!=="unenforced",restrict_network:true,deny_paths:observed}; -fs.appendFileSync(path.join(home,"sandbox-events.jsonl"),JSON.stringify(event)+"\\n",{mode:0o600}); -fs.chmodSync(path.join(home,"sandbox-events.jsonl"),0o600); +fs.appendFileSync(path.join(home,"sessions","sandbox-events.jsonl"),JSON.stringify(event)+"\\n",{mode:0o600}); +fs.chmodSync(path.join(home,"sessions","sandbox-events.jsonl"),0o600); `); await chmod(command, 0o700); return { diff --git a/src/pi/grokSandbox.ts b/src/pi/grokSandbox.ts index 956095b..ddc9ee4 100644 --- a/src/pi/grokSandbox.ts +++ b/src/pi/grokSandbox.ts @@ -1,17 +1,19 @@ import { randomUUID } from "node:crypto"; import { spawn } from "node:child_process"; import { constants } from "node:fs"; -import { lstat, open, realpath, rename, unlink } from "node:fs/promises"; +import { lstat, mkdir, open, realpath, rename, unlink } from "node:fs/promises"; import path from "node:path"; import { readChild } from "./cliChildOutput.js"; import { cliChildEnvironment } from "./cliEnvironment.js"; import { terminateChild, trackCliChild } from "./cliProcess.js"; import { renderGrokSandboxArgs } from "./cliEngineSpawn.js"; +import { GROK_WORKER_SANDBOX_EVENTS_RELATIVE_PATH, GROK_WORKER_SANDBOX_PROFILE, renderGrokWorkerSandboxProfile } from "../runtime/grokWorkerSandboxProfile.js"; -export const GROK_DAIMON_SANDBOX_PROFILE = "daimon-strict"; +export const GROK_DAIMON_SANDBOX_PROFILE = GROK_WORKER_SANDBOX_PROFILE; const SANDBOX_CONFIG = "sandbox.toml"; -const SANDBOX_EVENTS = "sandbox-events.jsonl"; +/** Grok 1.0.34 logs sandbox events under `sessions/`; the root file stays empty. */ +const SANDBOX_EVENTS = GROK_WORKER_SANDBOX_EVENTS_RELATIVE_PATH; const MAX_EVENTS_BYTES = 16 * 1024 * 1024; const ROTATE_EVENTS_BYTES = 8 * 1024 * 1024; @@ -61,13 +63,7 @@ export async function prepareAndVerifyGrokSandbox( await verifyProfile(engineHome, denied); } -const profileText = (denied: readonly string[]): string => [ - `[profiles.${GROK_DAIMON_SANDBOX_PROFILE}]`, - 'extends = "strict"', - "restrict_network = true", - `deny = [${denied.map((entry) => JSON.stringify(entry)).join(", ")}]`, - "" -].join("\n"); +const profileText = (denied: readonly string[]): string => renderGrokWorkerSandboxProfile(denied); async function writeProfile(engineHome: string, denied: readonly string[]): Promise { const target = path.join(engineHome, SANDBOX_CONFIG); @@ -155,6 +151,7 @@ async function eventFileSize(file: string): Promise { return Number(entry.size); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") { + await mkdir(path.dirname(file), { mode: 0o700, recursive: true }); const handle = await open(file, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | noFollow(), 0o600); try { await handle.sync(); } finally { await handle.close(); } await syncDirectory(path.dirname(file)); diff --git a/src/runtime/engineBrokerServiceCli.test.ts b/src/runtime/engineBrokerServiceCli.test.ts index 7e5ff97..5d11cf6 100644 --- a/src/runtime/engineBrokerServiceCli.test.ts +++ b/src/runtime/engineBrokerServiceCli.test.ts @@ -11,5 +11,8 @@ test("rejects caller-selected commands, duplicate identities, and traversal",()= assert.throws(()=>parseEngineBrokerServiceConfig({...base,grokCommand:"evil"})); assert.throws(()=>parseEngineBrokerServiceConfig({...base,turnStore:"/var/lib/../secret"})); assert.throws(()=>parseEngineBrokerServiceConfig({...base,registrations:[reg("agent-a",0),reg("agent-a",1)]})); + // Grok 1.0.34 logs sandbox events under $GROK_HOME/sessions/; the 1.0.13 root path stays empty and must not be attested. + assert.throws(()=>parseEngineBrokerServiceConfig({...base,registrations:[{...reg("agent-a",0),eventsPath:"/workers/0/.grok/sandbox-events.jsonl"}]})); + assert.throws(()=>parseEngineBrokerServiceConfig({...base,registrations:[{...reg("agent-a",0),eventsPath:"/workers/1/.grok/sessions/sandbox-events.jsonl"}]})); }); -const reg=(agentId:string,slot:number)=>({agentId,slot,workerUid:2200+slot,workspace:`/workspace/${slot}`,profilePath:`/workers/${slot}/.grok/sandbox.toml`,eventsPath:`/workers/${slot}/.grok/sandbox-events.jsonl`,profileSha256:"a".repeat(64)}); +const reg=(agentId:string,slot:number)=>({agentId,slot,workerUid:2200+slot,workspace:`/workspace/${slot}`,profilePath:`/workers/${slot}/.grok/sandbox.toml`,eventsPath:`/workers/${slot}/.grok/sessions/sandbox-events.jsonl`,profileSha256:"a".repeat(64)}); diff --git a/src/runtime/engineBrokerServiceCli.ts b/src/runtime/engineBrokerServiceCli.ts index bbc89cd..d70839c 100644 --- a/src/runtime/engineBrokerServiceCli.ts +++ b/src/runtime/engineBrokerServiceCli.ts @@ -2,6 +2,7 @@ import { constants } from "node:fs"; import { open } from "node:fs/promises"; import { startEngineBrokerService } from "./engineBrokerService.js"; import { startGrokEngineBroker, type GrokEngineBrokerRegistration } from "./grokEngineBroker.js"; +import { grokWorkerEventsPathFor } from "./grokWorkerSandboxProfile.js"; export const ENGINE_BROKER_SERVICE_CONFIG = "/etc/daimon-engine-broker/service.json"; const MAX_CONFIG_BYTES=65_536; @@ -19,7 +20,7 @@ export function parseEngineBrokerServiceConfig(value:unknown):Readonly<{credenti if(value===null||typeof value!=="object"||Array.isArray(value))throw new TypeError("invalid engine broker service config");const input=value as Record; if(Object.keys(input).length!==4||input.version!=="noopolis.daimon.engine-broker-service.v1"||typeof input.credentialHome!=="string"||typeof input.turnStore!=="string"||!Array.isArray(input.registrations))throw new TypeError("invalid engine broker service config"); const absolute=(item:string)=>item.startsWith("/")&&!item.includes("/../")&&!item.endsWith("/..");if(!absolute(input.credentialHome)||!absolute(input.turnStore))throw new TypeError("invalid engine broker service config"); - const seen=new Set();const registrations=input.registrations.map((entry)=>{if(entry===null||typeof entry!=="object"||Array.isArray(entry))throw new TypeError("invalid engine broker service config");const item=entry as Record;if(Object.keys(item).length!==7||typeof item.agentId!=="string"||!item.agentId.trim()||!Number.isSafeInteger(item.slot)||(item.slot as number)<0||!Number.isSafeInteger(item.workerUid)||(item.workerUid as number)<2200||typeof item.workspace!=="string"||!absolute(item.workspace)||typeof item.profilePath!=="string"||!absolute(item.profilePath)||typeof item.eventsPath!=="string"||!absolute(item.eventsPath)||typeof item.profileSha256!=="string"||!/^[a-f0-9]{64}$/u.test(item.profileSha256)||item.profilePath!==`${item.eventsPath.replace(/\/sandbox-events\.jsonl$/u,"")}/sandbox.toml`||seen.has(item.agentId))throw new TypeError("invalid engine broker service config");seen.add(item.agentId);return {agentId:item.agentId,slot:item.slot as number,workerUid:item.workerUid as number,workspace:item.workspace,profilePath:item.profilePath,eventsPath:item.eventsPath,profileSha256:item.profileSha256};}); + const seen=new Set();const registrations=input.registrations.map((entry)=>{if(entry===null||typeof entry!=="object"||Array.isArray(entry))throw new TypeError("invalid engine broker service config");const item=entry as Record;if(Object.keys(item).length!==7||typeof item.agentId!=="string"||!item.agentId.trim()||!Number.isSafeInteger(item.slot)||(item.slot as number)<0||!Number.isSafeInteger(item.workerUid)||(item.workerUid as number)<2200||typeof item.workspace!=="string"||!absolute(item.workspace)||typeof item.profilePath!=="string"||!absolute(item.profilePath)||typeof item.eventsPath!=="string"||!absolute(item.eventsPath)||typeof item.profileSha256!=="string"||!/^[a-f0-9]{64}$/u.test(item.profileSha256)||item.eventsPath!==grokWorkerEventsPathFor(item.profilePath)||!item.profilePath.endsWith("/sandbox.toml")||seen.has(item.agentId))throw new TypeError("invalid engine broker service config");seen.add(item.agentId);return {agentId:item.agentId,slot:item.slot as number,workerUid:item.workerUid as number,workspace:item.workspace,profilePath:item.profilePath,eventsPath:item.eventsPath,profileSha256:item.profileSha256};}); if(registrations.length===0)throw new TypeError("invalid engine broker service config");return {credentialHome:input.credentialHome,turnStore:input.turnStore,registrations}; } diff --git a/src/runtime/fixtures/grok-1.0.34-sandbox-events.jsonl b/src/runtime/fixtures/grok-1.0.34-sandbox-events.jsonl new file mode 100644 index 0000000..a728cef --- /dev/null +++ b/src/runtime/fixtures/grok-1.0.34-sandbox-events.jsonl @@ -0,0 +1,2 @@ +{"timestamp":"2026-09-17T02:33:59.302698678Z","event_type":"ProfileApplied","profile":"daimon-strict","workspace":"/var/lib/spawnfile/instance/workspace/agents/a1","platform":"linux/landlock","enforced":true,"restrict_network":true,"read_write_paths":["/var/lib/spawnfile/instance/workspace/agents/a1","/var/lib/daimon-workers/2200/.grok/sessions","/tmp","/var/tmp"],"read_only_paths":["/usr","/lib","/bin","/sbin","/etc","/dev","/proc","/sys","/tmp","/run","/var","/var/lib/spawnfile/instance/workspace/agents/a1","/var/lib/daimon-workers/2200/.grok"],"deny_paths":["/run/paideia"]} +{"timestamp":"2026-09-17T02:33:59.913077091Z","event_type":"FsViolation","profile":"daimon-strict","operation":"read","target":"/run/paideia/context.json"} diff --git a/src/runtime/grokEngineBroker.ts b/src/runtime/grokEngineBroker.ts index 6ae6eaf..bfcc3ec 100644 --- a/src/runtime/grokEngineBroker.ts +++ b/src/runtime/grokEngineBroker.ts @@ -7,6 +7,8 @@ import { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; import { startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; import { acquireGrokBrokerRealmLease } from "./grokBrokerRealmLease.js"; +import { grokBrokerWorkerConfigSha256 } from "./grokBrokerWorkerConfig.js"; +import { parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; import { GrokWorkerAttestationFailure,prepareGrokWorkerAttestation,verifyGrokWorkerAttestation } from "./grokWorkerAttestation.js"; export type GrokEngineBrokerRegistration = Readonly<{ agentId:string;slot:number;workerUid:number;workspace:string;profilePath:string;eventsPath:string;profileSha256:string }>; @@ -32,18 +34,20 @@ export async function finishBrokerTurnWithUsage(turns: EngineBrokerTurnRegistry, if (usage === undefined) return; await recordTurnUsage(usageLedgerPath, { agent: agentId, wake: wakeId, engine: "grok", usage }); } -export async function startGrokEngineBroker(options: Readonly<{ grokCommand: string; nativeClient: string; credentialHome: string; turnStore: string; registrations: readonly GrokEngineBrokerRegistration[]; usageLedgerPath?: string }>) { +export async function startGrokEngineBroker(options: Readonly<{ grokCommand: string; nativeClient: string; credentialHome: string; turnStore: string; registrations: readonly GrokEngineBrokerRegistration[]; usageLedgerPath?: string; modelPolicy?: GrokBrokerModelPolicy }>) { + // One declared model/effort drives both the worker config bytes the broker attests and the bodies the proxy forwards. + const modelPolicy = parseGrokBrokerModelPolicy(options.modelPolicy ?? {}); const configSha256 = grokBrokerWorkerConfigSha256(modelPolicy); const usageLedgerPath = options.usageLedgerPath ?? TURN_USAGE_LEDGER.filePath; const registrations = new Map(options.registrations.map((entry) => [entry.agentId, entry])); if (registrations.size !== options.registrations.length) throw new Error("engine broker registration conflict"); - const lease=await acquireGrokBrokerRealmLease(options.credentialHome);const authority = new DurableGrokBrokerCredentialAuthority(options.grokCommand, options.credentialHome);try{await authority.initialize();}catch(error){await lease.close();throw error;} let proxy:Awaited>;try{proxy=await startGrokBrokerProxy(authority);}catch(error){await lease.close();throw error;}let mcp:Awaited>|undefined;try{mcp=await startEngineBrokerMcpFacade();for(const registration of registrations.values())await prepareGrokWorkerAttestation({...registration,brokerGid:2100});}catch(error){if(mcp)await mcp.close().catch(()=>undefined);await proxy.close();await lease.close();throw error;}if(!mcp)throw new Error("engine broker unavailable");const turns = new EngineBrokerTurnRegistry(options.turnStore); const active = new Map; resolve: () => void }>(); let closed = false; + const lease=await acquireGrokBrokerRealmLease(options.credentialHome);const authority = new DurableGrokBrokerCredentialAuthority(options.grokCommand, options.credentialHome);try{await authority.initialize();}catch(error){await lease.close();throw error;} let proxy:Awaited>;try{proxy=await startGrokBrokerProxy(authority,undefined,modelPolicy);}catch(error){await lease.close();throw error;}let mcp:Awaited>|undefined;try{mcp=await startEngineBrokerMcpFacade();for(const registration of registrations.values())await prepareGrokWorkerAttestation({...registration,brokerGid:2100,configSha256});}catch(error){if(mcp)await mcp.close().catch(()=>undefined);await proxy.close();await lease.close();throw error;}if(!mcp)throw new Error("engine broker unavailable");const turns = new EngineBrokerTurnRegistry(options.turnStore); const active = new Map; resolve: () => void }>(); let closed = false; return { async turn(agentId: string, wakeId: string, prompt: string, mcpEndpoint: string, signal?: AbortSignal): Promise> { if (closed) throw new Error("engine broker unavailable"); const registration = registrations.get(agentId); if (registration === undefined) throw new Error("engine broker unavailable"); const turnId = createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"); const request = { version: "noopolis.daimon.engine-broker.v1", kind: "start_turn", requestId: randomUUID(), turnId, agentId, wakeId, prompt,mcpEndpoint } as const; const begun = await turns.begin(request); if (begun !== "start") { if (begun.replay.kind === "completed") return {text:begun.replay.text,workerPid:begun.replay.workerPid,workerUid:begun.replay.workerUid,workerStartTime:begun.replay.workerStartTime};const code=begun.replay.code==="auth_stale"||begun.replay.code==="cancelled"?begun.replay.code:"engine_failed";throw new EngineBrokerTurnFailure(code,begun.replay.diagnostic as NativeBrokerDiagnostic|undefined); } - const isolation=await prepareGrokWorkerAttestation({...registration,brokerGid:2100});proxy.registerIsolationGuard(turnId,()=>verifyGrokWorkerAttestation({...registration,brokerGid:2100},isolation));const providerCapability = proxy.capabilities.issue(agentId, turnId);const mcpCapability=mcp.register(agentId,turnId,mcpEndpoint); const controller = new AbortController();const onAbort=()=>controller.abort();signal?.addEventListener("abort",onAbort,{once:true});if(signal?.aborted)controller.abort(); let resolve!:()=>void;const done=new Promise((value)=>{resolve=value;});active.set(turnId,{controller,done,resolve}); + const attestation={...registration,brokerGid:2100,configSha256};const isolation=await prepareGrokWorkerAttestation(attestation);proxy.registerIsolationGuard(turnId,()=>verifyGrokWorkerAttestation(attestation,isolation));const providerCapability = proxy.capabilities.issue(agentId, turnId);const mcpCapability=mcp.register(agentId,turnId,mcpEndpoint); const controller = new AbortController();const onAbort=()=>controller.abort();signal?.addEventListener("abort",onAbort,{once:true});if(signal?.aborted)controller.abort(); let resolve!:()=>void;const done=new Promise((value)=>{resolve=value;});active.set(turnId,{controller,done,resolve}); let nativeDiagnostic:NativeBrokerDiagnostic|undefined,attested=false; - try { const result = await runNativeBrokerTurn(options.nativeClient, { slot: registration.slot, requestId: request.requestId, turnId, agentId, wakeId, prompt, providerCapability,mcpCapability }, controller.signal);nativeDiagnostic=result.diagnostic;if(result.workerUid!==registration.workerUid)throw new Error();await verifyGrokWorkerAttestation({...registration,brokerGid:2100},isolation);attested=true; const decoded = decodeGrokHeadlessTurn(result.text);const text = decoded.text;const completed={ version: request.version, kind: "completed", requestId: request.requestId, turnId, text, workerPid: result.workerPid, workerUid: result.workerUid, workerStartTime: result.startTicks.toString() } as const; await finishBrokerTurnWithUsage(turns,request,completed,usageLedgerPath,decoded.usage,agentId,wakeId); return {text,workerPid:result.workerPid,workerUid:result.workerUid,workerStartTime:result.startTicks.toString()}; } + try { const result = await runNativeBrokerTurn(options.nativeClient, { slot: registration.slot, requestId: request.requestId, turnId, agentId, wakeId, prompt, providerCapability,mcpCapability }, controller.signal);nativeDiagnostic=result.diagnostic;if(result.workerUid!==registration.workerUid)throw new Error();await verifyGrokWorkerAttestation(attestation,isolation);attested=true; const decoded = decodeGrokHeadlessTurn(result.text);const text = decoded.text;const completed={ version: request.version, kind: "completed", requestId: request.requestId, turnId, text, workerPid: result.workerPid, workerUid: result.workerUid, workerStartTime: result.startTicks.toString() } as const; await finishBrokerTurnWithUsage(turns,request,completed,usageLedgerPath,decoded.usage,agentId,wakeId); return {text,workerPid:result.workerPid,workerUid:result.workerUid,workerStartTime:result.startTicks.toString()}; } catch(error) { const code=authority.isStale()?"auth_stale":controller.signal.aborted?"cancelled":"engine_failed";const diagnostic=error instanceof NativeBrokerTurnFailure?error.diagnostic:nativeDiagnostic&&!attested?{...nativeDiagnostic,status:"worker_failed" as const,stage:"attestation" as const,failureClass:error instanceof GrokWorkerAttestationFailure?error.failureClass:"profile_invalid" as const,profileApplied:false}:undefined;await turns.finish(request, { version: request.version, kind: "failed", requestId: request.requestId, turnId, code,...(diagnostic?{diagnostic}:{}) }); throw new EngineBrokerTurnFailure(code,diagnostic); } finally { signal?.removeEventListener("abort",onAbort);active.get(turnId)?.resolve(); active.delete(turnId); proxy.revokeIsolationGuard(turnId);proxy.capabilities.revoke(turnId);mcp.revoke(turnId); } }, diff --git a/src/runtime/grokWorkerAttestation.test.ts b/src/runtime/grokWorkerAttestation.test.ts index 7dc230d..7c1b30a 100644 --- a/src/runtime/grokWorkerAttestation.test.ts +++ b/src/runtime/grokWorkerAttestation.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; -import { appendFile, chmod, mkdtemp, rm, stat, symlink, truncate, writeFile } from "node:fs/promises"; +import { appendFile, chmod, mkdir, mkdtemp, readFile, rm, stat, symlink, truncate, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import test from "node:test"; @@ -220,8 +220,9 @@ test("refuses a sandbox profile that is not root-owned, and one reached through const text = `[profiles.daimon-strict]\nextends = "strict"\nrestrict_network = true\ndeny = []\n`; await writeFile(profile, text); await chmod(profile, 0o444); - const events = await eventsFile(dir); - const input = { profilePath: profile, eventsPath: events, profileSha256: sha256(text), workerUid: self.uid, brokerGid: self.gid }; + await mkdir(path.join(dir, "sessions")); + const events = await eventsFile(path.join(dir, "sessions")); + const input = { profilePath: profile, eventsPath: events, profileSha256: sha256(text), workerUid: self.uid, brokerGid: self.gid, configSha256: "0".repeat(64) }; // Owned by the test user rather than root: `secureOpen` must refuse it even // though its bytes hash correctly. await assert.rejects(prepareGrokWorkerAttestation(input), /attestation unavailable/u); @@ -244,3 +245,28 @@ test("holds the kernel's reported deny_paths to exactly what the pinned profile } assert.throws(() => parseGrokWorkerProfileApplied(bytes(["/a"]), workspace, []), /attestation unavailable/u); }); + +test("reads sandbox events only from the Grok 1.0.34 sessions log", async (t) => { + // Mutation-critical: 1.0.34 writes nothing to `$GROK_HOME/sandbox-events.jsonl`, + // so attesting that path would fail every turn as "not enforced" — or worse, + // accept a stale file. Restoring the old relation must turn this red. + const dir = await workspaceDir(); + t.after(() => rm(dir, { recursive: true, force: true })); + const input = { profilePath: path.join(dir, "sandbox.toml"), profileSha256: "0".repeat(64), workerUid: self.uid, brokerGid: self.gid, configSha256: "0".repeat(64) }; + await assert.rejects(prepareGrokWorkerAttestation({ ...input, eventsPath: path.join(dir, "sandbox-events.jsonl") }), /sessions\/sandbox-events\.jsonl/u); + await assert.rejects(prepareGrokWorkerAttestation({ ...input, eventsPath: path.join(dir, "sessions", "sandbox-events.jsonl") }), /attestation unavailable/u); +}); + +test("attests real Grok 1.0.34 events: ProfileApplied with a non-empty deny list, then the denial it caused", async (t) => { + const fixture = await readFile(new URL("./fixtures/grok-1.0.34-sandbox-events.jsonl", import.meta.url)); + const liveWorkspace = "/var/lib/spawnfile/instance/workspace/agents/a1"; + assert.doesNotThrow(() => parseGrokWorkerProfileApplied(fixture, liveWorkspace, ["/run/paideia"])); + assert.throws(() => parseGrokWorkerProfileApplied(fixture, liveWorkspace, []), /attestation unavailable/u); + assert.throws(() => parseGrokWorkerProfileApplied(fixture, liveWorkspace, ["/run/paideia", "/run/training"]), /attestation unavailable/u); + const dir = await workspaceDir(); + t.after(() => rm(dir, { recursive: true, force: true })); + const file = await eventsFile(dir); + const before = await watermark(file, ["/run/paideia"]); + await appendFile(file, fixture); + await verifyGrokWorkerAttestation({ eventsPath: file, workerUid: self.uid, brokerGid: self.gid, workspace: liveWorkspace }, before); +}); diff --git a/src/runtime/grokWorkerAttestation.ts b/src/runtime/grokWorkerAttestation.ts index ce450f3..70eae1b 100644 --- a/src/runtime/grokWorkerAttestation.ts +++ b/src/runtime/grokWorkerAttestation.ts @@ -1,6 +1,10 @@ import { createHash } from "node:crypto"; import { constants } from "node:fs"; import { lstat,open } from "node:fs/promises"; +import path from "node:path"; + +import { verifyGrokWorkerHome } from "./grokWorkerHomeAttestation.js"; +import { grokWorkerEventsPathFor } from "./grokWorkerSandboxProfile.js"; /** * The per-turn freshness watermark taken before the worker is launched: the @@ -15,15 +19,13 @@ export class GrokWorkerAttestationFailure extends Error { constructor(readonly f * Reads the `deny` list out of a worker sandbox profile, but only after the * bytes match `profileSha256` exactly. * - * There is no minimum length. The floor used to be 3 (subscription realm, - * bootstrap credential, peer roots), then 1, and an empty list is now the - * *expected* shape: Grok 1.0.13 re-execs itself inside bubblewrap whenever - * `deny` is non-empty and then opens each deny-path placeholder — which it - * created at mode 000 — from a capability-stripped process, gets EACCES, and - * refuses to start ("possible __GROK_INSIDE_BWRAP spoof") without ever - * emitting a `ProfileApplied` event. Spawnfile therefore renders `deny = []` - * and confines the worker with unix permissions plus builtin-`strict` - * Landlock instead (`containerDaimonBrokerRender.ts`). + * There is no minimum length, and a populated list is supported. Grok 1.0.13 + * refused to start on any non-empty `deny` (its mode-000 placeholders failed + * an EACCES "__GROK_INSIDE_BWRAP spoof" check), so deployments rendered + * `deny = []`. Grok 1.0.34 runs every profile inside bubblewrap and enforces a + * non-empty list, and since its strict base reads all of `/run`, `/var` and + * `/tmp`, the deny list is what keeps evaluator and host-bind paths away from + * the worker. `grokWorkerSandboxProfile.ts` renders these bytes. * * The integrity guarantee is the hash pin, not the length. `profileSha256` * comes from `/etc/daimon-engine-broker/service.json`, which the root @@ -52,8 +54,17 @@ export function parseGrokWorkerSandboxProfile(bytes:Uint8Array,profileSha256:str if(!Array.isArray(parsed)||parsed.some((entry)=>typeof entry!=="string")||new Set(parsed).size!==parsed.length)throw new Error("Grok worker isolation attestation unavailable"); return [...parsed as string[]].sort(); } -export async function prepareGrokWorkerAttestation(input:Readonly<{profilePath:string;eventsPath:string;profileSha256:string;workerUid:number;brokerGid:number}>):Promise{ +/** + * Pre-launch half of the per-turn attestation. Besides the pinned profile and + * the events watermark it requires the 1.0.34 layout: events under + * `$GROK_HOME/sessions/` (the root `sandbox-events.jsonl` stays empty on + * 1.0.34), and a root-owned read-only worker home whose `config.toml` hashes to + * the declared renderer output (`grokWorkerHomeAttestation.ts`). + */ +export async function prepareGrokWorkerAttestation(input:Readonly<{profilePath:string;eventsPath:string;profileSha256:string;workerUid:number;brokerGid:number;configSha256:string}>):Promise{ + if(input.eventsPath!==grokWorkerEventsPathFor(input.profilePath))throw new Error("Grok worker sandbox events must be read from $GROK_HOME/sessions/sandbox-events.jsonl"); const profile=await secureOpen(input.profilePath,0,0,0o444,65_536);let bytes:Buffer|undefined;let denyPaths:readonly string[]=[];try{bytes=await profile.readFile();denyPaths=parseGrokWorkerSandboxProfile(bytes,input.profileSha256);}catch{throw new Error("Grok worker isolation attestation unavailable");}finally{bytes?.fill(0);await profile.close();} + await verifyGrokWorkerHome(path.dirname(input.profilePath),input.configSha256); const events=await secureOpen(input.eventsPath,input.workerUid,input.brokerGid,0o640,16*1024*1024);try{const stat=await events.stat();return{dev:Number(stat.dev),ino:Number(stat.ino),size:Number(stat.size),mtimeMs:Number(stat.mtimeMs),denyPaths};}finally{await events.close();} } export async function verifyGrokWorkerAttestation(input:Readonly<{eventsPath:string;workerUid:number;brokerGid:number;workspace:string}>,before:Snapshot):Promise{ @@ -63,14 +74,16 @@ export async function verifyGrokWorkerAttestation(input:Readonly<{eventsPath:str * Requires one fully-conforming `ProfileApplied` event *somewhere* in the * fresh region — not as its last line. * - * `sandbox-events.jsonl` is not a profile log. Grok 1.0.13 writes its whole + * `sandbox-events.jsonl` is not a profile log. Grok (1.0.13 and 1.0.34) writes its whole * sandbox event vocabulary there — verified by reading the shipped binary: * `ProfileApplied, ApplyFailed, FsViolation, NetViolation, BypassGranted, * BypassDenied` (one contiguous enum blob beside the record fields * `timestamp, event_type, read_only_paths, deny_paths, operation, target, * command, tool_call_id`, emitted from `xai_grok_sandbox::logging`), and its * own embedded documentation says so outright: "Sandbox events (profile - * applied, violations) are logged to `~/.grok/sandbox-events.jsonl`". + * applied, violations) are logged to `~/.grok/sandbox-events.jsonl`" (1.0.34 + * moved the file to `~/.grok/sessions/`; a 1.0.34 turn logs `ProfileApplied` + * followed by an `FsViolation` for every denied read). * * Requiring `ProfileApplied` to be the *last* line therefore failed on the * first denied access of any turn: the violation Grok logged next became the diff --git a/src/runtime/grokWorkerHomeAttestation.test.ts b/src/runtime/grokWorkerHomeAttestation.test.ts new file mode 100644 index 0000000..1c2eceb --- /dev/null +++ b/src/runtime/grokWorkerHomeAttestation.test.ts @@ -0,0 +1,62 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { grokBrokerWorkerConfigSha256, renderGrokBrokerWorkerConfig } from "./grokBrokerWorkerConfig.js"; +import { assertGrokWorkerConfigBytes, assertGrokWorkerHomeEntries, verifyGrokWorkerHome } from "./grokWorkerHomeAttestation.js"; + +type Kind = "dir" | "file" | "link"; +const entry = (kind: Kind, mode: number, uid = 0, nlink = 1) => ({ + uid, mode: (kind === "dir" ? 0o040000 : kind === "file" ? 0o100000 : 0o120000) | mode, nlink, + isFile: () => kind === "file", isDirectory: () => kind === "dir", isSymbolicLink: () => kind === "link" +}); +const files = ["config.toml", "managed_config.toml", "requirements.toml", "sandbox.toml", "trusted_folders.toml"]; +const layout = (overrides: Record | undefined> = {}) => ({ + ".": entry("dir", 0o1771), sessions: entry("dir", 0o1771), ...Object.fromEntries(files.map((name) => [name, entry("file", 0o444)])), ...overrides +}); + +test("accepts the root-owned sticky worker home with root-owned read-only config and trust files", () => { + assert.doesNotThrow(() => assertGrokWorkerHomeEntries(layout())); + assert.doesNotThrow(() => assertGrokWorkerHomeEntries(layout({ ".": entry("dir", 0o711), sessions: entry("dir", 0o750) }))); +}); + +test("refuses a worker home where the worker could change its config or trust state", () => { + const refusals: Record> = { + "worker-owned home": layout({ ".": entry("dir", 0o1771, 2200) }), + "group-writable home without sticky bit": layout({ ".": entry("dir", 0o771) }), + "world-writable home": layout({ ".": entry("dir", 0o1777) }), + "worker-owned sessions": layout({ sessions: entry("dir", 0o700, 2200) }), + "group-writable sessions without sticky bit": layout({ sessions: entry("dir", 0o770) }) + }; + for (const name of files) { + refusals[`${name} missing`] = layout({ [name]: undefined }); + refusals[`${name} worker-owned`] = layout({ [name]: entry("file", 0o444, 2200) }); + refusals[`${name} owner-writable`] = layout({ [name]: entry("file", 0o644) }); + refusals[`${name} group-writable`] = layout({ [name]: entry("file", 0o464) }); + refusals[`${name} symlink`] = layout({ [name]: entry("link", 0o444) }); + refusals[`${name} hard-linked`] = layout({ [name]: entry("file", 0o444, 0, 2) }); + } + for (const [label, entries] of Object.entries(refusals)) assert.throws(() => assertGrokWorkerHomeEntries(entries), /attestation unavailable/u, label); +}); + +test("refuses a real home that is not root-owned even when every file exists", async (t) => { + const home = await mkdtemp(path.join(os.tmpdir(), "daimon-grok-home-")); + t.after(() => rm(home, { recursive: true, force: true })); + await mkdir(path.join(home, "sessions")); + for (const name of files) await writeFile(path.join(home, name), "", { mode: 0o444 }); + await assert.rejects(verifyGrokWorkerHome(home, "0".repeat(64)), /attestation unavailable/u); +}); + +test("accepts only the renderer's exact config bytes for the declared model policy", () => { + const declared = { model: "grok-4.6", reasoningEffort: "low" } as const; + const bytes = Buffer.from(renderGrokBrokerWorkerConfig(declared)); + assert.doesNotThrow(() => assertGrokWorkerConfigBytes(bytes, grokBrokerWorkerConfigSha256(declared))); + for (const tampered of [ + renderGrokBrokerWorkerConfig({ model: "grok-4.6", reasoningEffort: "high" }), + renderGrokBrokerWorkerConfig(declared).replace(/\[skills\]\ndisabled = \[[^\]]*\]\n/u, ""), + renderGrokBrokerWorkerConfig(declared).replace('session_summary = "daimon-session-title-disabled"', 'session_summary = "grok-4.6"'), + `${renderGrokBrokerWorkerConfig(declared)}\n[mcp_servers.extra]\nurl = "http://127.0.0.1:1/mcp"\n` + ]) assert.throws(() => assertGrokWorkerConfigBytes(Buffer.from(tampered), grokBrokerWorkerConfigSha256(declared)), /attestation unavailable/u); +}); diff --git a/src/runtime/grokWorkerHomeAttestation.ts b/src/runtime/grokWorkerHomeAttestation.ts new file mode 100644 index 0000000..f029757 --- /dev/null +++ b/src/runtime/grokWorkerHomeAttestation.ts @@ -0,0 +1,61 @@ +import { createHash } from "node:crypto"; +import { constants, type Stats } from "node:fs"; +import { lstat, open } from "node:fs/promises"; +import path from "node:path"; + +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; + +type Entry = Pick & Readonly<{ isFile(): boolean; isDirectory(): boolean; isSymbolicLink(): boolean }>; +const HOME = GROK_ENGINE_BROKER.worker.home; + +/** + * The worker must not be able to change what Grok reads before its next turn. + * + * Grok writes its own state (sessions, hooks, locks, docs) into `$GROK_HOME`, + * so the directory is worker-group writable — but root-owned and sticky, so a + * worker can neither rename nor unlink a root-owned file in it. Every file + * that decides a turn's behaviour is root-owned with no write bit: `config.toml` + * (model, effort, MCP, skills), `sandbox.toml`, `trusted_folders.toml` (trust + * would re-enable cwd `AGENTS.md` and project skills), and the managed and + * requirements layers that can override user config. A missing file fails + * too: the worker could create it. + * + * Pure so every refusal is testable without root. + */ +export function assertGrokWorkerHomeEntries(entries: Readonly>): void { + const directory = (entry: Entry | undefined): boolean => + entry !== undefined && entry.isDirectory() && !entry.isSymbolicLink() && entry.uid === HOME.directory.uid + && (Number(entry.mode) & 0o002) === 0 && ((Number(entry.mode) & 0o020) === 0 || (Number(entry.mode) & 0o1000) !== 0); + if (!directory(entries["."]) || !directory(entries[HOME.sessionsDirectory.relativePath])) throw unavailable(); + for (const name of HOME.readOnlyFiles.names) { + const entry = entries[name]; + if (entry === undefined || !entry.isFile() || entry.isSymbolicLink() || entry.uid !== HOME.readOnlyFiles.uid || entry.nlink !== 1 || (Number(entry.mode) & 0o7222) !== 0) throw unavailable(); + } +} + +/** The worker's `config.toml` must be exactly the renderer's bytes for the declared policy. */ +export function assertGrokWorkerConfigBytes(bytes: Uint8Array, configSha256: string): void { + if (!/^[0-9a-f]{64}$/u.test(configSha256) || createHash("sha256").update(bytes).digest("hex") !== configSha256) throw unavailable(); +} + +/** + * Attests the worker home layout and that `config.toml` is exactly the + * renderer's bytes for the declared model policy (`configSha256`). + */ +export async function verifyGrokWorkerHome(grokHome: string, configSha256: string): Promise { + const entries: Record = {}; + for (const name of [".", HOME.sessionsDirectory.relativePath, ...HOME.readOnlyFiles.names]) { + try { entries[name] = await lstat(path.join(grokHome, name)); } catch { entries[name] = undefined; } + } + assertGrokWorkerHomeEntries(entries); + let handle: Awaited> | undefined; + try { + handle = await open(path.join(grokHome, "config.toml"), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + const opened = await handle.stat(); + const before = entries["config.toml"]!; + if (!opened.isFile() || opened.size > 65_536 || opened.uid !== before.uid || opened.mode !== before.mode || opened.nlink !== 1) throw unavailable(); + assertGrokWorkerConfigBytes(await handle.readFile(), configSha256); + } catch { throw unavailable(); } finally { await handle?.close().catch(() => undefined); } +} + +const unavailable = (): Error => new Error("Grok worker isolation attestation unavailable"); diff --git a/src/runtime/grokWorkerSandboxProfile.test.ts b/src/runtime/grokWorkerSandboxProfile.test.ts new file mode 100644 index 0000000..7c24264 --- /dev/null +++ b/src/runtime/grokWorkerSandboxProfile.test.ts @@ -0,0 +1,26 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import test from "node:test"; + +import { parseGrokWorkerSandboxProfile } from "./grokWorkerAttestation.js"; +import { grokWorkerEventsPathFor, grokWorkerSandboxProfileSha256, renderGrokWorkerSandboxProfile } from "./grokWorkerSandboxProfile.js"; + +test("renders a non-empty deny list deterministically and round-trips through the attestation parser", () => { + const profile = renderGrokWorkerSandboxProfile(["/run/training/inputs", "/run/paideia", "/run/paideia"]); + assert.equal(profile, '[profiles.daimon-strict]\nextends = "strict"\nrestrict_network = true\ndeny = ["/run/paideia", "/run/training/inputs"]\n'); + assert.equal(renderGrokWorkerSandboxProfile(["/run/paideia", "/run/training/inputs"]), profile); + const digest = createHash("sha256").update(profile).digest("hex"); + assert.equal(grokWorkerSandboxProfileSha256(["/run/training/inputs", "/run/paideia"]), digest); + assert.deepEqual(parseGrokWorkerSandboxProfile(Buffer.from(profile), digest), ["/run/paideia", "/run/training/inputs"]); + assert.equal(renderGrokWorkerSandboxProfile(), '[profiles.daimon-strict]\nextends = "strict"\nrestrict_network = true\ndeny = []\n'); +}); + +test("refuses deny paths that are relative, non-canonical, root, globbed, or TOML-breaking", () => { + for (const entry of ["run/paideia", "/run/../etc", "/run/paideia/", "/", "/run/*", '/run/"x', "/run/a\nb", "/run/a\\b", ""]) { + assert.throws(() => renderGrokWorkerSandboxProfile([entry]), /deny path/u, JSON.stringify(entry)); + } +}); + +test("derives the Grok 1.0.34 sandbox events log from the profile location", () => { + assert.equal(grokWorkerEventsPathFor("/var/lib/daimon-workers/2200/.grok/sandbox.toml"), "/var/lib/daimon-workers/2200/.grok/sessions/sandbox-events.jsonl"); +}); diff --git a/src/runtime/grokWorkerSandboxProfile.ts b/src/runtime/grokWorkerSandboxProfile.ts new file mode 100644 index 0000000000000000000000000000000000000000..423fe217979515e49fe5b61d832766f700d5b48b GIT binary patch literal 2154 zcma)7;cnYT4Bqd@Ffq`;QDb=!{w?vSBw(>0BQ*k}eghIN+c=oyF1l!(_nju zJ>i~YT>vIb~U_wGq{|MPR?KaH2h_DaXEbX`fLDwhzcQ>T6L{6 zGItRO3*PPc%OHbygY)rdb~!jZ8NYrvm|dKVPZ`mowUR>jtWd>V-*xc=DreL8R;y|^ z!d~z2a1Rb)Tq2NZ4OTZMr{@J^e@iQa77A*k7qUW_uN~TyoZ0`C(YNqT`ZRs^9hgR0 zpnzr#XB58D`R&$8Dk}@}ZRYa|S4Pwwpd;W)Gzfx`6fuFz+N~)8RO<{XNer&ExwS`t zA}@g|A!un~E>+Q83VVwMEVO~Srd|*f*b=LX!JO8W4S^3I7C$f&c$j7!KMzIb{B zS&uUA)q)a}A_IylvFLCQoY9p9shr4N!liniLZ4tREJbp%s(=wK2zp}P7%uD3{|90G zzgyOs&m!xg%juKVoF?OOiJ`(jfB%DOcU#I1JfL3r^4UlQuS((I7O`f2Mb!vLRWpC3 zwyw)`FeXcooL6EUgvpFBe`r#;DkU)(PBHrbQ#~3UZCS(VBDGaq74UC6x&-jej_ewWaK`D_S(9{N9htUl#8jA(j z*xO9(qEWeLGUU(RSH1%y3QzNR5oRdn_Fr(jjtE8+VFHOGbu*dL98adn5lM{S;eHRo zT4)8z<}^(eu3&`j5h*=)Dq%4*ea+4ZRXN+C}mpES6I50+=IFgE_FN;k}Sl8R#$LwWPso+)%ZWOT%t2BTok zd_vTz{rfSNTWIwa(qK;8ZGF7CRVsaBw|x%R88p0gAlu${S-LZWLH3WF4Ez Date: Thu, 17 Sep 2026 04:46:18 +0200 Subject: [PATCH 006/124] fix: register direct Grok MCP endpoints in the Daimon-owned GROK_HOME config --- src/pi/cliMcpRegistration.test.ts | 11 +---- src/pi/cliMcpRegistration.ts | 17 +++---- src/pi/cliSession.test.ts | 6 +-- src/pi/cliSession.ts | 32 ++++++------- src/pi/cliSessionOutput.test.ts | 6 +-- src/pi/cliSessionProcess.test.ts | 23 +++++----- src/pi/cliSessionRemoval.test.ts | 74 ++++++++++++------------------- src/pi/grokHomeMcpRegistration.ts | 58 ++++++++++++++++++++++++ 8 files changed, 124 insertions(+), 103 deletions(-) create mode 100644 src/pi/grokHomeMcpRegistration.ts diff --git a/src/pi/cliMcpRegistration.test.ts b/src/pi/cliMcpRegistration.test.ts index f725b3d..efa1776 100644 --- a/src/pi/cliMcpRegistration.test.ts +++ b/src/pi/cliMcpRegistration.test.ts @@ -5,18 +5,9 @@ import { renderAgyArgs } from "./cliEngineSpawn.js"; import { DAIMON_MCP_SERVER_NAME, renderAgyMcpAddArgs, - renderAgyMcpRemoveArgs, - renderGrokMcpAddArgs, - renderGrokMcpRemoveArgs + renderAgyMcpRemoveArgs } from "./cliMcpRegistration.js"; -test("Grok's registration arguments are unchanged by the AGY generalization", () => { - assert.deepEqual(renderGrokMcpAddArgs([], "strict", "http://127.0.0.1:1/mcp"), - ["--sandbox", "strict", "mcp", "add", "--transport", "http", "--scope", "project", "daimon", "http://127.0.0.1:1/mcp"]); - assert.deepEqual(renderGrokMcpRemoveArgs([], "strict"), - ["--sandbox", "strict", "mcp", "remove", "--scope", "project", "daimon"]); -}); - test("AGY registers the per-wake endpoint as an http server, flags before the name", () => { const args = renderAgyMcpAddArgs([], "http://127.0.0.1:54321/mcp"); assert.deepEqual(args, ["mcp", "add", "--type", "http", DAIMON_MCP_SERVER_NAME, "http://127.0.0.1:54321/mcp"]); diff --git a/src/pi/cliMcpRegistration.ts b/src/pi/cliMcpRegistration.ts index 83d5993..0d5118f 100644 --- a/src/pi/cliMcpRegistration.ts +++ b/src/pi/cliMcpRegistration.ts @@ -2,16 +2,17 @@ import { spawn, type ChildProcess } from "node:child_process"; import { readChild } from "./cliChildOutput.js"; import { terminateChild, trackCliChild } from "./cliProcess.js"; -import { renderGrokSandboxArgs } from "./cliEngineSpawn.js"; /** * Per-wake MCP endpoint registration for the CLI engines that cannot take the * endpoint on their own command line. * * Codex takes `-c mcp_servers.daimon.url=` per invocation and needs - * nothing here. Grok and AGY are both config-file driven, so Daimon registers - * the ephemeral endpoint before the turn and removes it afterwards, through - * each CLI's own `mcp add`/`mcp remove` subcommands. + * nothing here. AGY is config-file driven, so Daimon registers the ephemeral + * endpoint before the turn and removes it afterwards through its own `mcp + * add`/`mcp remove` subcommands. Grok no longer uses this path: its project + * scope is ignored in untrusted workspaces on 1.0.34, so its endpoint is written + * into the agent's Daimon-owned GROK_HOME (`grokHomeMcpRegistration.ts`). * * The registration is deliberately performed by the engine CLI rather than by * writing its config file directly: the file format belongs to the engine, and @@ -90,15 +91,9 @@ export const registerCliMcpServer = async ( }; }; -/** The MCP server name both engines register Daimon's per-wake endpoint under. */ +/** The MCP server name every CLI engine registers Daimon's per-wake endpoint under. */ export const DAIMON_MCP_SERVER_NAME = "daimon" as const; -export const renderGrokMcpAddArgs = (commandArgs: readonly string[] | undefined, profile: string, endpoint: string): string[] => - [...renderGrokSandboxArgs(commandArgs, profile), "mcp", "add", "--transport", "http", "--scope", "project", DAIMON_MCP_SERVER_NAME, endpoint]; - -export const renderGrokMcpRemoveArgs = (commandArgs: readonly string[] | undefined, profile: string): string[] => - [...renderGrokSandboxArgs(commandArgs, profile), "mcp", "remove", "--scope", "project", DAIMON_MCP_SERVER_NAME]; - /** * `agy mcp add --type http `. * diff --git a/src/pi/cliSession.test.ts b/src/pi/cliSession.test.ts index 3e76d9d..4c9a489 100644 --- a/src/pi/cliSession.test.ts +++ b/src/pi/cliSession.test.ts @@ -471,16 +471,16 @@ test("disposing from a Server.prototype.listen interleaving never leaves an MCP } }); -test("disposing during Grok registration terminates setup before the engine starts", async (context) => { +test("disposing during AGY MCP registration terminates setup before the engine starts", async (context) => { if (!requirePosixProcessGroups(context)) return; - const root = await mkdtemp(path.join(os.tmpdir(), "daimon-cli-grok-cancel-")); + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-cli-agy-cancel-")); const ready = path.join(root, "add-ready"); const marker = path.join(root, "engine-started"); const grok = path.join(root, "grok.mjs"); await writeFile(grok, `import { writeFileSync } from "node:fs"; const args = process.argv.slice(2); if (args.includes("add")) { writeFileSync(${JSON.stringify(ready)}, "ready"); process.on("SIGTERM", () => undefined); setInterval(() => undefined, 1000); } else if (args.includes("remove")) process.exit(0); else writeFileSync(${JSON.stringify(marker)}, "started");`); try { const { session } = await createCliSessionFactory({ - command: process.execPath, commandArgs: [grok], engine: "grok", maxToolTurns: 1, timeoutMs: 10_000 + command: process.execPath, commandArgs: [grok], engine: "agy", maxToolTurns: 1, timeoutMs: 10_000 })({ cwd: root }); const pending = session.prompt("cancel"); void pending.catch(() => undefined); diff --git a/src/pi/cliSession.ts b/src/pi/cliSession.ts index ac5982a..07b964c 100644 --- a/src/pi/cliSession.ts +++ b/src/pi/cliSession.ts @@ -18,7 +18,6 @@ import type { TurnUsageOutcome } from "../runtime/turnUsageLedger.js"; import { readChild } from "./cliChildOutput.js"; import { cliChildEnvironment } from "./cliEnvironment.js"; import { - GROK_STRICT_SANDBOX_PROFILE, renderCodexArgs, spawnEngine } from "./cliEngineSpawn.js"; @@ -26,10 +25,9 @@ import { registerCliMcpServer, renderAgyMcpAddArgs, renderAgyMcpRemoveArgs, - renderGrokMcpAddArgs, - renderGrokMcpRemoveArgs, type CliMcpRegistration } from "./cliMcpRegistration.js"; +import { registerGrokHomeMcpServer } from "./grokHomeMcpRegistration.js"; import { decodeAgyHeadlessTurn, type AgyTurnUsage } from "./agyHeadlessResult.js"; import { type CodexTurnUsage } from "./codexHeadlessResult.js"; import { createCliTurnMeter, decodeCodexTurn, failedTurnOutcome, publishTurnRequests, publishTurnUsage } from "./cliTurnMetering.js"; @@ -321,35 +319,33 @@ class CliSession implements PiSessionLike { const controller=new AbortController();this.activeBrokerTurn=controller; try{output=await this.options.grokBrokerTurn(`${this.options.identityPrompt ?? ""}${text}`,mount.endpoint,controller.signal);}finally{if(this.activeBrokerTurn===controller)this.activeBrokerTurn=undefined;} } else { - if ((this.options.engine === "grok" || this.options.engine === "agy") && mount !== undefined) { + if (this.options.engine === "grok" && mount !== undefined) { + await this.options.verifyExecutable?.(); + registration = await registerGrokHomeMcpServer({ + engineHomePath: this.options.engineHomePath, + endpoint: mount.endpoint, + ...(this.options.verifyGrokSandbox !== undefined ? { verify: this.options.verifyGrokSandbox } : {}) + }); + this.mcpRegistration = registration; + } else if (this.options.engine === "agy" && mount !== undefined) { await this.options.verifyExecutable?.(); - const profile = this.options.grokSandboxProfile ?? GROK_STRICT_SANDBOX_PROFILE; - const grok = this.options.engine === "grok"; registration = await registerCliMcpServer({ - addArgs: grok - ? renderGrokMcpAddArgs(this.options.commandArgs, profile, mount.endpoint) - : renderAgyMcpAddArgs(this.options.commandArgs, mount.endpoint), - removeArgs: grok - ? renderGrokMcpRemoveArgs(this.options.commandArgs, profile) - : renderAgyMcpRemoveArgs(this.options.commandArgs), + addArgs: renderAgyMcpAddArgs(this.options.commandArgs, mount.endpoint), + removeArgs: renderAgyMcpRemoveArgs(this.options.commandArgs), command: this.options.command ?? this.options.engine, cwd: this.input.cwd, env: cliChildEnvironment([ ...(this.options.redactedEnvironmentNames ?? []), ...(this.input.daimonSecretEnvironmentNames ?? []) ], this.input.runtimeHomePath, { - ...(this.options.engine === "agy" && this.options.dbusSessionBusAddress !== undefined - ? { dbusSessionBusAddress: this.options.dbusSessionBusAddress } - : {}), + ...(this.options.dbusSessionBusAddress !== undefined ? { dbusSessionBusAddress: this.options.dbusSessionBusAddress } : {}), engine: this.options.engine, executablePath: this.options.command, engineHomePath: this.options.engineHomePath }), - ...(grok ? { failureClassifier: classifyGrokAuthenticationDiagnostic } : {}), onChild: (setupChild) => { this.setupChildren.add(setupChild); }, onChildSettled: (setupChild) => this.setupChildren.delete(setupChild), - secretValues, - ...(grok && this.options.verifyGrokSandbox !== undefined ? { verify: this.options.verifyGrokSandbox } : {}) + secretValues }); this.mcpRegistration = registration; } diff --git a/src/pi/cliSessionOutput.test.ts b/src/pi/cliSessionOutput.test.ts index 2dc92ac..3e6ec4a 100644 --- a/src/pi/cliSessionOutput.test.ts +++ b/src/pi/cliSessionOutput.test.ts @@ -98,7 +98,7 @@ test("Grok replies redact both the staged credential and a credential rotated du let reads = 0; try { const { session } = await createCliSessionFactory({ - command: process.execPath, commandArgs: [engine], engine: "grok", + command: process.execPath, commandArgs: [engine], engine: "grok", engineHomePath: path.join(root, ".grok"), credentialSecretValues: async () => ++reads === 1 ? [oldSecret] : [rotatedSecret] })({ cwd: root }); let emitted = ""; @@ -123,7 +123,7 @@ test("Grok auth rejection is typed and never retains raw credential diagnostics" await writeFile(engine, `const a=process.argv.slice(2);if(a.includes("mcp"))process.stdout.write("ok");else{process.stderr.write("Authentication rejected by server ${secret}");process.exitCode=7;}`); try { const { session } = await createCliSessionFactory({ - command: process.execPath, commandArgs: [engine], engine: "grok", + command: process.execPath, commandArgs: [engine], engine: "grok", engineHomePath: path.join(root, ".grok"), credentialSecretValues: async () => [secret] })({ cwd: root }); await assert.rejects(session.prompt("work"), (error: unknown) => { @@ -146,7 +146,7 @@ test("Grok auth rejection is classified before a long secret and verbose tail ar ].join("\n")); try { const { session } = await createCliSessionFactory({ - command: process.execPath, commandArgs: [engine], engine: "grok", + command: process.execPath, commandArgs: [engine], engine: "grok", engineHomePath: path.join(root, ".grok"), credentialSecretValues: async () => [secret] })({ cwd: root }); await assert.rejects(session.prompt("work"), (error: unknown) => { diff --git a/src/pi/cliSessionProcess.test.ts b/src/pi/cliSessionProcess.test.ts index 937b347..7b150c9 100644 --- a/src/pi/cliSessionProcess.test.ts +++ b/src/pi/cliSessionProcess.test.ts @@ -6,10 +6,9 @@ import test from "node:test"; import { createCliSessionFactory, readChild, spawnEngine, terminateChild } from "./cliSession.js"; -const grokStream = (text: string): string => [ - { type: "assistant", parent_tool_use_id: null, session_id: "fake", message: { role: "assistant", stop_reason: "end_turn", content: [{ type: "text", text }] } }, - { type: "result", subtype: "success", is_error: false, result: text, stop_reason: "end_turn", session_id: "fake" } -].map((event) => JSON.stringify(event)).join("\n"); +// Only AGY still registers its per-wake MCP endpoint through setup/removal +// children; Grok writes it into its own GROK_HOME (`grokHomeMcpRegistration.ts`). +const agyStream = (text: string): string => JSON.stringify({ event: "result", result: { conversation_id: "fake", status: "SUCCESS", response: text, num_turns: 1, usage: { input_tokens: 1, output_tokens: 1, thinking_tokens: 0, cache_read_tokens: 0, total_tokens: 2 } } }); test("terminates a process group after its leader has exited", async (context) => { if (!requirePosixProcessGroups(context)) return; @@ -29,9 +28,9 @@ test("terminates a process group after its leader has exited", async (context) = } }); -test("Grok setup reaps a stubborn descendant after its successful leader exits", async (context) => { +test("AGY MCP setup reaps a stubborn descendant after its successful leader exits", async (context) => { if (!requirePosixProcessGroups(context)) return; - const root = await mkdtemp(path.join(os.tmpdir(), "daimon-grok-setup-group-")); + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-agy-setup-group-")); const descendant = path.join(root, "setup-descendant-pid"); const grok = path.join(root, "grok.mjs"); await writeFile(grok, `import { spawn } from "node:child_process"; import { writeFileSync } from "node:fs"; @@ -41,10 +40,10 @@ test("Grok setup reaps a stubborn descendant after its successful leader exits", child.stdout.once("data", () => { writeFileSync(${JSON.stringify(descendant)}, String(child.pid)); process.exit(0); }); } if (args.includes("remove")) process.exit(0); - process.stdout.write(${JSON.stringify(grokStream("engine complete"))});`); + process.stdout.write(${JSON.stringify(agyStream("engine complete"))});`); try { const { session } = await createCliSessionFactory({ - command: process.execPath, commandArgs: [grok], engine: "grok", maxToolTurns: 1, timeoutMs: 10_000 + command: process.execPath, commandArgs: [grok], engine: "agy", maxToolTurns: 1, timeoutMs: 10_000 })({ cwd: root }); await session.prompt("research"); const pid = Number(await readFile(descendant, "utf8")); @@ -56,9 +55,9 @@ test("Grok setup reaps a stubborn descendant after its successful leader exits", } }); -test("Grok removal reaps a stubborn descendant after its successful leader exits", async (context) => { +test("AGY MCP removal reaps a stubborn descendant after its successful leader exits", async (context) => { if (!requirePosixProcessGroups(context)) return; - const root = await mkdtemp(path.join(os.tmpdir(), "daimon-grok-remove-group-")); + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-agy-remove-group-")); const descendant = path.join(root, "remove-descendant-pid"); const grok = path.join(root, "grok.mjs"); await writeFile(grok, `import { spawn } from "node:child_process"; import { writeFileSync } from "node:fs"; @@ -66,10 +65,10 @@ test("Grok removal reaps a stubborn descendant after its successful leader exits if (args.includes("remove")) { const child = spawn(process.execPath, ["-e", "process.on('SIGTERM', () => undefined); process.stdout.write('ready'); setInterval(() => undefined, 1000)"], { stdio: ["ignore", "pipe", "ignore"] }); child.stdout.once("data", () => { writeFileSync(${JSON.stringify(descendant)}, String(child.pid)); process.exit(0); }); - } else if (args.includes("add")) process.exit(0); else process.stdout.write(${JSON.stringify(grokStream("engine complete"))});`); + } else if (args.includes("add")) process.exit(0); else process.stdout.write(${JSON.stringify(agyStream("engine complete"))});`); try { const { session } = await createCliSessionFactory({ - command: process.execPath, commandArgs: [grok], engine: "grok", maxToolTurns: 1, timeoutMs: 10_000 + command: process.execPath, commandArgs: [grok], engine: "agy", maxToolTurns: 1, timeoutMs: 10_000 })({ cwd: root }); await session.prompt("research"); const pid = Number(await readFile(descendant, "utf8")); diff --git a/src/pi/cliSessionRemoval.test.ts b/src/pi/cliSessionRemoval.test.ts index aa880ac..f8320f1 100644 --- a/src/pi/cliSessionRemoval.test.ts +++ b/src/pi/cliSessionRemoval.test.ts @@ -1,23 +1,21 @@ import assert from "node:assert/strict"; -import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import test from "node:test"; import { createCliSessionFactory } from "./cliSession.js"; -import { GrokSubscriptionAuthenticationRejectedError } from "../runtime/grokAuthenticationError.js"; -const grokStream = (text: string): string => [ - { type: "assistant", parent_tool_use_id: null, session_id: "fake", message: { role: "assistant", stop_reason: "end_turn", content: [{ type: "text", text }] } }, - { type: "result", subtype: "success", is_error: false, result: text, stop_reason: "end_turn", session_id: "fake" } -].map((event) => JSON.stringify(event)).join("\n"); +// Grok has no removal child any more (its endpoint lives in GROK_HOME config), +// so removal-failure semantics are exercised through AGY's `mcp remove`. +const agyStream = (text: string): string => JSON.stringify({ event: "result", result: { conversation_id: "fake", status: "SUCCESS", response: text, num_turns: 1, usage: { input_tokens: 1, output_tokens: 1, thinking_tokens: 0, cache_read_tokens: 0, total_tokens: 2 } } }); -test("Grok removal failure rejects without emitting a successful turn", async () => { - const root = await mkdtemp(path.join(os.tmpdir(), "daimon-grok-remove-failure-")); +test("MCP removal failure rejects without emitting a successful turn", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-agy-remove-failure-")); const grok = path.join(root, "grok.mjs"); - await writeFile(grok, `const args = process.argv.slice(2); if (args.includes("remove")) process.exit(23); else if (args.includes("add")) process.exit(0); else process.stdout.write(${JSON.stringify(grokStream("engine complete"))});`); + await writeFile(grok, `const args = process.argv.slice(2); if (args.includes("remove")) process.exit(23); else if (args.includes("add")) process.exit(0); else process.stdout.write(${JSON.stringify(agyStream("engine complete"))});`); try { - const { session } = await createCliSessionFactory({ command: process.execPath, commandArgs: [grok], engine: "grok", maxToolTurns: 1, timeoutMs: 10_000 })({ cwd: root }); + const { session } = await createCliSessionFactory({ command: process.execPath, commandArgs: [grok], engine: "agy", maxToolTurns: 1, timeoutMs: 10_000 })({ cwd: root }); let turns = 0; session.subscribe((event) => { if (event.type === "turn_end") turns += 1; }); await assert.rejects(session.prompt("research"), /CLI engine exited 23/); @@ -29,43 +27,27 @@ test("Grok removal failure rejects without emitting a successful turn", async () } }); -test("Grok authentication rejection keeps precedence over bounded removal failure", async () => { - const root = await mkdtemp(path.join(os.tmpdir(), "daimon-grok-auth-remove-failure-")); +test("Grok direct sessions register the per-wake MCP endpoint in GROK_HOME, never through a project-scoped child", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-grok-home-mcp-")); const grok = path.join(root, "grok.mjs"); - const authSecret = "auth-rejection-secret-canary"; - const cleanupSecret = "cleanup-secret-canary"; - await writeFile(grok, `const args=process.argv.slice(2);if(args.includes("remove")){process.stderr.write("cleanup ${cleanupSecret}");process.exit(23)}else if(args.includes("add"))process.exit(0);else{process.stderr.write("RefreshTokenRejected ${authSecret}");process.exit(7)}`); + const engineHomePath = path.join(root, "home", ".grok"); + const seen = path.join(root, "seen"); + await writeFile(grok, `import { appendFileSync, readFileSync } from "node:fs"; const args = process.argv.slice(2); appendFileSync(${JSON.stringify(seen)}, args.includes("mcp") ? "MCP-CHILD\\n" : readFileSync(${JSON.stringify(path.join(engineHomePath, "config.toml"))}, "utf8")); process.stdout.write(${JSON.stringify([{ type: "assistant", parent_tool_use_id: null, session_id: "fake", message: { role: "assistant", stop_reason: "end_turn", content: [{ type: "text", text: "done" }] } }, { type: "result", subtype: "success", is_error: false, result: "done", stop_reason: "end_turn", session_id: "fake" }].map((event) => JSON.stringify(event)).join("\n"))});`); try { - const { session } = await createCliSessionFactory({ - command: process.execPath, commandArgs: [grok], engine: "grok", maxToolTurns: 1, - timeoutMs: 10_000, credentialSecretValues: async () => [authSecret, cleanupSecret] - })({ cwd: root }); - await assert.rejects(session.prompt("research"), (error: unknown) => { - assert.ok(error instanceof GrokSubscriptionAuthenticationRejectedError); - assert.doesNotMatch(error.message, /canary|cleanup|RefreshTokenRejected/u); - assert.ok(error.cause instanceof Error); - assert.match(error.cause.message, /CLI engine exited 23/u); - assert.doesNotMatch(error.cause.message, /canary|cleanup-secret/u); - assert.ok(Buffer.byteLength(error.cause.message) < 1_024); - return true; - }); - } finally { await rm(root, { recursive: true, force: true }); } -}); + const { session } = await createCliSessionFactory({ command: process.execPath, commandArgs: [grok], engine: "grok", maxToolTurns: 1, timeoutMs: 10_000, engineHomePath })({ cwd: root }); + await session.prompt("research"); + const during = await readFile(seen, "utf8"); + assert.doesNotMatch(during, /MCP-CHILD/u); + assert.match(during, /\[mcp_servers\.daimon\]\nurl = "http:\/\/127\.0\.0\.1:\d+\/mcp"/u); + assert.match(during, /\[skills\]\ndisabled = \[/u); + const after = await readFile(path.join(engineHomePath, "config.toml"), "utf8"); + assert.doesNotMatch(after, /mcp_servers/u); + await session.disposeAsync?.(); -test("Grok removal auth rejection is typed before verbose diagnostics are truncated", async () => { - const root = await mkdtemp(path.join(os.tmpdir(), "daimon-grok-remove-auth-rejection-")); - const grok = path.join(root, "grok.mjs"); - const secret = `refresh-start-${"r".repeat(1970)}-refresh-end`; - await writeFile(grok, `const a=process.argv.slice(2);if(a.includes("remove")){process.stderr.write("RefreshTokenRejected "+${JSON.stringify(secret)}+" "+"tail".repeat(250));process.exit(23)}else if(a.includes("add"))process.exit(0);else process.stdout.write(${JSON.stringify(grokStream("complete"))});`); - try { - const { session } = await createCliSessionFactory({ - command: process.execPath, commandArgs: [grok], engine: "grok", - credentialSecretValues: async () => [secret] - })({ cwd: root }); - await assert.rejects(session.prompt("research"), (error: unknown) => { - assert.ok(error instanceof GrokSubscriptionAuthenticationRejectedError); - assert.doesNotMatch(error.message, /RefreshToken|refresh-start|refresh-end|r{32}|tail/u); - return true; - }); - } finally { await rm(root, { recursive: true, force: true }); } + const unowned = await createCliSessionFactory({ command: process.execPath, commandArgs: [grok], engine: "grok", maxToolTurns: 1, timeoutMs: 10_000 })({ cwd: root }); + await assert.rejects(unowned.session.prompt("research"), /Daimon-owned GROK_HOME/u); + await unowned.session.disposeAsync?.(); + } finally { + await rm(root, { recursive: true, force: true }); + } }); diff --git a/src/pi/grokHomeMcpRegistration.ts b/src/pi/grokHomeMcpRegistration.ts new file mode 100644 index 0000000..2c1fcef --- /dev/null +++ b/src/pi/grokHomeMcpRegistration.ts @@ -0,0 +1,58 @@ +import { randomUUID } from "node:crypto"; +import { constants } from "node:fs"; +import { lstat, mkdir, open, rename, unlink } from "node:fs/promises"; +import path from "node:path"; + +import { renderGrokLeanBaseConfig } from "../runtime/grokBrokerWorkerConfig.js"; +import type { CliMcpRegistration } from "./cliMcpRegistration.js"; + +/** + * Per-wake MCP registration for the direct (non-broker) Grok CLI path. + * + * `grok mcp add --scope project` writes `/.grok/config.toml`, which Grok + * 1.0.34 skips entirely in an untrusted workspace — and Daimon keeps every + * workspace untrusted so cwd `AGENTS.md` and project skills never load. The + * endpoint therefore goes into the agent's own Daimon-owned `GROK_HOME` + * config, rendered from the same lean base the broker worker uses + * (`renderGrokLeanBaseConfig`), and is removed again after the turn by + * rewriting the base alone. + * + * There is no fallback to the operator's `~/.grok`: without an explicit + * `engineHomePath` the session refuses, rather than editing a human's config. + */ +export async function registerGrokHomeMcpServer(input: Readonly<{ engineHomePath: string | undefined; endpoint: string; verify?: () => Promise }>): Promise { + const home = input.engineHomePath; + if (home === undefined || !path.isAbsolute(home)) { + throw new Error("Grok direct CLI sessions require a Daimon-owned GROK_HOME (engineHomePath): Grok 1.0.34 ignores project-scoped MCP servers in untrusted workspaces"); + } + if (!/^http:\/\/127\.0\.0\.1:\d{1,5}\/[A-Za-z0-9/_-]*$/u.test(input.endpoint)) throw new Error("Grok MCP endpoint must be a loopback http URL"); + await input.verify?.(); + await writeConfig(home, `${renderGrokLeanBaseConfig()}[mcp_servers.daimon]\nurl = ${JSON.stringify(input.endpoint)}\n`); + let closePromise: Promise | undefined; + return { + close: (): Promise => closePromise ??= (async () => { + await input.verify?.(); + await writeConfig(home, renderGrokLeanBaseConfig()); + })() + }; +} + +async function writeConfig(home: string, text: string): Promise { + await mkdir(home, { recursive: true, mode: 0o700 }); + const directory = await lstat(home); + if (!directory.isDirectory() || directory.isSymbolicLink()) throw new Error("Grok engine home is not a private directory"); + const target = path.join(home, "config.toml"); + const temporary = path.join(home, `.config.toml.${randomUUID()}.tmp`); + let handle: Awaited> | undefined; + try { + handle = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, 0o600); + await handle.writeFile(text); + await handle.sync(); + await handle.close(); + handle = undefined; + await rename(temporary, target); + } finally { + await handle?.close().catch(() => undefined); + await unlink(temporary).catch(() => undefined); + } +} From 7529ef95f671fedb678aa4749078654c7eab5ba6 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 04:47:25 +0200 Subject: [PATCH 007/124] feat: compile the lean Grok worker argv and system prompt into the native launcher --- src/runtime/native/engineBrokerLauncher.h | 6 ++ .../native/engineBrokerLauncherCore.inc | 8 +++ ...ngineBrokerLauncherIntegrationLauncher.inc | 13 +++++ src/runtime/native/launcherArgv.test.ts | 55 +++++++++++++++++++ 4 files changed, 82 insertions(+) create mode 100644 src/runtime/native/launcherArgv.test.ts diff --git a/src/runtime/native/engineBrokerLauncher.h b/src/runtime/native/engineBrokerLauncher.h index 76e7ce9..d3a9512 100644 --- a/src/runtime/native/engineBrokerLauncher.h +++ b/src/runtime/native/engineBrokerLauncher.h @@ -21,6 +21,12 @@ #ifndef DBL_EXECUTABLE #define DBL_EXECUTABLE "/usr/local/bin/grok" #endif +/* Lean Grok worker contract; mirrored byte-for-byte by src/contracts/grokWorkerContract.ts + and checked by launcherArgv.test.ts. */ +#define DBL_GROK_SYSTEM_PROMPT \ + "You are a headless Daimon agent; no human is present. Your identity, instructions and wake event are in the user prompt. Daimon tools are MCP tools on server daimon: call a known one directly with use_tool (tool_name daimon__moltnet_read, daimon__moltnet_send, daimon__memory_search, daimon__memory_register, or another daimon__ name you were given); use search_tool only for a name you do not know. If a tool result says output was saved to a file, read that path with read_file. If a tool fails, do not retry it in a loop: stop and report the failure. Your final answer is a private note to the runtime: one line, or empty." +#define DBL_GROK_TOOLS "run_terminal_cmd,read_file,grep,list_dir,search_tool,use_tool" +#define DBL_GROK_MAX_TURNS "48" struct dbl_request { uint32_t version, slot; diff --git a/src/runtime/native/engineBrokerLauncherCore.inc b/src/runtime/native/engineBrokerLauncherCore.inc index 8329892..1bb8f45 100644 --- a/src/runtime/native/engineBrokerLauncherCore.inc +++ b/src/runtime/native/engineBrokerLauncherCore.inc @@ -321,6 +321,14 @@ static pid_t launch(const struct dbl_registration *r, int executable, "/proc/self/fd/3", "--no-memory", "--disable-web-search", + "--no-plan", + "--verbatim", + "--system-prompt-override", + DBL_GROK_SYSTEM_PROMPT, + "--tools", + DBL_GROK_TOOLS, + "--max-turns", + DBL_GROK_MAX_TURNS, "--cwd", (char *)r->workspace, "--output-format", diff --git a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc index 8458226..adab67f 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc @@ -112,6 +112,19 @@ static void org_cases(void) { strstr(out, "uid=2200") && strstr(out, "--always-approve") && !strstr(out, "EVIL"), "fixed worker boundary"); + check(strstr(out, "=--verbatim\n") && strstr(out, "=--no-plan\n") && + strstr(out, "=--no-subagents\n") && strstr(out, "=--no-memory\n") && + strstr(out, "=--disable-web-search\n") && + strstr(out, "=--tools\n") && strstr(out, "=" DBL_GROK_TOOLS "\n") && + strstr(out, "=--max-turns\n") && + strstr(out, "=" DBL_GROK_MAX_TURNS "\n") && + strstr(out, "=--system-prompt-override\n") && + strstr(out, "=" DBL_GROK_SYSTEM_PROMPT "\n") && + strstr(out, "=--prompt-file\n") && + strstr(out, "=/proc/self/fd/3\n") && + !strstr(out, "--reasoning-effort") && + strstr(out, " argc=23 "), + "lean worker argv"); free(out); close(s); q = request(); diff --git a/src/runtime/native/launcherArgv.test.ts b/src/runtime/native/launcherArgv.test.ts new file mode 100644 index 0000000..3712891 --- /dev/null +++ b/src/runtime/native/launcherArgv.test.ts @@ -0,0 +1,55 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import test from "node:test"; + +import { GROK_ENGINE_BROKER } from "../../contracts/runtimeContractManifest.js"; +import { DAIMON_GROK_SYSTEM_PROMPT, GROK_WORKER_MAX_TURNS, GROK_WORKER_TOOL_IDS } from "../../contracts/grokWorkerContract.js"; +import { renderGrokBrokerWorkerArgs } from "../grokBrokerWorkerConfig.js"; + +const read = (name: string): string => readFileSync(new URL(`./${name}`, import.meta.url), "utf8"); +const unquote = (literal: string): string => { + assert.match(literal, /^"[^"\\]*"$/u, "launcher literals must not need escaping"); + return literal.slice(1, -1); +}; + +/** `#define NAME "value"` (optionally continued onto the next line) from the launcher header. */ +const defines = (): ReadonlyMap => { + const header = read("engineBrokerLauncher.h").replace(/\\\n\s*/gu, ""); + return new Map([...header.matchAll(/^#define (DBL_GROK_[A-Z_]+)\s+("[^"\n]*")$/gmu)].map((match) => [match[1]!, unquote(match[2]!)])); +}; + +/** The compiled worker argv, token by token, exactly as `launch()` passes it to `execveat`. */ +const compiledArgv = (): readonly string[] => { + const source = read("engineBrokerLauncherCore.inc"); + const block = source.match(/char \*const argv\[\] = \{([\s\S]*?)NULL\};/u); + assert.ok(block, "launcher argv array not found"); + const values = defines(); + return block[1]!.split(",").map((token) => token.trim()).filter(Boolean).map((token) => { + if (token.startsWith('"')) return unquote(token); + if (token === "(char *)r->workspace") return "/registered/workspace"; + const value = values.get(token); + assert.ok(value !== undefined, `unexpected launcher argv token ${token}`); + return value; + }); +}; + +test("the native launcher compiles exactly the lean Grok worker argv Daimon renders", () => { + const argv = compiledArgv(); + assert.equal(argv[0], "grok"); + assert.deepEqual(argv.slice(1), renderGrokBrokerWorkerArgs("/proc/self/fd/3", "/registered/workspace")); + for (const flag of ["--verbatim", "--no-plan", "--no-subagents", "--no-memory", "--disable-web-search", "--always-approve"]) assert.ok(argv.includes(flag), flag); + assert.equal(argv[argv.indexOf("--tools") + 1], GROK_WORKER_TOOL_IDS.join(",")); + assert.equal(argv[argv.indexOf("--max-turns") + 1], String(GROK_WORKER_MAX_TURNS)); + assert.equal(argv.includes("--reasoning-effort"), false, "effort is declared per deployment in config.toml, never compiled"); + assert.equal(argv.includes("--disallowed-tools"), false, "--disallowed-tools is ignored under --tools"); +}); + +test("the compiled system prompt is byte-identical to the contract prompt pinned in the manifest", () => { + const compiled = defines().get("DBL_GROK_SYSTEM_PROMPT"); + assert.equal(compiled, DAIMON_GROK_SYSTEM_PROMPT); + assert.equal(createHash("sha256").update(DAIMON_GROK_SYSTEM_PROMPT).digest("hex"), GROK_ENGINE_BROKER.worker.systemPromptSha256); + assert.match(DAIMON_GROK_SYSTEM_PROMPT, /^[\x20-\x7e]+$/u); + assert.ok(DAIMON_GROK_SYSTEM_PROMPT.length >= 320 && DAIMON_GROK_SYSTEM_PROMPT.length <= 700, "roughly 80-150 tokens"); + for (const tool of ["daimon__moltnet_read", "daimon__moltnet_send", "use_tool", "search_tool", "read_file"]) assert.ok(DAIMON_GROK_SYSTEM_PROMPT.includes(tool), tool); +}); From 4bbf22462a36ba3d30220479fc9c9c917bec457d Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 04:48:00 +0200 Subject: [PATCH 008/124] build: rebuild native engine broker artifacts for the lean Grok worker argv --- src/contracts/runtimeContractManifest.ts | 6 ++--- .../artifacts/daimon-engine-broker-arm64 | Bin 67456 -> 67456 bytes ...daimon-engine-broker-arm64.provenance.json | 2 +- .../native/artifacts/daimon-engine-broker-x64 | Bin 55392 -> 55392 bytes .../daimon-engine-broker-x64.provenance.json | 2 +- src/runtime/native/artifactsManifest.test.ts | 24 ++++++++++++++++++ 6 files changed, 29 insertions(+), 5 deletions(-) create mode 100644 src/runtime/native/artifactsManifest.test.ts diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 561e781..4dbf679 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -69,9 +69,9 @@ export const GROK_ENGINE_BROKER = { }, bounds: { promptBytes: 65_536, capabilityBytes: 4_096, capabilityBundleBytes: 8_196, outputBytes: 65_536 }, artifacts: { - sourceSha256: "bdcab1e12dcc531ed8e56f890263ca23a9ee7bac468191dd598e143df4ff8c58", - x64Sha256: "e3fe2738fc8a979861085b4003bf2d5d7c284874897cb6ec2e2e2383211768bd", - arm64Sha256: "ad44e02c38e6a3207ac4a3d5fd98b6d2e55341ce42dfd2f07204bbe54a7a653d" + sourceSha256: "7830ff308420cf1b84aba118903c1e8ef40f8c2a320136977a1e7f9e3292a734", + x64Sha256: "b3da1618ca218ff44ce178385f94146736439acccaedaf047d41709ab0aa5d43", + arm64Sha256: "b4f41d429db5f8fefaf880f9ff9d64e76d0687829e6002fe7c7052cc7a41af6e" } } as const; export const AGY_SUBSCRIPTION_REALM = { diff --git a/src/runtime/native/artifacts/daimon-engine-broker-arm64 b/src/runtime/native/artifacts/daimon-engine-broker-arm64 index f028a3ca832c3adb81418ca17d8b062aed9f056f..0fd341da3ef4f6496e05941770aefbb39a3ad5b7 100755 GIT binary patch delta 4173 zcmZWt4Nz3q6~6bq6;NPf5Cs9*x1b0cK&_&pD9ewciD?*QS`*X223FayKd`%)n40YK zC$af4@_I3ZBr1-vW~sKBLZh2z6rxSjX==#kSDR!O@Fz8?)exhr=<9bMpw9Ho+&%Yv z=iGbG`ObOwq4IW}yj|DaJFa;h(VpgY%Bdr9X4kNGTXuHPx02{owmRi}u9UBY#0ed< zRJD~vkU2&|-r~BkI$y~KRt-$kfg9WNJ3Uj9sPJBX=l0YjQFwF#`@L}u7ZcJR6AmBi z^raao{gUlTzbbSWu-B%xQX`u^t%v$MFHdWXoSd%kW3GzC)W6jn=o8~RBtn`=dph&w zc<9v5!JJ2_P+ivf%Ne_MW2Y(z`ywIJI@Q^3QK&0po_T44y^OWaTQISR9sp)S~LL8zC`DNzsuR`_hQg zBW9$kWh88nn+r}4Qe|MXpqx#hfwLu1%F#{F79eX=njlwv(>-x$sD#|Ov6($N-$zr~ zwfQDrbn52dR3ZlrI`V)h%E2RWdS;d!JYx{UXG$dHYRx?_2$Eb8-kYvmJC>*%9g6k$ ziHRM2Q-QqczLG44Ri7@b7CyV^+UA(>>8&y0@Cz~Ff$cG2mBoa&UQt^OGb1i-6i8)X z`hx*7My@C%NH*9c6R@Q1fLwu>psb}k!%Sbyn)`}ddYZY zV1H>~pq>N-f#?&{xPnV;PJ&jL!CD^i+I6j0jkp3?kLL8v5A`L(PRq&D{!Y>vu~ zBRAB%-JC;kNu}buwSJc}lq&G@GuW^2CcduDUN(`XHs7L)lNmYSi zWvsUAB!RSa6Ova;Nc?cSC3j2%Mo?n`l^n(rOuxZy{UYfw5OV25@I|A+hd5U0Lu5n1 zL$gWP!#*yUFmWkAcPvUFCZKWU3NRGxW8s1+;>4jmHhEEU*3F;zUX#(U4X zfY86dKBL%9j^UIbuu!Wx(2LX3$0$JG*Ups}NBNq#4d;xeyadsWu9rvFcfrR<*V5A4L4Ycc610 z9qgjEwOnC@B*^_7Z01Cg*N>N}PZUfcj8lAl0x+#sf?flA01|*@jG7Z7KI1MES*m%G z;{>MN@n{#9Ilu$w(S|1AGSDs`x#CaItDz6#73Zh&8}uMw6UTifn2Lb;p_`$rlSpGb z;jNrt+5>zq@OI!oA#0u;V|&bnzUe>*@oJ7Er1##Q3}q~2R$qp4etM!`!;E?S)6*M` zLE5vFdgC1ByL$}&^Z0l~0gJ?lGEP^{RZmwUFdu{Qe)n``AIVg7m{W``a;9Re&cX{d z#lNyFLJ7?_habWvBNF24)x)92oT;4X$yCq`y|Ea@QkHViXi{P@=K;M)=O|g0Jmp)% zWdE&0H|8nl+Vk+0O7h=2IFzUCGtX59CM5g$;ajTbDi@+t`~&!gZ=5Cv|1&$)&(A3C zo~vAoreW1kO2f;I#9!7%{bv$r_;fN2|M9ZgI=X;&qP8G3SMdX<;-=t$_If7{|5@K%elwd@oLSBK`c`@$YV~aRDo5kVRbz}E2wz70_-0@*p)Q!L& z9AkS+x6uu3){?5bYk%bL%_RQXb)?}v2(dj&7SfN|*Gmd%5u36!nZCpFm)=JiduFMD z?(W>TG@agEj76gh`fx#2be=3Lqn+mEIs^5wC3W}E3!RO1ZPe#<Sa&p#L z-H%$lHpi&$S?}>$9XSndm!rX()+{kJXJ$cp$l>$9P>XJJukC`%B8DVGXCl6Ol@_S}Y5s8jIbI z80(#`$DNYPX_abiZflL#zFzV@Zu8dhovZj-DU<(JIV}#jXltvg94@=pY4ujQkzn@! zp-AXX)ar1#*H?M07I#hE9nQnB+q%{UGTqsd%Z)T$$jR-ywWFk+x*mi+Zbdq4ZAd?B zArB4N@co7(bU6|A8ot5p1+^~l?A6lCm6or8NlONFH`%?C$FkldxthEUOB4^hAm#H!2&ikN;p=v#%8zPx#Jp(&F(>7!{@oJUiW&*=0*Lmm)+%R zSRi@4t_H1yZfk?f?bYhWS2ek@kyiILHm3!}aC(rYWb>hTsHtt8#cSnBSb1Q)*zP8D zhr_x+E0^8oM3_k0irTNw9qGQD9If62u2rDb!8fj|anxqtiamQR!pOFR^(s_6`wkwn zM@pV(*Sg+ec_PQ#?@qq*M3U7Rs9I?Y`~=KT)R|NhYLnNq^c>nTtkpE zA;%w9)l$e6kgFiuAvZvFLpDS9LcYoE0aXn_h9Gsg+8W+b)ojS`A!{M!zo=>pA(?H zm9PrCNIP}A#HpB01(97xSeh$iQq)n<#?5$2F{)u&b(B5gS}JsPv)@5%>0uXKkNB2> z*LZwwNwNN7W9iOa`dvHq#jUUCUuo4Bv)A=4ujxy6ysCe>P5;!mFTVWj^G`ng`Wrnd zieB>S4t?9pde^v9CyS$ndtYbz*IM-UQzwhQ`d1NfSc^1M5ujlmc6gNl0+)s{Q3a^Ug2H9ul;?h4aay|*dm?i*nG z5Ewq(+LMXjW?;vA*-LKI(q+Nh?%d9+@+r*aKcHVYFj5u1)%U{pIx1Zet^GQhJMQ-X2DB!P Up>xMAYZzX(tl`i@V`zr(KW^+-m;e9( delta 3347 zcmYjU3vg7`89wLU%{y6>u-U{s*t>aW!GIG&h;GPk-dd~7l9^I;BoL=kLmNXPV-?*S zrP3CVTu*d~@(3y;yCM^3eRiut~0T=b`t(}8xF&FOKIt>tmU`Q>(y6p*sdw+JEV)r(Xi~Rgyed8$yQl9Sqm$MSO>bBWH#< zPmMRS%VkPRk~XoHx+gG4P)_ws-Y>Z)bs1b_n&#rid|c>CDn1!pEWa6O=f3@*%Par)W~kEvN&^z8jxi* zVA7HMi1FFLDJ1>*b~SK`OQB2EiZ<1(?2;7K5qd3KyD^-qy+0G<9+A^LVo-@3-TKqC z6yh6nA-?RniW|-Gq49^~L!p-V&{%7Hh_}awcK*a&rsa{>IwjILGPce~B2-5uA&*eu zx84&MsR!C3g!iQ3r1HNLSH;u}Mi zoH}yL=o@RxGrIn1rKHP|+4V0grM+9Tx<0l(srFbGcYR{ruD)y)=lGQRij_3wYEP^C zt)s6+l{-9e z^gwo75sDb(waj^Eit`+Jg_3?-eWZp13s9))Q*bIF{lTd9<))LQ&5t^VT58&f6=1V*S5IzlFSIf{S`E+~15P-CvCpLB)e|h9?_UR%iK&5sT!+<;(Cm z(UHvZ(nF%4C@fW~lEUIw^0W;oR0-lg2}MuyGEt;2;rHrE=?H41phhOt$gGDLh8uh~ zNLC>vmJrFJCXiAkW==T@WHCJ)AD)j>((Nat^dVv*hB;Ygc=QtKCeVU}6QAQ!Eq`Ti zTT{xSwGk|GMkbzM2?;%bJR8(tuaCuGnI1`}2I|FvVn`{j9OSJ!P{qI~o3&=iSu-nH zRz*e;|B<^!v2>v+5nT>VsnO_BEosnP}QZ?`cCccn_r(+9U zSlft6C1NYzl*SyD`E~CiG=lCU_mPm;^PrLRk2KGf{4Y1~Iv63dVM!COEqBJbmmuE{ z4(ktK{ec1aHX}P#X>?wJ*_1*`M|mx-vV`|nvV1Ml4d8I~7PujB3b+Ogufwo%ctv5+ zI0#Dih3Jz^N`DzMBge19)kG|q1lhFOd}1eJ`6LO?hir7d(MdeG z$3>@GMA9OQ9h?V#6MSyOSrEJh(7|EwuY>o4{}FW%`|V~&Y?aPJuoJk=gpf0&DPN;N zvyzWHk>pN*9gh|)v4!3m%-43)0xb?cZr_;}dx3U$XMq+2I~uKMut1A87iv+kBhmh2 zE7U$x3$+N?IwyK1hjTS9>&~kH`O-ruVY$Oj{O*(IX*8daTG`43yF2YJ~D4_g#GHgZHIkm-aZU_Kbx$o zr?af0x|JSgZ&g>)&)E0X1#~A%so9-;WvD__?OA#5j&*)F#Jn}DldsOvGPI&#jLz#r zOs?HSyO~zIA!++{A$Fsn@>|Knz7 zVjt&^<9L<#IM<0|Kf!q>6~IE7I7uIjH6RQhArbiAY0i%U&-{aPyd&iL$N0sG zD)xQB`3%r@p7X0%a2HSqjRt{fK-&e*w*YH^DsT<38%Vz4+%MuYoSz3a0j~-kUm{_5 zKbpS!n2+-n!na2)P6RAPn*z67aQFmgLo=W@;tvJ!DQJyo))Va2N3)B!AcVz*KLdX5 zMtdBb=t}JR!~@bnDyDcH?a>o#TU#+x+M=cY6U@? zN5Huc*y{ENo%>MNe%yIdrSEI2Iq;JH;EVbySC@X@UVRno)^{^~^@}g)U()nX#eRAI z^DoYx`}ajXvB%ZEyhq>jyuLN|!dF#s(p~#^>0jBc-*VwA`!`?P!6Rx9hwPx{e8iDe z4OZmDNq-AlETf}HI`$c>>v%%y^01+fZI*@r=l6(p;|tdbr@G@8+#2u(aQ*<>^q9l) z$=RDF7U|m&ItdQ%R`@H#&n|HNXW7NaEOo0sznL!L;#AHd+%yTj2x9(J#8BUX-wa;V zR(zt61R62C?Y(!~jUC?JZEIxKFujI$czdQ#N7D}1Mjc%i)3I6P5z^u6(b2Nxu6cI> hH^{D09W9Q%`9F;-J%X-`o!Tmz8s^nc!yfyvD5%!CgY zoCHFbF-beki%O}s1Iu=mdfb7b5HR2%2vHEHSP;d^tjBHV0v=QB;iMebyZ`6Ccl0Ia z^gU+sdPZSw$K?_w>ol5IHquCo2|`$LZCu9|Nxd(QOJG5@S3uhEh>mYwnAdxPnY zvh+qIX|mCg(MOucZ90$i<4X_d{FQ$Ehf$qJ`*E+qyL29u(j9adj(EQcuNu6kAMY`E zq94ycsCRs&ACDZ;dGanE1TLHRu{ig)S__PDISKuWA?-TfZfm=S+1vO6RDP4vO+?0l}3OMyIQH2b;F-pErAu|nO zrwE*8moBpeo?Z>|l?psN9Sm##jt8Y2!BO>{L%LVss{}qs;57pOp1|t`K3L#hftxw! z^OvNJg2N(IXcl<-i6!?Afe#hp!vg=lz;_9Jn7|`J!C_<#?tVq!_X!oE0>5A2T>>92 z@VLN72)xJOLE8HV1V=)sFjC-G1fD1Gq`)5(cuL@-1l}v~(Z>1X?moex2o)sbw9`Px z2s}&RV+C#z_&9+N7r6CX9>ilHI3D`eAxZfHpCIrl0?!xtEP)pYyj0*51#TxE#EWJ& zNpMsN6^yLQy{iOnWNFT81YRh#Q!nr$fqMlWoFX_j3XWodHw%2Kz;_6In!v*XpDyrS z0-vGtAU6@g@dKg4D*~S>@TkCN3A{_-vjrX(_#A=vTr-{(+Jd=)BXKR3llwwWr;-(X z6ZvX;JYdoisdrj%zwLC^DT>r)e0&_&@t@#o9$kt%()Hw0lN!qSi=-(Ls|l{(B~1xf zjdT4LY0A55lXGQ(v)x2Qm(g>raY@o;rbcUlwVbaYaev5o)W8S;b1KVP(oEDuAe4NiB#?V24MV8 zNK^h)lUzSWn)0Ta;JSh|)VxPkB)db1;ztC?BfL zT#qG9c~JFo{Qzm&Q?-Wcp`Ss9_HFlcqGKHgmm~G^Hoi%k^&3l$O*Qt~*InI#R2+evUMyA+?n2t)waY zs8hIphBRdzRpC15BS7g!wQ#VOG^H6;;`(XQlwQ={ulX4uO<6@va{U--$|h=p>k878 zMbtRg3rSPq>7*N!JSbS;%a#P+zk^?iifJ7zp(;6EDpypC7K){UPm%Ri6( z>Y?i!H^0K&N z#?}~Xx8))BRokcnAgO86d#rQ8HG3hJ9LcQhp{DnUiUfH?ZH$ zyMSxce7C%weLjDFb`O@OvSXjMhD};f0k(dDA~&%Y7iQ%C8+x3}dkY?7p}eqR~Vj#4VB4DGY_B1eIsQHl$FX%FqR+K~7vJ6_|y z%Fthq;S*`8{cNkX53%j!3%zDdAez_}SXn|IsSI^nlLkz&J*6YZ_Cn>Gw|{|$&EB@t znj+`6J7)9Gu5!|DO|nl*R}|JDZJO+DPgu;`{p7%qDU+Tk!w6IJZt9!-1U!`;hpex% znzB6kI@?mF$m`f|%Bo~P`?4(0qI%GHsFe@Bt<_q~MlPH!_prwoE;7@+Oik~$m0Pps zv|3NI|5`YK&*Vj{kNIa;a9mottzTnmIyehp?`XBQuv-hQatphE(YVnqSgj)Sb`M<5 z-PRVQZ72s6t&(&Pdty|>!t_M#qI^m$Sd7m-6?q2u!qjt zLj$a-FFMOdnFi-DrF_N&iTrKTt=xyPrVb;!)A^2I#qDjItyy-=zlCisFO>h4{l0vf z9A($a7hB%K*vzek2rAaF1#vcg@sH&LtbOqWxsM%MyaD&NC6A0zFu~LppJ>{)lx%Jt z3RQf&SWCSDf=;%}ne5j~W+V2}k}+Ao!1AAIwZ+*tOGf0mA?(}En!4!S?#2Mz%I3rO zw_Y^2t{A|kEiDL6r+|)twdxoqDy7E!Rd2BoxX%1#oLt^FAtZA_xz(E*b5-{*0OCVX zKGs1Yz~bU~@jJyAPhd9Eo4mP$hSvgbo4!w>JE3n<6(8xf`i5g%aM0Q0j$2oBN0NQD zbPU!g*FO3uVT##owVa?ir0(buJ6t*)c7g_%`rM!^i9Tr^&c}c>bKcix^~sL#b2{-_umpL?nfYev~5^#^z=L;pB2+=6c`m7%}Fc?{0tc9cq2 zv>sq{D;~`K4{&=&yR{VcXv!%}^>@r&F(R`Uch9r-3T2+l$tU+VO+QL!Wg&u#<1~;k zBu(pO+@$9p0ZV;CgTn)Og{c*J!KFwn#c>>p8hfbRTE%xeH5HJMuB>T=qa~&8ODld_ zveHUFQ8$cwbUsoy4RvEb+Ili|{F3fW9l}jzNS|pgCBx83`?k1zM#j5MqUmqyZ@*%{ zTE0Zy!jj89?ET7tY-Qy#)A2vCJ(Y!WBTH1ajG9z0NoSY3^3aQU$iO|Q-Y`7R{8p8e;;Zqqv-#Bv{5Ae(kvjIDgMaiHmsJ7RCIe7Ry;;e1Ln zfx&q4E}|9l=w?)v${x3O%v*qub7dD&9N0;KwV{*Yf*0?upx;8YX?QI#<8I3P=dnS8b4!1OU@_MK5 zX-A-;5x4$L{(!S_ve)Nn^adt-;PExoI^hU-JT5~;!x_Hj#`(~?^ntl4UHbR;#lK!2j(blk5b>%=x{3ye#Pr^`T+|S z)bc6l!yRY{Y_cg0Zhyenv^vn>ar+gAyH?rYSm#un>*42(H@KBRy;EuOJAG(_Mp)sYdDO#;stkqR5sG;zwR|ik%4&zpg)y#kdp5WgkK3u#Hu#*Y1FlWVhK4{rwOmaN zDfx7*b~_po!K+qRH+ozFw=+=f!~AUhX|Ng-60x(<eSJ#GxTj_Ol|pi`k5wH~ZZE z!|lgB=~Otdrw!{J0VmDDN#ml;_BG*fG&)Q8Zn+xV7$zp|#NKa8e@911Ni~+Uks7RC z-B@e;cF?vp=-%eSYFA^YZNdWD(tFMRd|>mC7EL>Zm#10NJ_u^suUlek4`!Hx_4w$B z$E?PsX%?iD2y879P&}Y%UEuL1P0I&Qz|Mu8vR>2LDGv4!=?$8egbl;eM-9tHO{;blnSE13LgI&9Y6?s$hdstEO$nP2}G-?L2IJr=|_Z0IFWV0ARzg=V24D zN!Z9PO)JM|kSOdb*f^{UHoRNYUV@FlMv23ohE2d;A^no3{T+77%bJ!o5b>~i1L;SV z-(chbHL&Hd3D{MzrN2ePuu0fv*vKJG>mhwa(>@`6M$;60W{4oo*1{%KO_T5qy7{`M zmBLz(vYTN`VRyn-!M*}p1KSPjg-v7+!bouQIY9I#yZwhr_<%X~aFJZc9y>f?@>-0Q z(xXuUrTBdKPCu_{4%|~AHWy66fve_3XH}Wf8p#E?PJr~N^nk4p z*ha8nFWw#34(0``7vdsd=LJ>{-yyJ7yV&(NmW)WehLyws;Y%WgmJ&N&h09{u3(wgERW%CH=$nuB*DvV}Fys&s^u5!;0VY$b;FQ_g2YQ*#XmiG0TO2$Z|54lQ^7#uXZOtD9TB< zk#C9J`QRSeG^&}IE*0jahp6mhC6|`U1KHL~McGG;ZjP{Hmu6-kHF`bD?p*poc6ttB xHv96-?DU9tu}zm}4oUxM7CEO^S;bCXo`MnGy1W20nfl=ZImvwddho;7{~tqWLEZoW delta 6223 zcmZ9Q3v?4z8pku!7TSs=DAj@m(lXFOOCLbxA%(WIC6ox2#T8xD=uxc5qats?q;0B% zww^4PRm9Z|kFyJi8c3HdrO+TOAkT<^pdRfiYDPgSx}3_{b6|p9Zp%s{J`1y|;x~p}> z8hjb6R;&q;pw8oLyK=f$li_mwX4x9`vDB_DM0G`^rHVj8%tN<1qd4AZ#AF&~oHR$S z5#_-wg9p2Cb*I6PcHs~B4Ib*kt0Me@!JVe)1sWrc_AV8+MR>RiKO5m4U3hz>L-8S_ z1JU-Q`od|0(`?6@oQ9xdyX5G83!B0cA0+V(iI0_dr^Lrg+;Ykg)HW@*D0KZ~$)QF# z8LPx&B%UGhIEm*~9i^N?L?GI9P~Mk<^hE`BV9=@y`7^BIzyV0 zu^tlo7-`DEdQj*ONK@|B{X(~prU$6*6M7eECnaIsE5LRFlzerU(3?op0_#;mZy-%c zSa%4$mNeyFy+Y_Uq$%g>g+kYmrd+G%3cZ3fMo(vNK-D+b?oNg8R z+78@9m6Y3bivZtK0HrkDB=lFLDV6CR|D_rDl(dy}yU=GyQ|i(~LLVbdDN7Fu{Q+sD zD$Ib;Eu<+!={})%ISJ64f$kMxJ84Q)x=ZLyq$x$|RYGqdO{qzD2)&jxr6j#V=ryD% z6X}IQ*N~NO$~4>;cl0S@d?Hr<10P zqKAYoCrz0|4+>pOnlgy)7kVsdN*ub6^rS(<2~ft+y&_;JY04D3OXxJxbgS!CLiZ(2 znL&35P4#DcS+Cg9Yqgh@Zp^9e%Ng8&JK6JPIE;w8pxI5Sz}A?b9Kjz&@CgI=3OqMx zn4;7N-WdG7s;p-2Av%uphpxi$^3a3ol>uyLW}~v1O~^{>y_s5y+eKzf=b|KAecY1G ztTHPlUO4h=HnWDT6vS@J>OTa?v^euYTWvuao!UI#Eb3vaFBl9FzIdJU@Kcz9!*TsK z2RgE5bW;jh>4;k8rND#`FU~5GPZYYnzEFAG4_D6G~l~0 zPE{7MzZRUraq;;1$_wn$`0)v=F*WwaEt-{OPACUkGQp}WWUo(H=_Jn|9xa+x^k~sz zMUNk}H@0e3$R;*e7w)M&bmQCVlZqFbOh3P4_wYTGitL^(njbj|1dURh=c>)KPRl~# zvp4P#KD+1J7QB@f+K#xjX2dp-&$C1GBAVEGScTrn?4DN57lB*Z#=?8j{7@O@x+}QZ zZ1wB3t>mn|VK%?~nIH|Ck9|=%Bio9!soLrvwwUW)AP2gf0kppeJq#CIHWv9|ctW8|n!tnx$YIdw61wvVeIe+HzLYq!*Z6Xu2*B&x`ABjnHJXdmt3G+dUa^jBX^VAhiS^dKhIRzFe`KJ+zwFS0xBZ=Z0i)Z%Rze-T?*lC8{V@05(d$iFNp zwG^XoW_LD%@;H{j#YUDsq&&|WO4F5Q)>OI_=UHV_Q+=4y@RiRwuMI0^_gbjD>v=r9 z7X%lntBz%FmW@&J*txRQcr(0*-B~Vnv#d|bk1^<1?Gd#Z*7QRxP8IX+yWMBZ?z1s$ z#H1n4CJJa=q*b(Fpr&x@cSeg>fM=Vp2FX>`h89icAthQ>IQ28b-v+|d)Z5+1H#XQV=(#_fE zGTqGKe0c1hU-u{D0pG{&`3}y1!vNi+|Dw{KER!%as2zhkb!M?=1VsK2#Y#a(`?Yx z`Re4efj>>1ps4X52hLBM*Ij*Jd7$*6x5`JnhvzZgQF~(NpFu0;0WMTE6+LKcENH{) zSJ8|Ie`l+`9DgLP8iAN*Q+Pc~+B8bFWwWW9>iQnQK=F@cCGaXS=`Q}EVOebYj0ZK%guQcehX?j|G0+pH{F3be~(d} z&*9O9%QzNrZoz!|V4uPY1ebE&3?5v@c?Nh0b{uT(a?Tql4z`K(3eMYMy|B4h9m|WH zJ7BAD89%HK_7HJgx&zh$8;doyyv#X$fpVH&;d~`dd}}yA0bBSw=gH_mXgxXrYk314 zwgR>t*7_#rCCGufurpx`VduhH-s1evuvS<&X?) z3-KoJ{SozGgRtvhL$F(6+hJQ_J77ZzJUSy~?gPXDUlr+MXQcSvGx7`mlNE zD{YTRq1W|U_<5IbPEUAKNl%L%rV77?C9J%qj9uQ7fEKRrNyo8&OD5hh%3G%4xS=Hn z$73zilAN5U8g;E1u$@@svrd-1H#5n_c^P?7Z1|1Fvd(a^`Fp1*)vRT2rc%Sc-8(I= z4tu8u8{0Zi4L;8vZ%tLLYgtX}6lD%O)jEBsg>fGJ*hSsNr!~_!i8+9JgHfzhMv#5#6 zQ3dt`25Xv>5ECEsrE(Cb6rTZK2{zjkIHy8v9GDO6aX}1L5odbFbQ$nD3DT`%!_J%` zv1MTEz~s0Funw?FDb5FG-*AhSz}EzJVFUYqe_5Xj+~MPK$51K6&w|anau?_OSYz8j zHn%NB?F_OtZN+NSX?Ch@gKGPl%?XZIo4;n;gX5j?-*BErRZ(a}v~x8uv27wu?BXu0 z192%`SSIOz^IeA;ao3u)?(EkyX=yzk4Z9JZy82xD5 zzLQ-$o;}95*$9e$`HFswj(*uf!=gmLfJHw_r?8)tJK5Y5iCWj+z5EKJKQdZ#$WK zE;}*WMc-yN_S_^TmerigNoa~(rirzj8 readFileSync(new URL(`./${name}`, import.meta.url)); + +test("the manifest pins the committed native artifacts, their provenance, and the current launcher source", () => { + const source = createHash("sha256"); + for (const name of sources) source.update(read(name)); + assert.equal(GROK_ENGINE_BROKER.artifacts.sourceSha256, source.digest("hex"), "launcher source changed without rebuilding the native artifacts"); + for (const [architecture, pinned] of [["x64", GROK_ENGINE_BROKER.artifacts.x64Sha256], ["arm64", GROK_ENGINE_BROKER.artifacts.arm64Sha256]] as const) { + const binary = read(`artifacts/daimon-engine-broker-${architecture}`); + const provenance = JSON.parse(read(`artifacts/daimon-engine-broker-${architecture}.provenance.json`).toString("utf8")) as Record; + const digest = createHash("sha256").update(binary).digest("hex"); + assert.equal(digest, pinned, architecture); + assert.equal(provenance.binary_sha256, `sha256:${digest}`, architecture); + assert.equal(provenance.source_sha256, `sha256:${GROK_ENGINE_BROKER.artifacts.sourceSha256}`, architecture); + assert.equal(provenance.install_path, GROK_ENGINE_BROKER.nativeExecutablePath, architecture); + } +}); From 9761d8d4ee920b1da4ae364556719e383b0ae03f Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 04:51:41 +0200 Subject: [PATCH 009/124] fix: keep the sealed prompt fd open across the Grok worker exec --- .../native/engineBrokerLauncherCore.inc | 25 ++++++++++++++----- ...ngineBrokerLauncherIntegrationLauncher.inc | 1 + src/runtime/native/fixtureWorker.c | 2 +- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/runtime/native/engineBrokerLauncherCore.inc b/src/runtime/native/engineBrokerLauncherCore.inc index 1bb8f45..c093307 100644 --- a/src/runtime/native/engineBrokerLauncherCore.inc +++ b/src/runtime/native/engineBrokerLauncherCore.inc @@ -306,12 +306,25 @@ static pid_t launch(const struct dbl_registration *r, int executable, if (prctl(PR_SET_KEEPCAPS, 0, 0, 0, 0) || prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) launch_fail(status_pipe[1], 3); - if (dup2(prompt, 3) < 0 || dup2(capability, 4) < 0 || - dup2(output, STDOUT_FILENO) < 0 || dup2(output, STDERR_FILENO) < 0) + /* Received fds carry MSG_CMSG_CLOEXEC and may already sit on 3-5: dup2 onto + itself keeps close-on-exec (the prompt vanished at exec) and an + overlapping order clobbers a source. Lift all of them above the targets + first so every dup2 below changes the fd number and clears CLOEXEC. */ + int status_fd = fcntl(status_pipe[1], F_DUPFD_CLOEXEC, 16); + if (status_fd < 0) launch_fail(status_pipe[1], 4); - if (executable != 5 && dup2(executable, 5) < 0) - launch_fail(status_pipe[1], 5); - close_other_fds(status_pipe[1]); + int high_prompt = fcntl(prompt, F_DUPFD_CLOEXEC, 16), + high_capability = fcntl(capability, F_DUPFD_CLOEXEC, 16), + high_output = fcntl(output, F_DUPFD_CLOEXEC, 16), + high_executable = fcntl(executable, F_DUPFD_CLOEXEC, 16); + if (high_prompt < 0 || high_capability < 0 || high_output < 0 || + high_executable < 0 || dup2(high_prompt, 3) < 0 || + dup2(high_capability, 4) < 0 || dup2(high_output, STDOUT_FILENO) < 0 || + dup2(high_output, STDERR_FILENO) < 0) + launch_fail(status_fd, 4); + if (dup2(high_executable, 5) < 0) + launch_fail(status_fd, 5); + close_other_fds(status_fd); char *const argv[] = {"grok", "--sandbox", "daimon-strict", @@ -351,6 +364,6 @@ static pid_t launch(const struct dbl_registration *r, int executable, NULL}; syscall(SYS_execveat, 5, "", argv, envp, AT_EMPTY_PATH); erase(mcp_env, sizeof(mcp_env)); - launch_fail(status_pipe[1], 6); + launch_fail(status_fd, 6); } diff --git a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc index adab67f..20d6849 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc @@ -110,6 +110,7 @@ static void org_cases(void) { char *out = calloc(1, r.output_length + 1); check(read(s, out, r.output_length) == (ssize_t)r.output_length && strstr(out, "uid=2200") && strstr(out, "--always-approve") && + strstr(out, " prompt=prompt\n") && !strstr(out, "EVIL"), "fixed worker boundary"); check(strstr(out, "=--verbatim\n") && strstr(out, "=--no-plan\n") && diff --git a/src/runtime/native/fixtureWorker.c b/src/runtime/native/fixtureWorker.c index 6baaad7..63af5b8 100644 --- a/src/runtime/native/fixtureWorker.c +++ b/src/runtime/native/fixtureWorker.c @@ -5,4 +5,4 @@ #include #include #include "engineBrokerLauncher.h" -int main(int argc,char**argv){unsigned char bundle[8200]={0};ssize_t size=pread(4,bundle,sizeof(bundle),0);if(size<6)return 21;size_t offset=0;unsigned provider_length=bundle[0]|(bundle[1]<<8);offset=2;if(offset+provider_length+2>(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output");if(!strcmp(provider,"sleep"))sleep(30);if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;i(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output");if(!strcmp(provider,"sleep"))sleep(30);if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;i Date: Thu, 17 Sep 2026 04:51:41 +0200 Subject: [PATCH 010/124] build: rebuild native engine broker artifacts with the prompt fd fix --- src/contracts/runtimeContractManifest.ts | 6 +++--- .../artifacts/daimon-engine-broker-arm64 | Bin 67456 -> 67456 bytes ...daimon-engine-broker-arm64.provenance.json | 2 +- .../native/artifacts/daimon-engine-broker-x64 | Bin 55392 -> 55392 bytes .../daimon-engine-broker-x64.provenance.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 4dbf679..0f4b786 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -69,9 +69,9 @@ export const GROK_ENGINE_BROKER = { }, bounds: { promptBytes: 65_536, capabilityBytes: 4_096, capabilityBundleBytes: 8_196, outputBytes: 65_536 }, artifacts: { - sourceSha256: "7830ff308420cf1b84aba118903c1e8ef40f8c2a320136977a1e7f9e3292a734", - x64Sha256: "b3da1618ca218ff44ce178385f94146736439acccaedaf047d41709ab0aa5d43", - arm64Sha256: "b4f41d429db5f8fefaf880f9ff9d64e76d0687829e6002fe7c7052cc7a41af6e" + sourceSha256: "5eadf15faeccd5fec14f03e701c1b7001d7f4058941987070bf5e512e9984e5e", + x64Sha256: "51dc67387cd0c9dbbbc36b577b5c98ea296ecaf3f2f015be52eb4df98affcbbc", + arm64Sha256: "5821e547aa6e7c5682eaae1a3f6106138ccd5b5666909c6e135862a61b35cc47" } } as const; export const AGY_SUBSCRIPTION_REALM = { diff --git a/src/runtime/native/artifacts/daimon-engine-broker-arm64 b/src/runtime/native/artifacts/daimon-engine-broker-arm64 index 0fd341da3ef4f6496e05941770aefbb39a3ad5b7..2afa07296b6cb9e546701b836575a7752a43c5ad 100755 GIT binary patch delta 2387 zcmYjTdr*|u6~FiU7F4P1nlhWxI8N*=ij9qvsg9x&Grj%YMUgvm z_IJ+joO{l9&wb1qLndR$G(443rU@c-ONPoYY7V#fvUtLqnqN-)= zTSE9rJgKx3@{Y9ao?2E8&I2xHgSQ82L;X2fl-7oZas zTPO{Lo>&(!B@{7y?lnS+L!zQjrAK+6BcC4QFFR`Kn>^y!int9mg9(-0kg85XQfJhRv8b9g*2s_4Ip|Pmy6!4nr{|L@|1IISnus@=N8FScE&_>w z^W>4LXMpM+ohzu_WR&p#KAfY3Z9J(Vx9D$H;|nR-`r|r5dYlxMGW;hHF~R$s|>F@Jsz+li~Up zz8P#l)US_^W{@gHFS(ustbv9I~-;>+qptJD^vSK;*q9{dyqs^5NGM(!Grmw~<61 zx0;E!bFcf^4no``@g(K85b3hG)K_5s4IhX^LZ%*_DPhZj-IFEkYFS1w#@QY6OioFj z*veVAy&67i@C((<62O@urS?*0?kHu;pvOZ#50|ntL}PK#O^~;AjSYr1wgUP=TFs&r zxj}bpJW!wiQLDmyDjX{zPZ=6}+g`?S5T_`l$y3Jcfim`={LJ8Du?`0_4>;K8S*wGK z1p^LtChB09AIb{KqHpOnY}Sz-l*I=64F4SNYZRrf|NshPN984J9=p1m-~Pu8#IdQ(#K zdSuMt9%$9FNzUW;Myi>d~c*U9&{I~C+U?FEs z**{P%+9W3Z{I2ke$H=@qwszP%VdF)L9fH1l`T+P|sUt?}J@@nu=w0{pM(9sN|5bBc z(uY_@zd{aJ3Az5InLpZ`Yu<=+VzsH$yr;ROMo}$)9Z)O>Nd{eVVr=qMb9VcYB@=#TBJRzq<*FGIEV%CX1DloXNtaBjB|KMy$nO$I2 zT>cn)g)5J|L_M56vN@&grhE<3u!*T-dHAwfPFrf}G_P-IpoRPwEm?GeA8Ppy-OVqz zSX160WaPDNWSp9f2(Iwh*3ha}3r#6PlKTl}KtI9Pw-xhi#mOO0TY-&k4n6O7J@j?l`}&vy9g8gk8WwBxHA!O*Xp{u@g2+AT%#aOrp{;E^AAx4hy7XO6oAR+wX1o z$(=dxJLfy+o^yZBd-K#?CUuu-cq)H*fMmz;06TTGSlK*l-NJ-M|FV!e`6HF*8=9Fe zTSmVxM2zH;r4B;=Y}k%aGg}NU1TJfXcf^|$kwq1hHYavfRmgNX|C8f!JT29m=;6e# ztE(yfl<%*Zp)LH6&KPy@#@Y$0C9c$l=UHo*QQ?|QJfH1U#xrussGX2u60hOWh6r^g zE;VeRG?e(!lDH|)$*|lDgw!TPW4}y$dB3lQKEeOsTS0qx+V?o(mM%y@(87M zys_os1*H-hNfF}hA*3WDmyhal#b_-c|vut*9m;)ShMb-$=2L0ezrG97sxIci`xLQ@qaNz8<;M+%83 z&Hf4=Xl-iRZzajhMu|yOP4E0x#Ozk$%^a7QiZBr^&^(wL2Z!ouR8Jp)ZMX8HtqY?{ zIq`nh+JVE7JUZl*5n2Xr#cgf^90Avf)Fz~qv(?o=b~98BOdE3+5bs={12HL+8aN4G zs)Ts;&C0m3>epcFmTNv;F7*r&H5&LUh!BWNeag5o3HR<9GbY_>1X(<7;566@Y*9+W zaYBN+T;|yi{s8zm_^UBx+}JCJeOV@>5$IkLPw(?97uEFd_e3Sqw~3NqsE|BlGn4S- z_BFp+k35mwC@KC%h%%#4B$pJskl+&Rd_tz~pQvMbKzF8&opY6ISA!MUc&rVL%GxWsNe2CxfW@8Qyy92thpy%jP=3eh*U)rqNt)hcob~f&1 zUz9D>ZWY$O>?Ng%owihHM)9Fg6Z=m=rFI%`+F-4k{<^VBd&nr_Pc^Y`3+Rln(V6=k zMC;s5wa?4w%$W*0^Xe55yE(z=?rgl?#5C|!-kv@!ThaC9v%KPC2RQS)7x#Mpzpk)?}t7F-Da1k4dGi>wG@7iwMHcJtc{S1^Yi$T zRaGW8x`FGf>RWCG8Ysz@cLpWPvqo<|GiT1&cOn=qBp3692wvVn*-W2Fvy!0zSK2Fs z<8uO^H<92v*>c{DKnvb8|D1%I=fwR`z8&|CRNhBXVD3Wr!FDG-&Oc~()g4AR8kd(r zTN4v`uv#T-`|HP78r%+94cM{a!4$8)x3ze3mX-0=W!blQ|GmGX+c@o5SKM=1#4>Uz zuBv3T8Y|OzxMKzVfRA;w(l-8eM+H5?^Ew}(oIl)YE8a)Q$i7WvoSG3SFBsjJ_)VvU z7W|ArVa}6l+pA_AxyY_^^g#iXnGxZ53mT^X$P(b zt_8+{+kh$HFmMX^is2s@LI>(V6IyP|5g`@>zXJ9E)xQZb0-T%>q6qKJ8DPCkHjvOi zgy;q_3G4y(P6}}lsJcJT^Nn2sjUg-sDRCp0$=U2i@csy*DdJgf?a5epizZZ5%ErB5nfaL1Dea$Y})Tj6g%=`8(m{wJSKfBfM`|N3_!X)Ea7v%|c5$ow-d zZ(S)VFLVFa0r%SXg;**iNm*&Ewq kqD}czH|(k2*eTd~w|+xn)*R}~>)J9K-gW3FIn*WnAIO2iSlKnkyNWAld@Q;uC<@y&&X`CLS9XD2T}K`LBMTc_$x88J-Db+YjqB9xAyW1XIJ?=p~gX6HHY?|I+% zoO{l>8}dYiJP{rpuWza1l&8fzmDeO`^560rP5U3?|M7Yju8HMMBr?~`r;ya#i<)hd z2`?t_yGfF0)hvl7#bTR=kMho2v6eR-)N-6WbcNDNnHQ?|AhO)EQ2Be1>VQyPLH6f8 zrCF>c7xPS-Xe}AdOL}}TlH+WZ-I~@2%s&{3AuoIzTh%92ohMIN7r2iwzpl*eJfTB^ zoP0xV6^q_3VK~67Qa^GyAyqV&%E^~ipBtOo^s@?vk+Z@Qu$-~<`FN!Jm3BwN?s>R1 zZRUrlk{(F^$_1Cys_$m1#^{*w*MMq#97*n4P3~gY{&{FOv9%>2XQOXTxw(kFwGBBZ z49M|K!bpA*9i&beyAoLMaq63qq}i&P_S+{lsKr(qajh|=&lmrjAw zpU}r{HsnaBq#jW}qL)RzEbMB+Hv8CCOXuVg6tZ=^!x+}WiPGQl`Y_zVAloQuSk4Wz zO^j(&md;ozsvQhsdr&3iAP zYR|bi&R$byPO+SE*@W#qKo@{)*an?Vsx^;d87`rugPB3Q^rtTkKuE1JXG4+PjbzqN zZeXFYdJEE#Isoe|ER>#PamWpD8Gw~KSx-Qj6}Bpc%A|~qAn%%5*vaBi#FxT#Md7+0 zR<}#7>t(sf3nK5dTGz#5wt!#5b)~4A#$bpQZcq!)y2$PPc=s|ySmbH8u6pFLjC7TC zGu9ktb5z#Bm?z9)Rd$lGfXYO(3Eesw35K)o*DEu0F%|-oofj#!)nuBno0b%OL`D(o zRQ6R~0?RJl_r>AL+NB%tmche~N+3rHQjBH<%)h4 z%R9*Z!o&?lcPLfv8`^9|j&gG`er~Lq-DKH07Ag1dnQ(OTbsC<6-C}tmu*LY>eM)bs)VVsLaB&<)URXbIGCfYJ}4Ca8yTXcyE89boZ6 zO7B8FP+b)An<$Nk?u2GQLr^h_b92s*&=?>9ErAv_Q@RssI!F)tcTOTh-&0bBJEx`?ZMCx&c}6nG-eY!k-p zVJs7@nY`+YS$+%7%L-SW&2Vfu(X|uUYhaCF6Z2}pMo5P*#%SM1X>mAPgdT%4KyLc1 zGo4>!bJ(|qCmTNlPPyDeX*;R&&mkrLcz%>f{#^bT`O$wsW4lJS2D128(h_*ut^1AA z*{lu^TQECcGuRY`s7yKQN7xAR;vZp=xC3!uu@h?Jz*xVDEDbC}t@TzK7fdD%KkVU~ ay~({-b$k=y`_uK!pQQ!eHm8e^`WpkZ%jyNbqm_UYG=YmsbLS5W^42Wiu1&Q*gS`{cB_L_aOBVv7XSwc`xa{ ziKAEYBwbS+9nVW$P!%T#uG$`5s~+nQ$KjEP=jPU37VApsN@s>|755vrTT0Fwksznt zR!da^d&N5uZdG?65JaoPQZ1+buKH_%l(nVnDp(G%&K9xt*)E3((mRxLXEZkf?w&Tw z1MJv8;eRL>TvKO_&%?H0j87W>Tvd&jo{l*sUl6ud)fQ3fF5%m>Yjp#B`c6win7^Go zp-$diq0*!<-?w~Aqpxtf*^;c3@ej)V8qWp~>yQc7s<8~PoZA1ahVAAswkA0IcjTj* zN8l+Zf?}R_1cLnDb^3DtV&6~5sQrOBR*&xd^PafNaC#pL0y?I!>zk;JZfqekH-<0A?AyC`08N2e2I8S-3GU81nKQ$RCeQBJGlopy8*5>g}lt;8^_zTq9 zEVmD+rhImQ+T1C|q2u=$6H^`ZbYaf)_hGv5`G2&A-Y@i$QR*pLPugj)C{tJYE4^8i zGRNy(E~eI@E^b^^%8C|=V-I5d&UPl) zSeKV(7B5bkp{7WWQcTePsJA{4`uf z32nWcO|HhQwMqEMN;hqE)s7f@3CXmp4%!6XPZe=}x>CZIQJpmykfXbJQ) zs0Z2xt%gRS0q8Ag5ISxOO|)j~$pX5%&1q<9VyubQw;jm{1Q^5FDgz&cAfWr&rZ5js z5||vI1?_K`OZPE`qf!P>Z-7-oJKGaUAsuMXtQ?__`?vU1u;eLo#=*wGM2&H8CW_t# z8|IhnH0h1{LDGdszD7E{?gNayj%PkNTLUK1cf*NU6}Y{(6|Op4;q-!)PGLL1l0JSs zuK}!@c7_wp{czB=%5dQSSvX~MDD0dU{St@6KR^sS{t|G@572Avw6Wu9TG5e0c2Ivu zE?Gq{bsW+4#^_s-Z1N)gJhIle`wzwz^LZG2!~A|vfz48g%9Oi)g55=4^Aju%J#YwY x?v%O7V7%T`whSz&&JER?Gp5tMKQ<95l-56JB#S5+$TUf()Dlu?^}w_G{{b8!2i^bx diff --git a/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json b/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json index ffc0023..405f166 100644 --- a/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json +++ b/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json @@ -1 +1 @@ -{"version":"daimon.engine-broker-native-build.v1","architecture":"x64","target":"linux/amd64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:7830ff308420cf1b84aba118903c1e8ef40f8c2a320136977a1e7f9e3292a734","binary_sha256":"sha256:b3da1618ca218ff44ce178385f94146736439acccaedaf047d41709ab0aa5d43","install_path":"/opt/daimon/bin/daimon-engine-broker"} +{"version":"daimon.engine-broker-native-build.v1","architecture":"x64","target":"linux/amd64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:5eadf15faeccd5fec14f03e701c1b7001d7f4058941987070bf5e512e9984e5e","binary_sha256":"sha256:51dc67387cd0c9dbbbc36b577b5c98ea296ecaf3f2f015be52eb4df98affcbbc","install_path":"/opt/daimon/bin/daimon-engine-broker"} From f41774ed9194761483cacb68f36c4ddc37b12e53 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 05:04:00 +0200 Subject: [PATCH 011/124] fix: pass the Grok provider capability through env_key because 1.0.34 never runs auth_provider helpers --- scripts/liveGrokBrokerSession.ts | 8 +++---- src/contracts/runtimeContractManifest.ts | 6 ++--- src/runtime/grokBrokerWorkerConfig.test.ts | 14 ++++++----- src/runtime/grokBrokerWorkerConfig.ts | 23 ++++++++++++++----- .../native/engineBrokerLauncherCore.inc | 22 +++++++++++++++--- src/runtime/native/fixtureWorker.c | 2 +- src/runtime/native/launcherArgv.test.ts | 9 +++++++- 7 files changed, 59 insertions(+), 25 deletions(-) diff --git a/scripts/liveGrokBrokerSession.ts b/scripts/liveGrokBrokerSession.ts index 0e0663e..05cf30d 100644 --- a/scripts/liveGrokBrokerSession.ts +++ b/scripts/liveGrokBrokerSession.ts @@ -10,7 +10,7 @@ import { decodeGrokHeadlessTurn } from "../src/pi/grokHeadlessResult.ts"; import { readGrokBrokerCredential } from "../src/runtime/grokBrokerCredentialReader.ts"; import { startGrokBrokerProxy } from "../src/runtime/grokBrokerProxy.ts"; import { DEFAULT_GROK_BROKER_MODEL_POLICY } from "../src/runtime/grokBrokerModelPolicy.ts"; -import { renderGrokBrokerWorkerArgs, renderGrokBrokerWorkerConfigWith } from "../src/runtime/grokBrokerWorkerConfig.ts"; +import { GROK_BROKER_PROVIDER_CAPABILITY_ENV, renderGrokBrokerWorkerArgs, renderGrokBrokerWorkerConfigWith } from "../src/runtime/grokBrokerWorkerConfig.ts"; // Explicit live auth/transport check, not the Linux native worker/isolation E2E. // Read the operator credential only in this process; never stage or rotate it. @@ -38,10 +38,8 @@ try { const capability = proxy.capabilities.issue("local-auth-probe", turnId); // This local transport probe deliberately does not attest a native worker. proxy.registerIsolationGuard(turnId, async () => undefined); - const helper = path.join(home, "auth-helper"); - await writeFile(helper, `#!/bin/sh\nprintf '{"access_token":"${capability}","expires_in":600}\\n'\n`, { mode: 0o700 }); // No MCP tools are needed for this exact-reply authentication probe. - await writeFile(path.join(home, "config.toml"), renderGrokBrokerWorkerConfigWith(DEFAULT_GROK_BROKER_MODEL_POLICY, { helperPath: helper, proxyPort: proxy.port, mcpUrl: "http://127.0.0.1:43124/mcp" }).split("[mcp_servers.daimon]")[0]); + await writeFile(path.join(home, "config.toml"), renderGrokBrokerWorkerConfigWith(DEFAULT_GROK_BROKER_MODEL_POLICY, { proxyPort: proxy.port, mcpUrl: "http://127.0.0.1:43124/mcp" }).split("[mcp_servers.daimon]")[0]); const prompt = path.join(home, "prompt.txt"); await writeFile(prompt, `Reply exactly ${sentinel}. Do not use tools.`); stage = `model turn ${round}`; @@ -52,7 +50,7 @@ try { args[args.indexOf("--max-turns") + 1] = "1"; const child = trackCliChild(spawn("grok", args, { cwd: home, detached: process.platform !== "win32", - env: { PATH: process.env.PATH, HOME: home, GROK_HOME: home, LANG: "C", LC_ALL: "C", TZ: "UTC" }, + env: { PATH: process.env.PATH, HOME: home, GROK_HOME: home, LANG: "C", LC_ALL: "C", TZ: "UTC", [GROK_BROKER_PROVIDER_CAPABILITY_ENV]: capability }, stdio: ["ignore", "pipe", "pipe"], })); let output: string; diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 0f4b786..f122193 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -53,9 +53,9 @@ export const GROK_ENGINE_BROKER = { systemPromptSha256: "2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8", // sha256 of `renderGrokBrokerWorkerConfig({ model, reasoningEffort })`, the only accepted config.toml bytes. configSha256: { - "grok-4.6": { low: "e09c127363094a7cad560586d89e994a9e6117b9c548feaffb1b5b01705cf363", medium: "b90807d3c73651a1a3f0bf89eacb0c73360e7cfc34d10e0185a381a27b40a718", high: "045171e44ba44b09522589ba85770f7c09fb8cab3e3f76fbb88cb534dcc01018" }, - "grok-4.5": { low: "848df2f71d0a88cf1185728cd6e0b7e15238b3a7536bf1538467f8c4868b0647", medium: "3e84795e834cf7b5857c24070d9f91b951c9c5f806823dd29c92cf88635a9318", high: "792f90bb7ae5154d6e002419f5b308e2ea975fc55ae7c324952f928329238f93" }, - "grok-build": { low: "1be3438799f9023b6acdaec991c139133bc791877f86a8b139129ef4b7b8386c", medium: "cadef8a2fcca778b515fcc10bfa8ab2ed8425fa46f37dc3a64095b477beb914a", high: "cb3ef9f71eefa913517dc775e5d72c49cbf718cf6c43ccc33d55b22401ef2e2f" } + "grok-4.6": { low: "cddeac5f845fa44890679934ad840ca728f5ddcdb0ffdb34a60ed232ef0eda3f", medium: "5a986b87f3888e8b2ec1fe5f7e5361a96494b67bd66a8b9f39836c26f2d602da", high: "e023b566aa28074c544c2f44dad634eca5511a329a2030c2584574e7f419a75a" }, + "grok-4.5": { low: "e22e543e72ea409cfbb3c229ed3bd605252374d28922f6ab05c29f6d590108de", medium: "706a74cb8401b045604078f526b7afcaf292dbe85c8c26f20b533e828f8a3acc", high: "3357ca76540480f4f3f7dc7e4e05ac92df770104a7ecf6d0aa3cb29eb9b6d0b6" }, + "grok-build": { low: "905efbe13bda08f3aa3907cb0215a56c07553965f8d81242687f7c993652b294", medium: "56aab068913b69aac68862ab729462b943a31b9efb7ddadd1d1deebc0ff72714", high: "52d66731dcad01ca001243424e324b8c4921a17ab31fe6b7b869dc8473d2c3b6" } }, // Worker `GROK_HOME` layout the broker attests before every turn. The home and // its `sessions/` directory are root-owned, worker-group writable and sticky so diff --git a/src/runtime/grokBrokerWorkerConfig.test.ts b/src/runtime/grokBrokerWorkerConfig.test.ts index af63d80..f5ea166 100644 --- a/src/runtime/grokBrokerWorkerConfig.test.ts +++ b/src/runtime/grokBrokerWorkerConfig.test.ts @@ -13,10 +13,11 @@ const section = (config: string, header: string): string => { return config.slice(start, end === -1 ? undefined : end); }; -test("worker config uses only named in-memory auth, the fixed loopback proxy, and the capability-scoped MCP facade", () => { +test("worker config uses only the launcher-set turn capability, the fixed loopback proxy, and the capability-scoped MCP facade", () => { const config = renderGrokBrokerWorkerConfig(); - assert.match(section(config, "[auth_provider.daimon]"), /command = "\/opt\/daimon\/bin\/daimon-engine-broker"\nargs = \["--auth-provider"\]/u); - assert.match(section(config, "[model.daimon-broker-grok]"), /base_url = "http:\/\/127\.0\.0\.1:43123\/v1"\nauth_provider = "daimon"/u); + assert.match(section(config, "[model.daimon-broker-grok]"), /base_url = "http:\/\/127\.0\.0\.1:43123\/v1"\nenv_key = "DAIMON_PROVIDER_CAPABILITY"\n/u); + // Grok 1.0.34 ignores [auth_provider.*] for custom models; a helper table would silently send no bearer. + assert.doesNotMatch(config, /auth_provider/u); assert.equal(section(config, "[mcp_servers.daimon]"), '[mcp_servers.daimon]\nurl = "http://127.0.0.1:43124/mcp"\nbearer_token_env_var = "DAIMON_MCP_CAPABILITY"\n'); assert.doesNotMatch(config, /access_token|refresh_token|auth\.json/u); }); @@ -54,10 +55,11 @@ test("the manifest pins the sha256 of every renderable worker config", () => { } }); -test("the probe-only renderer refuses non-loopback endpoints and injected helper paths", () => { +test("the probe-only renderer refuses non-loopback or injected endpoints", () => { const policy = { model: "grok-4.6", reasoningEffort: "low" } as const; - assert.throws(() => renderGrokBrokerWorkerConfigWith(policy, { helperPath: "/x\"\n[evil]", proxyPort: 1, mcpUrl: "http://127.0.0.1:1/mcp" }), /invalid/u); - assert.throws(() => renderGrokBrokerWorkerConfigWith(policy, { helperPath: "/x", proxyPort: 1, mcpUrl: "http://example.com/mcp" }), /invalid/u); + assert.throws(() => renderGrokBrokerWorkerConfigWith(policy, { proxyPort: 0, mcpUrl: "http://127.0.0.1:1/mcp" }), /invalid/u); + assert.throws(() => renderGrokBrokerWorkerConfigWith(policy, { proxyPort: 1, mcpUrl: "http://example.com/mcp" }), /invalid/u); + assert.throws(() => renderGrokBrokerWorkerConfigWith(policy, { proxyPort: 1, mcpUrl: "http://127.0.0.1:1/mcp\"\n[evil]" }), /invalid/u); const args = renderGrokBrokerWorkerArgs("/run/worker/prompt", "/workspace"); assert.equal(args.includes("--prompt-file"), true); assert.equal(args.includes("--single"), false); }); diff --git a/src/runtime/grokBrokerWorkerConfig.ts b/src/runtime/grokBrokerWorkerConfig.ts index da08c87..e204b80 100644 --- a/src/runtime/grokBrokerWorkerConfig.ts +++ b/src/runtime/grokBrokerWorkerConfig.ts @@ -39,9 +39,21 @@ const renderSessionTitleSink = (): readonly string[] => [ "max_retries = 0", "hidden = true", "" ]; -type WorkerEndpoints =Readonly<{ helperPath: string; proxyPort: number; mcpUrl: string }>; +/** + * The turn-scoped proxy capability reaches the worker's only model through + * `env_key`, set by the native launcher. Grok 1.0.34 accepts + * `[auth_provider.]` tables but never runs the helper for a custom model + * (verified against a loopback stub: no helper invocation and no Authorization + * header, with and without `args`, `api_backend`, `model_providers`, or a + * passwd-home config), while `env_key` attaches the bearer on every request. + * The capability is exactly as exposed as `DAIMON_MCP_CAPABILITY`: visible to + * the worker's own tool children, which run network-restricted, and revoked + * when the turn ends. + */ +export const GROK_BROKER_PROVIDER_CAPABILITY_ENV = "DAIMON_PROVIDER_CAPABILITY" as const; + +type WorkerEndpoints = Readonly<{ proxyPort: number; mcpUrl: string }>; const PRODUCTION_ENDPOINTS: WorkerEndpoints = Object.freeze({ - helperPath: GROK_ENGINE_BROKER.nativeExecutablePath, proxyPort: GROK_ENGINE_BROKER.providerProxy.port, mcpUrl: `http://${GROK_ENGINE_BROKER.mcpFacade.host}:${GROK_ENGINE_BROKER.mcpFacade.port}${GROK_ENGINE_BROKER.mcpFacade.path}` }); @@ -81,8 +93,8 @@ export function renderGrokBrokerWorkerConfig(policy: Partial 65_535 || !/^http:\/\/127\.0\.0\.1:\d{1,5}\/mcp$/u.test(mcpUrl)) { + const { proxyPort, mcpUrl } = endpoints; + if (!Number.isInteger(proxyPort) || proxyPort < 1 || proxyPort > 65_535 || !/^http:\/\/127\.0\.0\.1:\d{1,5}\/mcp$/u.test(mcpUrl)) { throw new TypeError("invalid Grok broker worker configuration"); } const label = `${declared.reasoningEffort[0]!.toUpperCase()}${declared.reasoningEffort.slice(1)}`; @@ -90,8 +102,7 @@ export function renderGrokBrokerWorkerConfigWith(policy: GrokBrokerModelPolicy, renderGrokLeanBaseConfig(), "[models]", `default = "${GROK_BROKER_WORKER_MODEL_ID}"`, `default_reasoning_effort = "${declared.reasoningEffort}"`, `session_summary = "${GROK_SESSION_TITLE_SINK_MODEL_ID}"`, "", ...renderSessionTitleSink(), - "[auth_provider.daimon]", `command = ${JSON.stringify(helperPath)}`, 'args = ["--auth-provider"]', "timeout_secs = 5", "token_ttl_secs = 600", "", - `[model.${GROK_BROKER_WORKER_MODEL_ID}]`, `model = "${declared.model}"`, `base_url = "http://127.0.0.1:${proxyPort}/v1"`, 'auth_provider = "daimon"', + `[model.${GROK_BROKER_WORKER_MODEL_ID}]`, `model = "${declared.model}"`, `base_url = "http://127.0.0.1:${proxyPort}/v1"`, `env_key = "${GROK_BROKER_PROVIDER_CAPABILITY_ENV}"`, 'api_backend = "chat_completions"', "context_window = 131072", "supports_backend_search = false", "", `[[model.${GROK_BROKER_WORKER_MODEL_ID}.reasoning_efforts]]`, `value = "${declared.reasoningEffort}"`, `label = "${label}"`, "default = true", "", "[mcp_servers.daimon]", `url = "${mcpUrl}"`, 'bearer_token_env_var = "DAIMON_MCP_CAPABILITY"', "" diff --git a/src/runtime/native/engineBrokerLauncherCore.inc b/src/runtime/native/engineBrokerLauncherCore.inc index c093307..db3abdd 100644 --- a/src/runtime/native/engineBrokerLauncherCore.inc +++ b/src/runtime/native/engineBrokerLauncherCore.inc @@ -257,14 +257,19 @@ static __attribute__((noreturn)) void launch_fail(int status_fd, static pid_t launch(const struct dbl_registration *r, int executable, int prompt, int capability, int output, uint32_t *failure, uint64_t *observed_start_ticks) { - unsigned char mcp[DBL_MAX_TOKEN + 1] = {0}; + unsigned char provider[DBL_MAX_TOKEN + 1] = {0}, mcp[DBL_MAX_TOKEN + 1] = {0}; int status_pipe[2]; - if (capability_bundle(capability, NULL, mcp) || pipe2(status_pipe, O_CLOEXEC)) + if (capability_bundle(capability, provider, mcp) || + pipe2(status_pipe, O_CLOEXEC)) { + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); return -1; + } pid_t p = fork(); if (p < 0) { close(status_pipe[0]); close(status_pipe[1]); + erase(provider, sizeof(provider)); erase(mcp, sizeof(mcp)); return -1; } @@ -275,6 +280,7 @@ static pid_t launch(const struct dbl_registration *r, int executable, sizeof(*observed_start_ticks)) || !*observed_start_ticks) { close(status_pipe[0]); + erase(provider, sizeof(provider)); erase(mcp, sizeof(mcp)); kill(p, SIGKILL); waitpid(p, NULL, 0); @@ -282,6 +288,7 @@ static pid_t launch(const struct dbl_registration *r, int executable, } ssize_t got = read(status_pipe[0], &code, sizeof(code)); close(status_pipe[0]); + erase(provider, sizeof(provider)); erase(mcp, sizeof(mcp)); if (got == 0) return p; @@ -349,20 +356,29 @@ static pid_t launch(const struct dbl_registration *r, int executable, "--model", "daimon-broker-grok", NULL}; - char home[300], grok[300], mcp_env[DBL_MAX_TOKEN + 24]; + char home[300], grok[300], mcp_env[DBL_MAX_TOKEN + 24], + provider_env[DBL_MAX_TOKEN + 32]; snprintf(home, sizeof(home), "HOME=%s", r->home); snprintf(grok, sizeof(grok), "GROK_HOME=%s/.grok", r->home); snprintf(mcp_env, sizeof(mcp_env), "DAIMON_MCP_CAPABILITY=%s", mcp); + /* Grok 1.0.34 never runs `[auth_provider.*]` helpers for a custom model, so + the worker's `[model.daimon-broker-grok] env_key` reads the turn-scoped + proxy capability from here. It is as exposed as the MCP capability. */ + snprintf(provider_env, sizeof(provider_env), "DAIMON_PROVIDER_CAPABILITY=%s", + provider); + erase(provider, sizeof(provider)); erase(mcp, sizeof(mcp)); char *const envp[] = {home, grok, mcp_env, + provider_env, "DAIMON_CAPABILITY_FD=4", "PATH=/usr/local/bin:/usr/bin:/bin", "LANG=C.UTF-8", "TZ=UTC", NULL}; syscall(SYS_execveat, 5, "", argv, envp, AT_EMPTY_PATH); + erase(provider_env, sizeof(provider_env)); erase(mcp_env, sizeof(mcp_env)); launch_fail(status_fd, 6); } diff --git a/src/runtime/native/fixtureWorker.c b/src/runtime/native/fixtureWorker.c index 63af5b8..f53b77a 100644 --- a/src/runtime/native/fixtureWorker.c +++ b/src/runtime/native/fixtureWorker.c @@ -5,4 +5,4 @@ #include #include #include "engineBrokerLauncher.h" -int main(int argc,char**argv){unsigned char bundle[8200]={0};ssize_t size=pread(4,bundle,sizeof(bundle),0);if(size<6)return 21;size_t offset=0;unsigned provider_length=bundle[0]|(bundle[1]<<8);offset=2;if(offset+provider_length+2>(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output");if(!strcmp(provider,"sleep"))sleep(30);if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;i(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output");if(!strcmp(provider,"sleep"))sleep(30);if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;i readFileSync(new URL(`./${name}`, import.meta.url), "utf8"); const unquote = (literal: string): string => { @@ -53,3 +53,10 @@ test("the compiled system prompt is byte-identical to the contract prompt pinned assert.ok(DAIMON_GROK_SYSTEM_PROMPT.length >= 320 && DAIMON_GROK_SYSTEM_PROMPT.length <= 700, "roughly 80-150 tokens"); for (const tool of ["daimon__moltnet_read", "daimon__moltnet_send", "use_tool", "search_tool", "read_file"]) assert.ok(DAIMON_GROK_SYSTEM_PROMPT.includes(tool), tool); }); + +test("the launcher exports the turn provider capability under the env_key the worker config reads", () => { + const source = read("engineBrokerLauncherCore.inc"); + assert.match(source, new RegExp(`"${GROK_BROKER_PROVIDER_CAPABILITY_ENV}=%s"`, "u")); + assert.match(source, /char \*const envp\[\] = \{home,\s*grok,\s*mcp_env,\s*provider_env,/u); + assert.match(renderGrokBrokerWorkerConfig(), new RegExp(`\\nenv_key = "${GROK_BROKER_PROVIDER_CAPABILITY_ENV}"\\n`, "u")); +}); From 7f5b62b614fadf236065ca3a1c11d1e656938313 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 05:04:02 +0200 Subject: [PATCH 012/124] build: rebuild native engine broker artifacts with the provider capability env --- src/contracts/runtimeContractManifest.ts | 6 +++--- .../artifacts/daimon-engine-broker-arm64 | Bin 67456 -> 67456 bytes ...daimon-engine-broker-arm64.provenance.json | 2 +- .../native/artifacts/daimon-engine-broker-x64 | Bin 55392 -> 55392 bytes .../daimon-engine-broker-x64.provenance.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index f122193..33ec1cd 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -69,9 +69,9 @@ export const GROK_ENGINE_BROKER = { }, bounds: { promptBytes: 65_536, capabilityBytes: 4_096, capabilityBundleBytes: 8_196, outputBytes: 65_536 }, artifacts: { - sourceSha256: "5eadf15faeccd5fec14f03e701c1b7001d7f4058941987070bf5e512e9984e5e", - x64Sha256: "51dc67387cd0c9dbbbc36b577b5c98ea296ecaf3f2f015be52eb4df98affcbbc", - arm64Sha256: "5821e547aa6e7c5682eaae1a3f6106138ccd5b5666909c6e135862a61b35cc47" + sourceSha256: "7b0a1da4add25abb2176bb5c882f247e03bf099f23fb295d72509253c7696beb", + x64Sha256: "e7028d17c0aff843c37dd9393f9c30c6218329bae64f8ddf907296edaf2db349", + arm64Sha256: "7a37683ebefc6d935260619d635bad919b7bdbfb552440e044449d909a8f8ce5" } } as const; export const AGY_SUBSCRIPTION_REALM = { diff --git a/src/runtime/native/artifacts/daimon-engine-broker-arm64 b/src/runtime/native/artifacts/daimon-engine-broker-arm64 index 2afa07296b6cb9e546701b836575a7752a43c5ad..b836e7380a5cae750f1c199877dfe3f7550f8d7c 100755 GIT binary patch delta 5460 zcma)AdsI|MnyPAQ3rwMcvPiPC#gK0EZ z-7w>FewVX_S2R8`SfK08#6*gmgmkg7Q{$ESzt3tQ0r&@n%intZ_QjtH>m{~ z_X=HmJwn^=j3nON+RaZ7YCVGb2j>GENMhzjYilo`5)?SO%8zFEb+k#DPReaJtN)sH zO=u-ztom|xGo7av&g$gcmbUA&w#TL}P}pV$dE)AMMhtcHNkIo8dq~RyRa#I_bK3?M zY^3H^{XuzYYOC0GcV2{}K6UNFMB1VLbYae24JJ};Pa^%RK(DvCpXe}=PY3zrpnbT! z|BV<@{XiU1x-r&xQ}8`zbqpywKb0s4xU?&W1XA4%yvN|D23_i37A~QO)YQUx)2tD_ zIhK$l#_Q%&gFfi_gZksb`j~scnW=tSn3A#!`a}~c<_Nj!X~|XK@vDU7sd`}ny`UBp zCC=SA0tucJ_rR=f_}M|^s=I-;aHP1KpB99l!VIfYA1+$gcC@IKo5!rTzblT2faO>c zI&i`?d|ua67=GO=DqlRnDF=YtkBVyZqGHLAZx=`83G(klKH=G2VZCPlnUN<+V1y^i zwGCY0T9#G0&c_4?*3rNK&%18a@PR=%AW|y?B0LG+NybFNI{ z50?+>&2fHz_`4cc6B{!bjZcz-C18x&Hw|7+bZxtZJSrl^+6k&q+ETRze65fa>o@e~ z8wj(Vkh=I{QhhL)RBQEOXeszkDpxT-yp6tN1}S!c+67YAkxr`FLIiCCFfpAJQ&9e& zL{-??80Lrs*ZcC6>owU;)GUWJgzFj>Vw5dzf-qaumpZtJfBNb0U`_Sz#hFvsS>X^Z&b0}cJRey6ZL9*A zHjfm4Jft@Z#$M$r4%Bcawsa5pE#NzWP2_h7a@AV!uw571YJ&^ujl{j7gZl`ooyS1X&iCfem216McIWznmJ7fbx=Zj83LQJb0)wJ4$EIBTO4#O z#^HX9Q;0lu1+$uZVF5FM@uLPL4(aXE zRMOuWfmaJ333YHJbOv6HLoKzS-AVJ6x$b<$jD9RT?dF5|$|WsdF#(7{JLxD;UiTF! z0{V?D1qx2()daK}vQT->y-X(7H5ar*1XO4$0 z6Up?C$>tmnmD5PJ@8{ZFLZm%&u+Us5Rm=@k$w7ZM+9e04ob+)DJ7#@ePU+pr8(xYb z{wj~QP;q3*;eiJ&f$KGKurfYyDLo;e*OG`Exx`LO^aOnW6c8k=R^wt^q4nk2BKkz-gm>p*v*JgN& z9qZ>oqq2q$>Hnc5n<>F28nB56*oL6UjegBopW(i-{ucB%p+C_B31=IPi+d-IHU-~b zAcv!u_-_}-NQcbI=>3ff&8Gy7d;O50BoXhcnMA087A$^U!v=7&FCqFymVmS?6^qQ69{-;ZOvpDuVbm?9a$wYDfR-8UOTuw$t zn#suVVRHERFg-l=xc{lr<7FhW9}CH-Yuax0g~b_Cu%`(V!n?3KM&vJu-z z?_`D9dh?J;QjTG>Pr+9=n$rRo_%wy*<*zBRE2ZMRqt6zfMpvqFqA+87ra(v z=^H!rb$sOb@ErmD5{^SUB-w`FA`g z$s|0xXH#Q*7-KmBV;845t#UZP_-vdj7~|I&KOSTF`Gs3PmEB|C;Zpr@2)DDtU?d~= z8pEBqe_G-o2#58Yy0ZkiJEb?z6YBArN_v2$TThl!vZrc>^;9XPEjwmfPnY)Al2C^A zqtc;)xWfdsiO8HXG8BJ!Gkp6Re4{-`Ld1+9KgjO6wBQN^29MNY?LXWhnlLG8YPmzK zqp@(-3AZ|QM}`DBKc9Yv^65<+WS`_olbw+Mhw(C({uI}5VUDR^9x|MA3n#o*3~7%q z55al0l#LND|N7C|mtm|ghF)Rxgg0d0m{~U$anVkV_CG>g9H_yFb(o!-zd}xOAV>}r z=qPvrAd zghw3-NmK1`Xg3_%bHa_|#9qk+w;1OV;dO^)G&?nX``qkh3#J7>BxElPEMABw%on&X z_Y!>4aY%>J)JJW^0zK?8Y8-{M;AVv56IjuSkD$nD z$l?1iSH|ZIJG0T3M&vt$gByJ3c+^1BP^IB<%Tj8y1PkwClPJD{@VaAnSRabOMV?bG zd`596CcCgG=f$=A0=YpDY|qVE5kVbGBdOwXgCKCc*IMeYJ}f?az( zC_FF@#WB$}=b}VH7Yb=;a0vu|Pa@yqP(&4-h9dje<4fJ{EuGAETF;GU=p>8!o;O37 zi_NH-cTQ;*v~u^~3z=oWmn}wKSO+S^eEkYLux|(paU66*H=x6O{i3 zzTV6yUNIy66H&)J6BK?qLE&BeZMhrV^jeAqCT>K1vm%?GP(Q86$$1w^$(|_eI|EnU znoOA}{-1O#W!O%h<&9j`@UspzYgxYI?}Z*372@&oH-xr#sP)T!!}SfQ-17TupA4`& znq9j|cudIQ%WB>7CG;!x@bYplFrfZzc^Z9RwXFCdH#DHGUy)(^J@W6sHoWnTkFjaN znzpA_BvM-tYN9IG!}ne_urjAjTj`)`%WTJ6cM5jyKd-2!tuA^|E!g@-n{(TGA-S?@ z&H7ap%c?ee-IZ?7s@1Eif4p>7z54LaH>l^fu2fI|d{|=t=D?94j-ale#&alXt3@W~(>j`*lb%Emekk#<- zL!cVyE1=oOb$vRI^HqUPKHi)ip!a}gcR&#IgHBy<2i5Q`j_)EOoYeJ=2u8;ru@I>I z88(CsTR+!z5${bOXf{F3nZ zIW!p(XDO=(-v|qNX*XwfCmO3+cv|SC>WpeY6sAubDb9;|~@dNJM5jA&5 zE?4@tdf$#;r3puM-GN!q;&@fq&u62X4NoQ{K7vKVfYAD$smjwE}BV3+m_5 zd(}<#DV+J3`dIyP?we!kvHJB=C8R3h*%o-Z6&@j;$z8js`_*S{`+sfoJpZ)qg{N#@ zb(d}LGd53iqwU3=HqT3b+r9?dvX^DsE4q~%j{EwXug-sY?(;7``}1GEolM;EH52f4s>2!UGeH!%lDZR2bPDYd3Ik-3H4IXmkudq`W|xOgwi_v^Ne_8Z7MP zmuRAZu%6ZRJyiYpk*wsq|8DileLKxBHJX2_@{g97jzY5XHP!Rzjs-=q&IbXz*qt?ZAHK@1J7nSek|rIf&DqycJ{G2zq)d0Ev;?aH+0HO*EHWR z&^$}6AxDa8oA(KHk#%fXIYxIkcSFdg)s0F;X+pCz2JqD#qi~e&d1p%uwR8UmsF_~h delta 4884 zcmaJ_4OCRuwchvM0Y<O;D;txy!@B7Zc7}BoQ zyVjX~_ILK#XP>?I+2`I-_nOqbCU<9S!cGc52|KlmXY7G3BiuhzlQ4pt+Wf-dDMB_| zn|@{Pal37irb~&wd4}PkXv((}9p}8gdXY8@ST(RP58U3mh}BR1mH1dRyFGQj@GzT~ z@gvjmmOn0P+nbRjncZDdickM>G5cTpMiC@2^PvQHkCf~)iK?p+&3eA0P06$i;*A*g z^0b>KHw6~67qdLVG?q8LU3}Nvc5C|9nAGVSH)cR0NB17t-z6paY(#Eqoz9fm^+I;r z_1O;zX1DPZ^`6w$inbr#9TEkVt(u=G= z9V+R2CyJ^z#*)?rUYDYMzsEwhD9XP$k+ehNl&c*ks`3+^=)#yTrTV%dHKOSyLOwoYPtd5D^^eh%qi3;@BWxuu4R;NpdA3T?g1-Xy7 z4lzLumJAu5?KDuROG%Eon4>qzepyro7W`Kjob1TuATmb=pM$;;P4lXWs<1mp7Wm=E zWiR*2^#`2Zn`=wG{WWD?c&Irbt@|V$<6pyT4`a8NV0@_7VQe(WbgWuT0av?MTti4vqJQ&jPoYj%@2slR}1K3K=C&-Scfz ze5iL%d??f$AL`i~A2L{cXivZ4vCfU|YY?fbYubZ;ic(!AM2`rZ-+N!Eq?r1=0#$W4 z-^bQD3zXw9&}ouG@X#BOyXrKIDy5qG1W^5za(|Us4q44q*EJrSMtJOIHF#hUXL&r% za<&lW_kq{r!8=6F4zE7P$2mX86y`NP9>>pqlH>W?)yz(1rRSVPH9~2AZcBQ*NEEa^;ZByMJ73*aw+T*HL zoZ;>)enS0CG0(4GsH~W(XK24vTZ+41iCU<@%g@3wdJXx>jBwZRT9E3SJCKSzi9wzW zR{Dkbq=_XqxlV|Ir=(@Z+CfdjA?f99b)H9!0EiwHnl=DZZHS`m+B?P8TipTOZ1-n~9+4B_sWJ5fr&X z)I^&a4oEPg7k0lR~xS-og4M-XcVj_qm4)M*Df7 ze6v5ryolT!j>W7C&$q+KwPlDZqNYn}K06|%4jcJhh$MK)_sf2ErYu>p$CKLESK@NsFU~t{NxICb+ zO83Zq<9do;;0rKh;50;AP=klRcF&(ySz0J&aU*A&P#-t^PY?j2OY+3q`!dUP8KcN$HMmRJWed>UVh$`7gx@ z{-H$3F$x{UMj{#V`fX$gJoYK{gfRu_b%28~APA7}r1Rex5>cdK&fEV+4q8U?`<^s7zG|hYOx@aRaL}sbCkV^n}d9 zH>sMV38-8=>3$BJQ^eb)ppTo>5wHy)p;LU_iW>YH7%nh`ILX2FD8I-?HrMha|4?HY`IB0egO(VWXTz+fpha1wJ7 zZ(r963UN;gQxWT$kzCk=Tu{@I10ABKLa(93c^$V9|G-o#;0mzA;P0V825*9Z6kc3) z*+B`)zra{E6m+AkbV;HEneDg;ERY5)oChOVJ1`rt3M@EQ57r3lImhRuQUTX%Qt%U5 zC}2`uA7U_^v9SJd3SL%l*4tIpE3GB2;4(@&( zbu%K%LvrhmY;ssuc?=*gJONA~N_WUf{LY@_SP#Q^i~@-xcX$JM23`*HlO5dpy?lHk zAD`@Sf^X*IVLsV$82lda)e{B^xx0DJz);_I^cUckI$J9DwFaM8_8$_d_7N0Y)k3Ry zwcpTK_1JpExh0w=^z(v>Wa0gID7X$DWv_F2SB@43THBtZUCmALhB12~f2Hf>IN4@t zwe|%VvwFXj%-T3$X0)01nVNb1Of3feXtXZ^%1rHTa%fRtOlYsk4sCb9p~a*BjIcn{ z^D?|uxFz&FIrJl3ArkQMXf>)sJ7&+-Z~{8lq`gL(}YmVO~VtB<;RZ8mg+^?e*+h!#$Zplnx>54@pxXf{*KrVU5^eL_j`=dvuX#hgTKyTkT0}twBLaK7T(8+ z;Qh^`{q^W?8tpsL--te|UQ%p%7u&2CshQts|1dM(k__`Q!if1W>la=Y&9ZS zI%a`YW90vVcan&FznD_nK4es})}>j(TkNf+Gjj2jlgw4{RNzI7@FJV3lGgZH$2}Zt zl13_YgWkasE9Tn1FVs6CB%y}b;EjeW9jvC}S+V*C`=;XI36(c^S>az(_{M>$y=;Bu zBHD*A!<4F2>C}U`y}5_+IB2U6eeUtO+*EM z*%L;qwGQEHHm~*wJG`}`?R@PIO>D*1RqV>vA%%b4 zA}%B!HH=+204q-zh6#s*P8miz#;QT*g1SM=LEi^m18RHIFrEOFLEWG$L67ipP#v_| z4?Rf9YoN10H^Z45K*h=DWLMFhOrs698?8e35se(bipw2NlSA>hH;+b zpx2_=m)kNIfA%}W$cF?uV=@vzW})Sr#J)L(hhs8YIWYdt%^w?D724*LYa)eur!crc zq&L~>?Q_HpZ?dPhKQ?978N=Y>&}2JcIw)nKn*}H4pJ7vWWT!rj;SGEkPdQpOhRy?f zl|8s4cgore7e76J?vs!I^0&_|5s4#wa)2G&u~am*v&%bHE8H;d{&(Q^XHc=J)N-w^ z{J=rW!2_03&kL3pJeE@SlBMNEOWAY#Ezj?>{37-{thp1ddT>c*qH8gvU6f%7Aj21##l9N4s$mEbiV2vLe!fUAxfm9|s-% z5QegaJF_OQ`JN^B?{6~i+il*+>US=T-fkia{11C$=eF5jpx!dUBMMu!4^)(;s}89PhLl9(RbprO{&*aou31VmUBLEo{8MlFqr5}TMI5+%>n5H;TS`(}nf>1pO1 z`0o9Ef9`j```z!}neD2J>Z*&XGxG_n$~o;>Rjv%@r^fdU)4P4)Tt1B-=Y3o64yCoc zj@O1?F~ueF)0AD7QKPf!bg=HCPR%;u3Y0V8{=(o!hHzh0cwyA&{RX#V2sc@A2%{EO z?bl6p4dM0}TIvw|*Y&!5?+|=feR#W2W;Mn8!%4PP(fk_4BOXeVD>+V*FRSWZ#sU$+ ze;UeCXvRhZK4ZWqb+|-%YQk=lkfI0^EBH`&$HY%f{QoMxN!@&vviia2`4r{agRQ1N zrG)DzD{=f@W!|(c{IT%yX_-9#tYVt6il3wu&-kt>$s3kuWJP(L^4HEK5C23e>p5@h)S${(Gqqgu4fWnV-Pv(i2`N+}+6er`r;N$RFo!?_ z!9kap?>HgJm%x)k`Meqrey-du%>qLRT*W{h$=4Snl)sOrOL)6h8$37U5?P*nRmx%l z#|gh`0n;6-6yGE&Uv_GC%EKod9T%6C;+8>1v!3KGDd z#}K8{isohZmW-n5S*1XvWFnlB}u>4#vGC6ctoOw z_OtpH0MZ+aovZGJ6`#rr{$c1jZaJ8`9tg292B&s+G3E}{iXG!Qt~ndpQYdNCI*^|q zcgvqRJP}p;5a~(|Q9q)_L)-A<1$ZEZ{9^Akj%#FygG*+T+$Le7R&^KDw7v@|_BcAU zhRIUktg!V_=(YJrlA4xc*5*m%_6@j#JP`cnM!P z=7a+ac*)n!lN1lB*|yRi-x1z-`@YhT8WW)5xTP+|;0RPI`DN1vxDrE-W3x3Cmd&+DfBIxy$nOV z8eC`a`9%4)d{2G%vaaIML9+&KHozwVs@r-PGmK%7h-S;9_83ELghIVi| zK#~+g+7ZVMaDj(VY(5PQ8|V+n0K5R<)ZZE44p`IH%&Cp1u18!NIgUq3=#YfPN8}^E zzNpQ1VJzni&HvihXIejEUgno277Qnk6xDW^362z;O<{A)YdbK16y8wG zl+{!^_lx;(ut*Zt90%qA)|?8!Eq_@g6dxuWH83U_TFTkZxkk?eRdfe0KR8*jq~ ziKOewG5Wa22+Gi*+r|2nI}DAGp^*)?BZ48?C?H)1_c?HZwG-BBU{C4TVZyL>oQDQ} zOb3q;Xa%S?zmK9NcWcpiM5qdTks}D=i*P`aXN&4iNJ#SEnv<~`P`AJefrV7{5d+zW z=bKcXA9O54ZKUg4I zi*kiEZHQdWS`2koxG!3y_;&&+*8Hj#y_lwel9Ek~Yb4)FtuBP&m>b7+wwBY52+8D2 zIIBHXMqr+J3ALnwgP-uZ?@8o(4k7`)qa<5MVOj_g0IdY9Q!BLybPjU(2pP1xV8IvR z+sN_brC8`{QAHuHa?4%0zE(Ozb2HDCe%_3NflcRO>bcLgYLM!Nb&sDLSTHI(|JD~u&Cj1YfoggrRbjSy=G98MJV z3s`cTb=;!%2F%jwJI!$9(}C=g`v#r)jtKnG%f41w)C5dL!Zy|ptZ4GeY)mCrKBk_0 zZ?K2inu8unf^YS3z`?x0XM(?k;cDjlI$8+ip@lHeSWcA1?kx1RE=kXW4BrT4_t z?xRql;q-caqF;)tx_S`3BIVYc)EsFdmB?DL7us4C9Jsh4Mt!4Gn@xIvp|cwu>F^z!!BaXt8gshF zECPrI=f;6$`Z)H;LKUG86$XBGX{Oz&BrSpj|}uCzxc1-#1^E zZ@6(O)Ft=pnsT)^7q@)9k>gi$*%Icx5+*D0=f;-HhYiA(f&v1DAq{LLXfiPM4tbYF zDnj56*6WsETnCq~hT%K?5^9sd z;XGi;mvO=5Ik^B=#gY1D%&&e8U*t~hMM*lW+T?av9HagNmn2*w{3nPVjM;-qNFJNv zaRYhmmhauHg|!60S&>!bi=HdkieEqoMn>WoIQ3VIifd{h;t?t~!g(;Uk0WW#0xEfHLypKa%o`9Bt*JD>IqK_L>-|nLQsqshEZf* z3-n=tr2b1DH_U$@=>p6PsZ7Do75s}p+_;;s!;MR|!9{HSF=W~NxohbwN`b3={A`scV-LCHd)h?=hkU*ZS>VEjITf!& z9=jXmZnm0&vuW9Kr`(FcwtjhAuS04RM#3?+L`-+dSCPpdVL)UuWd?lBLX|-JYR+ick%E2I9oFqy)yHACJBXgkg7`d2z0K$qRVj7uxd~au{Inj^8Rf&gF~k4qMa++pIECeXU-$ysSs3f3z9Z6oRY_em z;rrhyf{zP|#{d(C#T5x_W8#4b!xmT_>}ZjawkT!fHRKFLYy%?QPbj}%w8kW#2`4R{ z%bTQ*@FV%F<4oJi!^a-|?ZWBTXVIFKSn%?77FLWQoeyKzj~$K^m%hdLdd%%Hhards z=hV)P%1gU{XsVc{oZ20@p9f6K`{Ot0Q%dW#H|cHC9h%ZTQ(665m8r!VX0JIDw%1Jv zXVxv^&G`>4ek9j5?;+*s`k5orN}qiut$6LvR}`oH_o_8%|MSe+r<6a|PgG*x8l!w# zZwcGq{LrL)RiCDG)Zee9y!FFj@0BrDjlacS#=_We8^f#Ky2E?u>l?iT*Q{iWs&IBO zW67u&UeFHYWDPD85#TkYjHLsw19gE~*TWpifi_Y9ImT4bb)Xr@j9$={p!pc?9Z)am zNy0(n3fc>rP5s|7RuG4spZ_Xj>j2b&`ax|q zjO_sRg6;$D1&x3f?t)=ZZZ~EM8aaT7pfcjK19O2!jx$z4^sksHXzw*d03SqZa9ZpI zO}`F3&_d9=pzA=bVZof0p)cK5((l8ZY0Ds)@z-Zf( zjM2FL5ANe!0sdAh+Zslg9)DI@*f?M5X)yC8%0C(&;G2~1HBRPjN^av~K3#dGF^%^s zCmI(gRc&Hy9NBPLY<3VjaQ7)w4o)833(kDv;F0i`fptxNp0U@Irw@)8JutH!ZgRnwiC6toem4%0vPwhlAv6?X?rS_0Y4=^(@ESvVE(tij>AdGTM!>*j=IbrMU&(!z}xgP zV=K_6hp?T%I)Oc=AsXi4>Hvz^F;5METq`gyuqA`APGBd8$n^lTzk*~QB*$Tg`3GSx z@J0f&V)wmI-~CN1{)2P+OwRJCoXTkBUh~xN6@XpwXG9tcKu#u-A4qyc`E|>j*giCSPAR8aUW%>8e)C>Yo{Y?lr7hTdMfqc7re`E}K{h@~rjR*2 zGRnE; zp0t_@_k>eUUNW1OZ&6I=rzRBc(V2yNl$qz}W5<`BPaF0xzq}}if!Z5%(SF(y<}W;E F`hOgqlpFv6 delta 6067 zcmaJ_4OkTCwVoON7LqV)B7&(gt4j5D5^kNKDMv_sk5{dY^V4 z@qFh!Ki~PzcfRw@PNY66QXf@s;S=h7oc`4L)I0g~_~J1JTO7>eGx$jb?-=Z@bT@C{ zyMwSbZPTfGG{%BCS(-F?oEaDebyNnyT8xvS9ABx<|K7{|o?z$qX7l{xsxadjK3*-F@g3ocC#cNGj`HLwSM0-%rOTA-kINO8 zI&C!dZi%uXD@!FyD4Gxiw^d`Tg&1W(6POX`6zqZ=D3NTyHeY1Sp|m4@|Lp`Uj@D+sMz8jw#%n1}W42auH?7Yi`@$x$45P4qxZT5ToB(>zB^(cl?gY{zukML%1R8aNP&uqn zJcqtuOgZO2^FoYv>O5n}V2+=GIR-{<``Tv8_8T&%F8dQ0hxa-wu{hzOhBT2rtoR?f z^Q=1T3LKE?F|iC9PT{!bL_N%;T!eY~ZI{BDV*$97T4^vwN02SjL(wWb^+mV>f6!$v zR5=W=AMdjz`GSfES!gVY6W!cKz%0(L-(7+8*^VSs;}HU){0v&lyEe~pS7_DO>f zq>BE2aB|g7T~+p86U8lObfg+y)VM{Ddua0nx#}sa=-)^ZtYj^q76Ci1fEkKaR-z@1^%>URG5xN?x^xBYGgHEUn@l^?LTgi&#ly7mK9>bneKUq({o9-Mmi*1Te>ZC2r<~-$;Pq%JRB&E!^_lFt< zj(a4KmF85gx~5*#3eLe4KDKU(;?O@x~kx( z`3Xdg{YD45Ebpm`RpQ`H`|K=Z6$kXae-+b8+ zwCCZE)>8qoz1u!abA0fTu`QKFx=l>fPslm?PYY+U{G(X4-DW1gxQ6cJl$_K#^!e?> zqb&xNjGJfBrCdX+{5@tLt94v9CTxZkIbDwITR6>aI(rka<^^L`K9Btsed!b(Ll>bw z1UCg*F=E7EIbF4K2+QHLk8!b7m3gG)w4N{!*;|ILE&Aw1C4_TD^{cHl4fB+CEe&Ir zGD6ot102grn||}~xOuuthZg%Ui2he$hIyPCM|MV=vkxl+RZFEDrBQ09X>fyuL7mDl z&atPHAmoGY3@5|mgkBeckVZhY!YhpRp7@F+SGBq`VBP{_>?Y;^6@q>NNUO%^Uk3>D zmUwbF1F;#KcUAK=gXvufYf6vy2t3%sx>_re$Z6LAP=a2=3Dx?c>(zMC_j7ReVLQ|I z10nHd5~u$!!a`NHZ!p>fogM!N}>&s(NaWRL0&JM@OohwYNR9YPa)aDCAj^5S2T zecht(eMri$*-9_r9l<*?uJl3(Ck3%<-7v?JmlepUI=AknfxMtV+(v;+(%(Epe==q- z6bHg2DsF`iCr))r#o8n@H5}x6GHh^3$ry-MZH}>?y(UW`R2`70$0^s8Z?#2#HA~vS z=_b_iFHalz8^E=gzcDu79=5+r8)Xl->LpuXdZf#%^?n3mYP8T+A*ngWnhGcg*#o7J z6@r#na`+>7r-X}@_khL9tv7@)T>vq_u{S`Z#R|c6Ly&SqfTIRH2{@|n2CN(b%1*(J zOR}O7dX!FQ{9wH1jMv1<*Re4zv^mNNtq7JB$A1%sSa1@qYCkZ+QUYl?DD3Z>*lNPi z_Xx`{vBiWTb1)vz?l!?Rfcgd<#K6O8myziZ-McuTvbdxxPwD9!7npzDJ0fhlZFXNt z9BlJp>pj9dXZM$Vc3IpUld*$}Am&6NO0B9yO0~aRD2wYm0i6V7v_z#^8bOSOVo@6j zyIjgp3%#w0Q+-knD3m)2(OJo#ww?GS|I^TiE?!_`j=>RZ4-p>$e+E36y2&RnjqI9q zJAG2Z2BCM}uscjVx=_qtO}hfqu9NI44ZGtsKPh%W@u*?gPCNy0)-)_O4I^auq+v+G zQR^9m%pOLVM<}#%aHKRBO2Cd1w#~#c46K8&8WWqc9r>sH5Lj=5$%f(~!omiIYaLpL z3F$JmzS^cw(N0(dnCAYF+R8j!$3ti(fg~p))YGlagv6zM)sliTM_+(!E@UXfdm!-q z>rRN3&^uE%QX$cD(UEg76QxH3_JE#(ol+CZ%hAlixi~IbA}Q^{IGUa+j{+54%{|Pc<63FYzq=E93vj>ctHA_6 zhP8emFKS?bdFOO+z-I79K z#Ruw`{3XKllj`dHbm8M}b$k8 zaZFsHxFMzvh&X1U)y@tVsss6x$5t6Cwi_zS)9R1h>xH}zf<5ksdBM^h9QW9>al)gO zLH|>)Ez0^+Hof7LB%--?!;3kkV;NqqTkpb#FJ1Py?7d{^h^c@gHjdMFy`(--J5!Le z)vdMuyW8f`4h?9J{+WJ_xA*N8@6Ewa#OZbO)aiS+39If5w(fBx1h*xH`5pE5-C4Tv zxs0L>KU`gu@n6rZ&v^Q|4eNqa>fOA$y+KmnZMaVzY)Dl{8Z(0_d(R8%P{VjNci+r0 z*D4rm#NUQ;#y;>c_N$<>?`z&epKQL1h{9`e*<)N##F!PiuK`_yk_?wH)&x9U%2+D! zF3@?PnVT4^COuGkyU%={u?XlkPAN8?0Wy^Oi3 zB7BUMf!6$tvE!ibUoe(}0J>g50HD@afrGk1BcQfl;<7*)XM#Qnngdz{YW)>syFqQB z^@M}AgStUa6aN}x{|8zRYKcQfv%b#Qgg9L3pqZc%P$#JS*KiCP1}y?DsKLuWsBIr( zoy0dW_7U+NjM;FP+Hl;~fV%O@&(TW}YBvXT8-DMVgN8wOf_8zv1sVZ8Fdl*5ZS0^{$FK`>xMa}7am-^G@2Y6Y1qd7Tg+eXId=Q8I^w7Niw@mH)wrdG!|5;M2Nem`-RH*R5Kd2ZBZUbXEXdmDZ!L*EDE zCiVROX?&S_$AN{3Npe;iyelbl)ccL|VnJ7W(KfQK*+Wisz3tN_qhU~9Ll-3M2u zzkuP%9gLw^^oLBUfpr0+H#U0G@K*!u_zrbt%Y&XS42yRDv&jWuYk|?5B0c101Xu|$ z`Z7Zg$>@!;7+4=AabbLvCF&f17(>!eh0KXUUX3w5gv|q118kL!=$MCd1H1%q-3^eV zV<@BQm)?ZE0Bk3)oAs)J4FX$xlU@xl*RC6w6S78Nr-6Mm28^)!4$k2f99yDttE1H| zt?75uOQrjd$TTuePY(X9I51`P>i@Jp6u%V1krq^_`hM#abyhefb{CMw_tYoCuf%5I zJjmzOFT-l(HV0tN=n0}gW!3Lo>;T9~~f{X)8 zCTDoaUkWhtcQcj-Eb~()u*6_RQd)v%x~|QIY18p6Y%u8P2kPQ_gCG4QU1##6AE8s} z+a#XR&(5QtnMXe>N6gRBwEy%RGza+kt3RssxK*gB4Ng9O)*`%Kt_nTr2{pBbtfp3- f)3bmtP%C;e#{9p}>S`NMuG#cs(&8Zh;VR+p@+R)6 diff --git a/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json b/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json index 405f166..5b2a269 100644 --- a/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json +++ b/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json @@ -1 +1 @@ -{"version":"daimon.engine-broker-native-build.v1","architecture":"x64","target":"linux/amd64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:5eadf15faeccd5fec14f03e701c1b7001d7f4058941987070bf5e512e9984e5e","binary_sha256":"sha256:51dc67387cd0c9dbbbc36b577b5c98ea296ecaf3f2f015be52eb4df98affcbbc","install_path":"/opt/daimon/bin/daimon-engine-broker"} +{"version":"daimon.engine-broker-native-build.v1","architecture":"x64","target":"linux/amd64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:7b0a1da4add25abb2176bb5c882f247e03bf099f23fb295d72509253c7696beb","binary_sha256":"sha256:e7028d17c0aff843c37dd9393f9c30c6218329bae64f8ddf907296edaf2db349","install_path":"/opt/daimon/bin/daimon-engine-broker"} From 4bd43d39e135173eb224fb8f3b682999f33dd62c Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 05:05:27 +0200 Subject: [PATCH 013/124] docs: document the Grok 1.0.34 lean worker contract and worker home layout --- docs/engines.md | 7 +++++ src/runtime/AGENTS.md | 54 ++++++++++++++++++++++++++++++------ src/runtime/native/AGENTS.md | 25 +++++++++++++++++ 3 files changed, 77 insertions(+), 9 deletions(-) diff --git a/docs/engines.md b/docs/engines.md index 7512b3a..ba2b288 100644 --- a/docs/engines.md +++ b/docs/engines.md @@ -81,6 +81,13 @@ the broker owns refresh and stale-credential recovery. The runtime checks broker readiness before admitting Grok agents and verifies their sandbox policy before turns. The older credential-lease helper is not the production host path. +The broker worker is pinned to Grok CLI 1.0.34 and runs lean: a fixed Daimon +system prompt, six tools (`run_terminal_command`, `read_file`, `grep`, +`list_dir`, and the MCP meta-tools `search_tool`/`use_tool`), no bundled +skills, workflows, plan mode, subagents, memory or web search, and a declared +model and reasoning effort from a closed list (default `grok-4.6` at `low`). +The broker proxy refuses any request outside that shape before it spends. + AGY uses OS-native secure storage through one private D-Bus and Secret Service realm. Enroll it once with: diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 93723c1..4ddaa1b 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -13,11 +13,44 @@ coordinate wakes. Every source file stays below 400 lines. Keep tests beside the contract they cover. -`grokBrokerProxyRequest.ts` preserves the worker CLI's bounded -`x-grok-client-version` and supplies its `grok-shell` client identity when -rebuilding provider headers. Dropping the version makes the subscription -provider reject an otherwise valid login with HTTP 426; never replace it with -a fabricated version or pass arbitrary worker headers through. +`grokBrokerProxyRequest.ts` preserves the worker CLI's `x-grok-client-version` +and supplies its `grok-shell` client identity when rebuilding provider headers. +Dropping the version makes the subscription provider reject an otherwise valid +login with HTTP 426; never replace it with a fabricated version or pass +arbitrary worker headers through. The version must equal the pinned +`GROK_ENGINE_BROKER.grokCliVersion` (1.0.34) exactly. + +The proxy is also the spend gate for the lean Grok worker. Before a bearer is +attached it refuses any body whose tool names are not exactly +`GROK_WORKER_VISIBLE_TOOLS` (Grok 1.0.34 turns an unmappable `--tools` entry +into its full 19-tool set, and its `session_title` request carries one forced +tool), and any body whose `model`/`reasoning_effort` differ from the declared +`grokBrokerModelPolicy.ts` policy (closed lists; default `grok-4.6`/`low`). The +model override header follows that declaration. + +`grokBrokerWorkerConfig.ts` is the only source of worker `config.toml` bytes; +the manifest pins the sha256 of every model/effort combination and the broker +refuses a turn whose worker config does not hash to the declared one. Three +1.0.34 facts shape it, each verified against a loopback stub model: +`[auth_provider.*]` helpers never run for a custom model, so the turn's proxy +capability reaches the model through `env_key = "DAIMON_PROVIDER_CAPABILITY"` +set by the native launcher (as exposed as `DAIMON_MCP_CAPABILITY`); the +per-turn `session_title` request cannot be disabled by any key, so +`[models] session_summary` points it at a hidden model on closed loopback port +9; and effort is only sent when the model declares it, so the declared effort is +the model's single `reasoning_efforts` entry. HTTP MCP needs CA certificates in +the image even for a loopback `http://` URL ("Failed to build HTTP client"). + +Worker `GROK_HOME` layout the deployment must provision (attested before every +turn by `grokWorkerHomeAttestation.ts`, recorded in `GROK_ENGINE_BROKER.worker.home`): +`$GROK_HOME` and `$GROK_HOME/sessions` `root: 1771`; `config.toml`, +`sandbox.toml`, `trusted_folders.toml` (empty), `managed_config.toml` (empty) +and `requirements.toml` (empty) `root:root 0444`; and +`sessions/sandbox-events.jsonl` `: 0640`. Grok 1.0.34 writes its +sandbox events there (the root `sandbox-events.jsonl` stays empty) and runs +every profile inside bubblewrap, where a non-empty `deny` list is enforced; +`grokWorkerSandboxProfile.ts` renders those profile bytes. A worker-uid process +can neither write, rename, nor unlink any of the root-owned files. `agySubscriptionRealm.ts` owns the one host-level private D-Bus/Secret Service realm, durable keyring lease, bounded unlock stdin, and cleanup. @@ -28,10 +61,13 @@ runtime-writable home without clobbering a newer CLI-refreshed credential. `grokSubscriptionRealm.ts` owns the single durable rotating Grok credential, the lifetime lease, crash journal, stale fence, and serialized per-turn stage/promote cycle while each agent retains private non-auth home state. -`../pi/grokSandbox.ts` owns the production Grok process boundary: it replaces -the provider's fail-open built-in profile with an exact custom profile denying -the realm, bootstrap, and peer roots, and requires a kernel-enforcement event -before every Grok setup, turn, and cleanup process. +`../pi/grokSandbox.ts` owns the direct (non-broker) Grok process boundary: it +replaces the provider's fail-open built-in profile with an exact custom profile +denying the realm, bootstrap, and peer roots, and requires a kernel-enforcement +event (read from `$GROK_HOME/sessions/sandbox-events.jsonl`) before every Grok +turn. The direct path registers its per-wake MCP endpoint in the agent's +Daimon-owned `GROK_HOME` config (`../pi/grokHomeMcpRegistration.ts`), because +1.0.34 skips project-scoped MCP servers in untrusted workspaces. Strict Codex uses its native permission profile only for model-run local commands: the profile denies current `.codex/auth.json`, current `.daimon-inbound`, `/proc`, `/run`, shared protected stores, and peer roots diff --git a/src/runtime/native/AGENTS.md b/src/runtime/native/AGENTS.md index f8e468b..3e7f8ae 100644 --- a/src/runtime/native/AGENTS.md +++ b/src/runtime/native/AGENTS.md @@ -11,3 +11,28 @@ forbidden. Prompt and scoped capability bytes cross as inherited sealed file descriptors, not protocol strings. Workers must start in a private process group with no-new-privileges, no capabilities, no core dump, and a parent-death signal. Unsupported platforms fail closed. + +The worker argv is one compiled constant: the lean Grok 1.0.34 flags, the +`DBL_GROK_SYSTEM_PROMPT` operating contract, the `--tools` allowlist and the +`--max-turns` backstop live in `engineBrokerLauncher.h`, mirrored byte-for-byte +by `src/contracts/grokWorkerContract.ts` and checked by `launcherArgv.test.ts`. +Model and reasoning effort are per-deployment and belong to the worker +`config.toml`, never to the argv. + +The launcher sets exactly two turn-scoped capabilities in the worker +environment: `DAIMON_MCP_CAPABILITY` and `DAIMON_PROVIDER_CAPABILITY` (Grok +1.0.34 ignores `[auth_provider.*]` helpers for custom models, so the worker +config reads the proxy capability through `env_key`). `--auth-provider` mode +remains for callers of the older contract. + +Received descriptors carry `MSG_CMSG_CLOEXEC` and can already occupy fds 3-5, +so `launch()` lifts prompt, capability, output, executable and status fds above +16 before `dup2`-ing them into place; a `dup2` onto itself keeps close-on-exec +and the prompt used to vanish at exec. `fixtureWorker.c` reads +`/proc/self/fd/3` so the integration suite fails if that regresses. + +The executable is re-hashed on every spawn (~58 ms for the 136 MB Grok binary). +Holding one verified descriptor and `execveat`-ing it would not make a replaced +binary unrunnable: Grok 1.0.34 re-executes itself inside bubblewrap by path +(`/usr/local/bin/grok`), so the image path's root ownership, not the launcher +descriptor, is what protects the sandboxed process. From 0e09c759fd83a01ec6b1614422b26cea66203796 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 05:05:49 +0200 Subject: [PATCH 014/124] docs: point the worker argv mirror comment at launcherArgv.test.ts --- src/runtime/grokBrokerWorkerConfig.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/grokBrokerWorkerConfig.ts b/src/runtime/grokBrokerWorkerConfig.ts index e204b80..e4967ef 100644 --- a/src/runtime/grokBrokerWorkerConfig.ts +++ b/src/runtime/grokBrokerWorkerConfig.ts @@ -114,7 +114,7 @@ export const grokBrokerWorkerConfigSha256 = (policy: Partial { if (!path.posix.isAbsolute(promptFile) || !path.posix.isAbsolute(cwd)) throw new TypeError("invalid Grok broker worker path"); From 596a87da7e8227f3046fea6b3fc5a2143cce2349 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 05:07:02 +0200 Subject: [PATCH 015/124] test: give Grok direct-session tests a GROK_HOME and unpool proxy rejection requests --- src/pi/grokHeadlessResult.test.ts | 4 ++-- src/runtime/grokBrokerProxy.test.ts | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/pi/grokHeadlessResult.test.ts b/src/pi/grokHeadlessResult.test.ts index 01d309b..376162e 100644 --- a/src/pi/grokHeadlessResult.test.ts +++ b/src/pi/grokHeadlessResult.test.ts @@ -77,7 +77,7 @@ test("exit-zero cancelled Grok sessions reject without emitting a turn", async ( ].join("\n")); try { const { session } = await createCliSessionFactory({ - command: process.execPath, commandArgs: [grok], engine: "grok", timeoutMs: 10_000 + command: process.execPath, commandArgs: [grok], engine: "grok", engineHomePath: path.join(root, ".grok"), timeoutMs: 10_000 })({ cwd: root }); let turns = 0; session.subscribe((event) => { if (event.type === "turn_end") turns += 1; }); @@ -108,7 +108,7 @@ test("successful Grok sessions emit only decoded terminal text", async () => { ].join("\n")); try { const { session } = await createCliSessionFactory({ - command: process.execPath, commandArgs: [grok], engine: "grok", timeoutMs: 10_000 + command: process.execPath, commandArgs: [grok], engine: "grok", engineHomePath: path.join(root, ".grok"), timeoutMs: 10_000 })({ cwd: root }); let reply = ""; session.subscribe((event) => { diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index 0d09eb8..3293903 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -1,4 +1,5 @@ import assert from "node:assert/strict"; +import { request as httpRequest } from "node:http"; import test from "node:test"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; @@ -36,11 +37,18 @@ test("proxy refuses a fail-open tool set or an undeclared effort without calling const token = proxy.capabilities.issue("agent", "turn"); proxy.registerIsolationGuard("turn", async () => undefined); const full = [...lean, ...["search_replace", "todo_write", "write", "monitor"].map((name) => ({ type: "function", function: { name } }))]; for (const payload of [leanBody({ tools: full }), leanBody({ tools: [{ type: "function", function: { name: "session_title" } }] }), leanBody({ reasoning_effort: "high" }), leanBody({ reasoning_effort: undefined })]) { - const response = await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`, { method: "POST", headers: { authorization: `Bearer ${token}`, "x-grok-client-version": "1.0.34" }, body: payload }); - assert.equal(response.status, 503); await response.text(); + assert.equal(await post(proxy.port, token, payload), 503); } assert.equal(calls, 0); - const accepted = await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`, { method: "POST", headers: { authorization: `Bearer ${token}`, "x-grok-client-version": "1.0.34" }, body: leanBody() }); - assert.equal(accepted.status, 200); await accepted.text(); assert.equal(calls, 1); assert.ok(accessed >= 1); + assert.equal(await post(proxy.port, token, leanBody()), 200); assert.equal(calls, 1); assert.ok(accessed >= 1); } finally { await proxy.close(); } }); + +// One unpooled connection per request: the proxy listens on a fixed port that the +// previous test just closed, and a pooled keep-alive socket to it would be stale. +function post(port: number, token: string, body: string): Promise { + return new Promise((resolve, reject) => { + const req = httpRequest({ host: "127.0.0.1", port, path: "/v1/chat/completions", method: "POST", agent: false, headers: { authorization: `Bearer ${token}`, "x-grok-client-version": "1.0.34", "content-type": "application/json" } }, (response) => { response.resume(); response.on("end", () => resolve(response.statusCode ?? 0)); }); + req.on("error", reject); req.end(body); + }); +} From 18c26530d7170a65262d34b45bee98287525b71b Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 05:30:01 +0200 Subject: [PATCH 016/124] fix: escape control characters in the sandbox deny-path regex and forbid raw control bytes in sources --- scripts/scriptSourcePolicy.test.ts | 34 +++++++++++++++++++++++- src/runtime/grokWorkerSandboxProfile.ts | Bin 2154 -> 2169 bytes 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/scripts/scriptSourcePolicy.test.ts b/scripts/scriptSourcePolicy.test.ts index 1a73f68..bea5b23 100644 --- a/scripts/scriptSourcePolicy.test.ts +++ b/scripts/scriptSourcePolicy.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import test from "node:test"; @@ -37,3 +37,35 @@ test("script source policy detects a maintained JavaScript regression", async () await rm(root, { force: true, recursive: true }); } }); + +// Raw control bytes make review tooling classify a source file as binary and skip it. +// Escape them (`\u0000`) instead; tab, newline and carriage return are the only exceptions. +const RAW_CONTROL = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/u; +const textSourcesWithRawControlBytes = async (roots: string[]): Promise => { + const results: string[] = []; + const walk = async (directory: string): Promise => { + for (const entry of await readdir(directory, { withFileTypes: true })) { + const entryPath = path.join(directory, entry.name); + if (entry.isDirectory()) { if (entry.name !== "artifacts" && entry.name !== "node_modules") await walk(entryPath); continue; } + if (!/\.(?:ts|mts|mjs|js|json|jsonl|md|c|h|inc|toml|sh|yml|yaml)$/u.test(entry.name)) continue; + if (RAW_CONTROL.test(await readFile(entryPath, "latin1"))) results.push(entryPath); + } + }; + await Promise.all(roots.map(walk)); + return results.sort(); +}; + +test("maintained text sources contain no raw control bytes", async () => { + assert.deepEqual(await textSourcesWithRawControlBytes(["src", "scripts", "docs"]), []); +}); + +test("raw control byte policy detects a regression", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-control-policy-")); + try { + await writeFile(path.join(root, "regex.ts"), `export const r = /[${String.fromCharCode(0)}-${String.fromCharCode(0x1f)}]/u;\n`); + await writeFile(path.join(root, "clean.ts"), "export const r = /[\\u0000-\\u001f]/u;\n"); + assert.deepEqual(await textSourcesWithRawControlBytes([root]), [path.join(root, "regex.ts")]); + } finally { + await rm(root, { force: true, recursive: true }); + } +}); diff --git a/src/runtime/grokWorkerSandboxProfile.ts b/src/runtime/grokWorkerSandboxProfile.ts index 423fe217979515e49fe5b61d832766f700d5b48b..b6ff1263af9c25dff9e796ca661f1d12bc563305 100644 GIT binary patch delta 32 hcmaDQ@Kazz4y$lXsR0n^f+)i@5NV#axq$T-69BdR3a|hG delta 17 Ycmew<@Je7q4l4_Tu6+IGde&b|06R(sCjbBd From 838b5075e9d473a82abb73a2dc422a06b4ad8bf4 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 05:30:31 +0200 Subject: [PATCH 017/124] fix: forward the validated Grok request object and refuse unknown top-level members --- src/runtime/grokBrokerProxyRequest.test.ts | 22 ++++++++++++++++++++++ src/runtime/grokBrokerProxyRequest.ts | 11 ++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/runtime/grokBrokerProxyRequest.test.ts b/src/runtime/grokBrokerProxyRequest.test.ts index fb4752c..576ff0a 100644 --- a/src/runtime/grokBrokerProxyRequest.test.ts +++ b/src/runtime/grokBrokerProxyRequest.test.ts @@ -49,3 +49,25 @@ test("proxy refuses a reasoning effort or model other than the declared policy", const other = request(leanBody()); assert.throws(() => authorizeGrokBrokerProxyRequest(other.input, other.caps, "real-bearer", { model: "grok-3" as never, reasoningEffort: "low" }), /model policy/); }); + +test("proxy forwards exactly the validated object, so duplicate members cannot smuggle a different tool set upstream", () => { + const full = [...leanTools, "search_replace", "kill_command_or_subagent", "todo_write", "get_command_or_subagent_output", "spawn_subagent", "scheduler_create", "scheduler_delete", "scheduler_list", "monitor", "workflow", "enter_plan_mode", "exit_plan_mode", "write"]; + const lean = leanTools.map(tool); + // First `tools`/`reasoning_effort`/`model` are the fail-open values; JSON.parse keeps the last (lean) ones. + const smuggled = `{"model":"grok-build","reasoning_effort":"high","tools":${JSON.stringify(full.map(tool))},"model":"grok-4.6","reasoning_effort":"low","stream":true,"messages":[],"stream_options":{"include_usage":true},"tools":${JSON.stringify(lean)}}`; + const { caps, input } = request(Buffer.from(smuggled)); + const upstream = authorizeGrokBrokerProxyRequest(input, caps, "real-bearer"); + const canonical = JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", tools: lean, stream: true, messages: [], stream_options: { include_usage: true } }); + assert.equal(Buffer.from(upstream.body).toString("utf8"), canonical); + assert.equal(Buffer.from(upstream.body).toString("utf8").split('"tools"').length, 2); + assert.doesNotMatch(Buffer.from(upstream.body).toString("utf8"), /search_replace|grok-build|"high"/u); +}); + +test("proxy refuses top-level members a lean Grok 1.0.34 worker never sends", () => { + for (const overrides of [{ functions: [{ name: "write" }] }, { n: 2 }, { tool_choice: "required" }, { max_tokens: 100 }, { temperature: 0 }, { stream_options: { include_usage: true, extra: 1 } }, { stream_options: "yes" }]) { + const { caps, input } = request(leanBody(overrides)); + assert.throws(() => authorizeGrokBrokerProxyRequest(input, caps, "real-bearer"), /rejected/, JSON.stringify(overrides)); + } + const { caps, input } = request(leanBody({ stream_options: { include_usage: true } })); + assert.equal(Buffer.from(authorizeGrokBrokerProxyRequest(input, caps, "real-bearer").body).toString("utf8"), Buffer.from(leanBody({ stream_options: { include_usage: true } })).toString("utf8")); +}); diff --git a/src/runtime/grokBrokerProxyRequest.ts b/src/runtime/grokBrokerProxyRequest.ts index 309ad7e..24dcbc6 100644 --- a/src/runtime/grokBrokerProxyRequest.ts +++ b/src/runtime/grokBrokerProxyRequest.ts @@ -34,9 +34,18 @@ export function authorizeGrokBrokerProxyRequest(input: GrokBrokerProxyInput, cap if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed) || parsed.stream !== true || !Array.isArray(parsed.messages)) throw new Error("broker proxy request rejected"); if (parsed.model !== declared.model || parsed.reasoning_effort !== declared.reasoningEffort) throw new Error("broker proxy request rejected"); if (!exactLeanTools(parsed.tools)) throw new Error("broker proxy request rejected"); - return { url: "https://cli-chat-proxy.grok.com/v1/chat/completions", headers: { authorization: `Bearer ${bearer}`, "content-type": "application/json", "x-xai-token-auth": "xai-grok-cli", "x-grok-model-override": declared.model, "x-grok-client-version": clientVersion, "x-grok-client-identifier": "grok-shell" }, body: input.body }; + if (Object.keys(parsed).some((key) => !LEAN_BODY_MEMBERS.has(key)) || !validStreamOptions(parsed.stream_options)) throw new Error("broker proxy request rejected"); + // Forward what was validated, never the worker's bytes: JSON.parse keeps the + // last of duplicate keys, and an upstream that keeps the first would + // otherwise see a different `tools`/`model`/`reasoning_effort` than the gate. + return { url: "https://cli-chat-proxy.grok.com/v1/chat/completions", headers: { authorization: `Bearer ${bearer}`, "content-type": "application/json", "x-xai-token-auth": "xai-grok-cli", "x-grok-model-override": declared.model, "x-grok-client-version": clientVersion, "x-grok-client-identifier": "grok-shell" }, body: Buffer.from(JSON.stringify(parsed)) }; } +/** Top-level members of a Grok 1.0.34 lean worker chat-completions body (live stub capture). */ +const LEAN_BODY_MEMBERS: ReadonlySet = new Set(["messages", "model", "reasoning_effort", "stream", "stream_options", "tools"]); +const validStreamOptions = (value: unknown): boolean => value === undefined + || (value !== null && typeof value === "object" && !Array.isArray(value) && Object.keys(value).every((key) => key === "include_usage") && typeof (value as { include_usage?: unknown }).include_usage === "boolean"); + export function exactLeanTools(tools: unknown): boolean { if (!Array.isArray(tools) || tools.length !== GROK_WORKER_VISIBLE_TOOLS.length) return false; const names = tools.map((tool) => { From 5fe9374ee8b04cd52366666a4a65f87a5f35678a Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 05:34:00 +0200 Subject: [PATCH 018/124] fix: send the Grok session-title request to the broker proxy where it is refused before any spend --- src/contracts/runtimeContractManifest.ts | 6 +++--- src/runtime/grokBrokerProxy.test.ts | 15 +++++++++++++++ src/runtime/grokBrokerWorkerConfig.test.ts | 4 +++- src/runtime/grokBrokerWorkerConfig.ts | 20 ++++++++++++-------- 4 files changed, 33 insertions(+), 12 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 33ec1cd..55eabfb 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -53,9 +53,9 @@ export const GROK_ENGINE_BROKER = { systemPromptSha256: "2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8", // sha256 of `renderGrokBrokerWorkerConfig({ model, reasoningEffort })`, the only accepted config.toml bytes. configSha256: { - "grok-4.6": { low: "cddeac5f845fa44890679934ad840ca728f5ddcdb0ffdb34a60ed232ef0eda3f", medium: "5a986b87f3888e8b2ec1fe5f7e5361a96494b67bd66a8b9f39836c26f2d602da", high: "e023b566aa28074c544c2f44dad634eca5511a329a2030c2584574e7f419a75a" }, - "grok-4.5": { low: "e22e543e72ea409cfbb3c229ed3bd605252374d28922f6ab05c29f6d590108de", medium: "706a74cb8401b045604078f526b7afcaf292dbe85c8c26f20b533e828f8a3acc", high: "3357ca76540480f4f3f7dc7e4e05ac92df770104a7ecf6d0aa3cb29eb9b6d0b6" }, - "grok-build": { low: "905efbe13bda08f3aa3907cb0215a56c07553965f8d81242687f7c993652b294", medium: "56aab068913b69aac68862ab729462b943a31b9efb7ddadd1d1deebc0ff72714", high: "52d66731dcad01ca001243424e324b8c4921a17ab31fe6b7b869dc8473d2c3b6" } + "grok-4.6": { low: "eed6a451150a72b2cb528b30c23b3d51c7d3bc38c67a8985d4dcdf956ff214d3", medium: "8850502dbebf8918c5161c63efcc4ccf18719488300f4cec1deceb2c112b451f", high: "3ce44ace503362326b47149b528b942ce638fe146313d62502f248acf9c7333d" }, + "grok-4.5": { low: "7aa13e90b9bc08d1a018f48b7a84de1dab41db586627ee2d5a25f69011ba7e25", medium: "218ba37e57a6f02fa36b265b4e154e68e30bd2d4794feb130cc226fdda7732a9", high: "0bb4ad8bfa5062169b28422d1d534b45420d4e46b1e546bda1c578eb34303646" }, + "grok-build": { low: "83ac7202442286a65c359cc596b0b8db7bc4529ee70e98224f6cd6f66deb6878", medium: "0146313f28739888eb4e861f1bfb285f7ee4e0a9164669256ebdf6492a2790ce", high: "bbe72aaf70c417dc7007823a7e9e1a7d1fa8d57e50bde6f24036083b32bcc859" } }, // Worker `GROK_HOME` layout the broker attests before every turn. The home and // its `sessions/` directory are root-owned, worker-group writable and sticky so diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index 3293903..0a935d5 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -2,6 +2,7 @@ import assert from "node:assert/strict"; import { request as httpRequest } from "node:http"; import test from "node:test"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; +import { GROK_SESSION_TITLE_SINK_KEY } from "./grokBrokerWorkerConfig.js"; const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); const leanBody = (overrides: Record = {}): string => JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: lean, ...overrides }); @@ -52,3 +53,17 @@ function post(port: number, token: string, body: string): Promise { req.on("error", reject); req.end(body); }); } + +test("the session-title sink is refused before capability, guard, credential, or upstream use", async () => { + let calls = 0, accessed = 0, guarded = 0; + const proxy = await startGrokBrokerProxy({ accessToken: async () => { accessed++; return "provider-token"; }, markRejected: async () => undefined }, async () => { calls++; return { status: 200, headers: { "content-type": "application/json" }, body: Buffer.from("{}") }; }); + try { + const token = proxy.capabilities.issue("agent", "turn", 60_000, 1); proxy.registerIsolationGuard("turn", async () => { guarded++; }); + const title = JSON.stringify({ model: "disabled", max_tokens: 100, temperature: 0, stream: true, messages: [{ role: "user", content: "prompt-derived" }], tool_choice: { type: "function", function: { name: "session_title" } }, tools: [{ type: "function", function: { name: "session_title" } }] }); + assert.equal(await post(proxy.port, GROK_SESSION_TITLE_SINK_KEY, title), 503); + assert.deepEqual({ calls, accessed, guarded }, { calls: 0, accessed: 0, guarded: 0 }); + // The turn capability (budget 1 request) is untouched and still serves the real request. + assert.equal(await post(proxy.port, token, leanBody()), 200); + assert.equal(calls, 1); + } finally { await proxy.close(); } +}); diff --git a/src/runtime/grokBrokerWorkerConfig.test.ts b/src/runtime/grokBrokerWorkerConfig.test.ts index f5ea166..091418a 100644 --- a/src/runtime/grokBrokerWorkerConfig.test.ts +++ b/src/runtime/grokBrokerWorkerConfig.test.ts @@ -28,7 +28,9 @@ test("worker config disables every bundled 1.0.34 skill, workflows, and the per- assert.equal(section(config, "[skills]"), `[skills]\ndisabled = [${GROK_1_0_34_BUNDLED_SKILLS.map((name) => JSON.stringify(name)).join(", ")}]\n`); assert.equal(section(config, "[workflows]"), "[workflows]\nenabled = false\n"); assert.match(section(config, "[models]"), /\nsession_summary = "daimon-session-title-disabled"\n/u); - assert.equal(section(config, "[model.daimon-session-title-disabled]"), '[model.daimon-session-title-disabled]\nmodel = "disabled"\nbase_url = "http://127.0.0.1:9/v1"\napi_key = "session-title-disabled"\nmax_retries = 0\nhidden = true\n'); + assert.equal(section(config, "[model.daimon-session-title-disabled]"), '[model.daimon-session-title-disabled]\nmodel = "disabled"\nbase_url = "http://127.0.0.1:43123/v1"\napi_key = "session-title-disabled"\nmax_retries = 0\nhidden = true\n'); + // The sink carries a static placeholder key only: no env_key, so it can never pick up the turn capability. + assert.doesNotMatch(section(config, "[model.daimon-session-title-disabled]"), /env_key|DAIMON_/u); for (const toggle of ["title_refresh", "telemetry", "session_recap", "turn_summary", "backend_tools", "ask_user_question"]) assert.match(section(config, "[features]"), new RegExp(`\\n${toggle} = false\\n`, "u")); assert.match(section(config, "[cli]"), /auto_update = false\nuse_leader = false/u); }); diff --git a/src/runtime/grokBrokerWorkerConfig.ts b/src/runtime/grokBrokerWorkerConfig.ts index e4967ef..5a2f88c 100644 --- a/src/runtime/grokBrokerWorkerConfig.ts +++ b/src/runtime/grokBrokerWorkerConfig.ts @@ -27,15 +27,19 @@ export const GROK_BROKER_WORKER_MODEL_ID = "daimon-broker-grok" as const; * turn, and no config key or environment variable disables it * (`features.title_refresh` governs only the later refresh; verified against a * loopback stub). `[models] session_summary` does select the model it uses, so - * the title goes to a hidden model whose endpoint is a closed privileged - * loopback port: the connection is refused locally, Grok falls back to the - * truncated prompt as the title, and neither the proxy nor the provider sees a - * request. The placeholder `api_key` is not a credential; it only stops Grok - * from looking for one. + * the title goes to a hidden model whose endpoint is the broker's own provider + * proxy with a placeholder key that can never be a turn capability (shorter + * than the 40-character capability alphabet). The proxy refuses it before any + * capability lookup, isolation guard, credential read, or upstream call, and + * Grok falls back to the truncated prompt as the title. The endpoint is always + * listening while a worker runs, so the refusal is bounded by one loopback + * round trip rather than by a connect timeout, and a prompt-derived title is + * never delivered anywhere but Daimon's own proxy. */ export const GROK_SESSION_TITLE_SINK_MODEL_ID = "daimon-session-title-disabled" as const; -const renderSessionTitleSink = (): readonly string[] => [ - `[model.${GROK_SESSION_TITLE_SINK_MODEL_ID}]`, 'model = "disabled"', 'base_url = "http://127.0.0.1:9/v1"', 'api_key = "session-title-disabled"', +export const GROK_SESSION_TITLE_SINK_KEY = "session-title-disabled" as const; +const renderSessionTitleSink = (proxyPort: number): readonly string[] => [ + `[model.${GROK_SESSION_TITLE_SINK_MODEL_ID}]`, 'model = "disabled"', `base_url = "http://127.0.0.1:${proxyPort}/v1"`, `api_key = "${GROK_SESSION_TITLE_SINK_KEY}"`, "max_retries = 0", "hidden = true", "" ]; @@ -101,7 +105,7 @@ export function renderGrokBrokerWorkerConfigWith(policy: GrokBrokerModelPolicy, return [ renderGrokLeanBaseConfig(), "[models]", `default = "${GROK_BROKER_WORKER_MODEL_ID}"`, `default_reasoning_effort = "${declared.reasoningEffort}"`, `session_summary = "${GROK_SESSION_TITLE_SINK_MODEL_ID}"`, "", - ...renderSessionTitleSink(), + ...renderSessionTitleSink(proxyPort), `[model.${GROK_BROKER_WORKER_MODEL_ID}]`, `model = "${declared.model}"`, `base_url = "http://127.0.0.1:${proxyPort}/v1"`, `env_key = "${GROK_BROKER_PROVIDER_CAPABILITY_ENV}"`, 'api_backend = "chat_completions"', "context_window = 131072", "supports_backend_search = false", "", `[[model.${GROK_BROKER_WORKER_MODEL_ID}.reasoning_efforts]]`, `value = "${declared.reasoningEffort}"`, `label = "${label}"`, "default = true", "", From 08ceda6cb93e0e7cdc7f15d154395643868bdb4b Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 05:34:00 +0200 Subject: [PATCH 019/124] fix: give Grok workers /dev/null stdin and keep the executable fd out of the worker --- src/runtime/native/engineBrokerLauncherCore.inc | 11 ++++++++--- .../engineBrokerLauncherIntegrationLauncher.inc | 2 +- .../native/engineBrokerLauncherIntegrationMain.inc | 7 ++++++- src/runtime/native/fixtureWorker.c | 3 ++- 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/runtime/native/engineBrokerLauncherCore.inc b/src/runtime/native/engineBrokerLauncherCore.inc index db3abdd..cb2b9ec 100644 --- a/src/runtime/native/engineBrokerLauncherCore.inc +++ b/src/runtime/native/engineBrokerLauncherCore.inc @@ -320,16 +320,21 @@ static pid_t launch(const struct dbl_registration *r, int executable, int status_fd = fcntl(status_pipe[1], F_DUPFD_CLOEXEC, 16); if (status_fd < 0) launch_fail(status_pipe[1], 4); + int null_input = open("/dev/null", O_RDONLY | O_CLOEXEC); int high_prompt = fcntl(prompt, F_DUPFD_CLOEXEC, 16), high_capability = fcntl(capability, F_DUPFD_CLOEXEC, 16), high_output = fcntl(output, F_DUPFD_CLOEXEC, 16), high_executable = fcntl(executable, F_DUPFD_CLOEXEC, 16); - if (high_prompt < 0 || high_capability < 0 || high_output < 0 || - high_executable < 0 || dup2(high_prompt, 3) < 0 || + if (null_input < 0 || high_prompt < 0 || high_capability < 0 || + high_output < 0 || high_executable < 0 || + dup2(null_input, STDIN_FILENO) < 0 || dup2(high_prompt, 3) < 0 || dup2(high_capability, 4) < 0 || dup2(high_output, STDOUT_FILENO) < 0 || dup2(high_output, STDERR_FILENO) < 0) launch_fail(status_fd, 4); - if (dup2(high_executable, 5) < 0) + /* The worker needs fd 5 only to be executed: close-on-exec keeps the Grok + image descriptor out of the worker and every tool child (execveat with + AT_EMPTY_PATH still works for an ELF; only a #! script would lose it). */ + if (dup2(high_executable, 5) < 0 || fcntl(5, F_SETFD, FD_CLOEXEC) < 0) launch_fail(status_fd, 5); close_other_fds(status_fd); char *const argv[] = {"grok", diff --git a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc index 20d6849..ee189d9 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc @@ -110,7 +110,7 @@ static void org_cases(void) { char *out = calloc(1, r.output_length + 1); check(read(s, out, r.output_length) == (ssize_t)r.output_length && strstr(out, "uid=2200") && strstr(out, "--always-approve") && - strstr(out, " prompt=prompt\n") && + strstr(out, " prompt=prompt fds=0,1,2,3,4 stdin=/dev/null\n") && !strstr(out, "EVIL"), "fixed worker boundary"); check(strstr(out, "=--verbatim\n") && strstr(out, "=--no-plan\n") && diff --git a/src/runtime/native/engineBrokerLauncherIntegrationMain.inc b/src/runtime/native/engineBrokerLauncherIntegrationMain.inc index 0b816dd..f5ea29b 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationMain.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationMain.inc @@ -43,8 +43,13 @@ int main(void) { auth_adversarial_cases(); puts("native-stage auth complete"); pid_t broker = fork(); - if (!broker) + if (!broker) { + /* A real launcher stdin, so a worker inheriting it is visible. */ + int launcher_stdin = open("/etc/hostname", O_RDONLY); + if (launcher_stdin < 0 || dup2(launcher_stdin, STDIN_FILENO) < 0) + _exit(1); execl("/opt/daimon/bin/daimon-engine-broker", "daimon-engine-broker", NULL); + } check(broker > 0, "broker fork"); wait_launcher_ready(broker); root_peer_rejects(); diff --git a/src/runtime/native/fixtureWorker.c b/src/runtime/native/fixtureWorker.c index f53b77a..b26a4ae 100644 --- a/src/runtime/native/fixtureWorker.c +++ b/src/runtime/native/fixtureWorker.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include "engineBrokerLauncher.h" -int main(int argc,char**argv){unsigned char bundle[8200]={0};ssize_t size=pread(4,bundle,sizeof(bundle),0);if(size<6)return 21;size_t offset=0;unsigned provider_length=bundle[0]|(bundle[1]<<8);offset=2;if(offset+provider_length+2>(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output");if(!strcmp(provider,"sleep"))sleep(30);if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;i(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output");if(!strcmp(provider,"sleep"))sleep(30);if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;id_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target);for(int i=0;i Date: Thu, 17 Sep 2026 05:34:13 +0200 Subject: [PATCH 020/124] build: rebuild native engine broker artifacts with the worker fd hardening --- src/contracts/runtimeContractManifest.ts | 6 +++--- .../artifacts/daimon-engine-broker-arm64 | Bin 67456 -> 67456 bytes ...daimon-engine-broker-arm64.provenance.json | 2 +- .../native/artifacts/daimon-engine-broker-x64 | Bin 55392 -> 55392 bytes .../daimon-engine-broker-x64.provenance.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 55eabfb..20ae6be 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -69,9 +69,9 @@ export const GROK_ENGINE_BROKER = { }, bounds: { promptBytes: 65_536, capabilityBytes: 4_096, capabilityBundleBytes: 8_196, outputBytes: 65_536 }, artifacts: { - sourceSha256: "7b0a1da4add25abb2176bb5c882f247e03bf099f23fb295d72509253c7696beb", - x64Sha256: "e7028d17c0aff843c37dd9393f9c30c6218329bae64f8ddf907296edaf2db349", - arm64Sha256: "7a37683ebefc6d935260619d635bad919b7bdbfb552440e044449d909a8f8ce5" + sourceSha256: "36f60689f0a8af0e3108f5f53d78ed52b7d4b6f934c75b6184606dfa82bc741e", + x64Sha256: "36dc76b134eb59cf5a6720b6f94228eb279108e20ea3343fa6efd9ffcb60a4d3", + arm64Sha256: "c93216cc6fa4ca50dc404fe41e68da9150a869b14f46eb42484ae77c3aa400a9" } } as const; export const AGY_SUBSCRIPTION_REALM = { diff --git a/src/runtime/native/artifacts/daimon-engine-broker-arm64 b/src/runtime/native/artifacts/daimon-engine-broker-arm64 index b836e7380a5cae750f1c199877dfe3f7550f8d7c..daeaf6e3c64540ced18ab7c60f8e232824e7ba86 100755 GIT binary patch delta 3362 zcma)9dr(}}89(RV1(t`%x;z$0vJVJ^4F*U+z(`mYNP>;lQgoVVhefP$LQ+kDq-i?t z-Gx{+j&kOtZcGzHYSZamO-EgPP@B5=noQLWZq-_+wz~-$ZJL&h;T0gazjJ{k)A0}Q z%(vh3evfm$^PRiO69(l8L*H;>-)^Emee7?qn?|?KF_&>jrH68_kIF16n}cPV79jDY z+)l`_;O#a8D~8Mina)FQ_wXHyAEn#)?BYJ^4?VslSC$-c{8{_G0+}6K=slqa9S%x& z@S}N?R0$o--$ZFsXl$t`k-i$bqa@cz!+b|&9({{HS-IN#2iL4Zlx8J?3?6wie&!8L z%Sw6l;fU5={BE+}ADP`l)LEIZufI>#3wdem>-OdQ#_p8VF*zY zADrqis263~IVBsM)17hZv@AIt#2<-mCW90)Y9_US-~}zpsRr*o1tJC~Q8~xC<&qX~ zRgjA4o0L)7-`oa+a5<^aCbYnWOq^Ciwx6vaExt_B676&i3Ws*8G~3|tWzb44sjvZC z4P?75m$ZmL1aK#0{W+w90?R*h@wlGeh(yS!+f!1^#@d(eqedkXl{1242qdrw9uH#? zG;EoyG(7ZBWU@VLrp=%rkFvc-xkf3Q1C$hZLpR46P`| zPEaYCOb^x4ij@FF21`1@UhN5Rh`nJ}5GbdXA=Ss3&KK7`PfG4ZM=t&W+KzIk15S3z2$N9O{tGzcNegYcG4@s=OaNo1A z(vK4IXaT)UkGgE1)h2&*YCdx7Rgnwy$OZHIx#&W_5Pj#gHt^5u$a(ukttGmuYp`D; zgMY@c7M*rrLh}afq(5RM^eh?n9o!9}*itLW?s$%nmSl8obpkQf$~hf`k`~*wdX*veQu=s3VTo-%fwV$sY5qWC%LiQCGwMRd}s>#pxP2UBkY#rm2@5A6>)Fgx9byvP|k_ z=cJY$^VG7DYp+xF07@+rXPW}t(Gq4SrR>wS@oeguc=qvHi4iXpB)pQHN=Q{N8@g68 zyRC+umUDvtmdz@v44z6zSA9#Bxj_-FWoMUXs9)t}s^iuyb-Xr1|5X-xd~yv)8*zMmHm)s%*j23d4H=b(u33>6qY$fl(1hxo~Q49p1!wY{{?(g z_c`<)aOig5)PY4vVi};QjWXj9Wrd&%0B1asm%Y<3v5|U-o$itKrHUi|X18q>6Nkf= zr$naiOjO6(lTdB4dL}nT)jG)3D!r907}X?5420F9;VABjSi60n=%qTnH!^}R%^!u= zC5wJcMEeiHuOdbs8s1L4{>YInwkqL7ym%BZ&MEv6@Bhh*&}8Ct_F-kjiG3V+!S>tw zum~SW!iOYUkwi2Pj_SbNB^b~`v2mVm1APZ*-e2pYxA9o5g}%x!*4|1F@+Mbd&4!Cw zpbO&#JL0SnIqh)`x}?b)cIvWBXO)n7Ap>6lT8k%T9vmtWzl#mnBjqyLZ*CghQAyZ821KdWod$DKakftA^khKipuYP5eYd9zDThpcir4Xr@^_qj6Q*rzmEs zqd}eSY*bHP(z3XxvDC3}5>83xcYTr>GdIK81lOGsLGzLG=ZrkqxWw3mc_mEwdyUJg zBzeI$)A`>ES8T7jq_ucc$@xU|-Gv^?#79du>1|$!!AsXJx^3}Q&bJvzORH=?Ys3p? zbnt?q`>FsRTovG)bj5x`vUCj1Xf3>dZ9aXQzrD85`Fr%Ea3N;Z{9*d8Lm|w6U${od zdgQt98Q)fZ-MU)S19MW@FfZ-mcdvV#M)~=5t)`P-Yk?Rhju;RQWH!Rv)~}`?@x$wD z=`sG<`ds=)KDGWkbb>cFSxwInvj4~yT;lWJI-5h?O=fEP4V)}Eh+$0K!JoRhkdGFo zg~B)6=!+Xlt@J;O^Fj~&C2;90nwEm090o22zUfs>+X_4k zdrXm<9|JtVeVtj-eFJ<+*NQ|N1b+uS$3N)IL%)x87Sdk6xT}($etvuRI%)C*f4X}ueUE?M-Fjmm zihd8}5O?bE!_VNT&>B+sOhCr|BK#$4&l;3pFXR80DVmhRy zN#p#%oiE4b3?G|D>eWC delta 3288 zcmZu!4Nz3q6~6bq1(zR{&GNH~y1V?dK?Ri}{_WcZMAMiUvi*svW$o6ON@k*fvDQxZ zEf8ZQjo!H`#u$Us+Rocb#$?G48CxN?PG+nfyG@$dNiq+y(Z*&*Mo|#(^}8z|opfgo z=bZ1Jd(OG{o_p>?-D^_!n)Y2tPlnEZu=#k91?K0D4Yq-QP+DJTE zih%in#dP3K2d|&KhyIL@&fZ78kyqwq%F-nh|Go1m!K_)EsXy|3Ru-jO z__6FkszeUwtfSNu8OZe~(36qx&(Ac|_j!F;Ha*1;mMsl-Dx)ey0z=brOjtk#-oN6~@*>yW0YlCupc&N^#4ap>YN2V@a zP771`8g7WwhGmm$M3!7xq$%niG};oInxZ!Yu2!K78(mHfmje%8g3ybJSf6Y2U95Ex zDb>$VMrnJ^1{mlINvUz$XuB;Fmz|KUNu|VhES>oD2Bm8`#+_8HalqlbkR7u~sRP_@ zB3rvNiBAL~peKPTnWU70%fD2(sktR85pvz@pI^jo1#()cS&iy)dUybV1Q)^MVJw3B zXAG9fyLLqf1KE6dv2(#x8}Zre$`m)O5yjEOnAJTo?D1bO#8t90D~ z?gtSfyVTslU@XuEEY5~WBv%Bsxs{v_gtB!C{G#jMa%%*>FdfXv5= z*%iH*#Y2t*opqG3*->(dF#x{OQ_#pDW;V1-oib z(I)(ESi!ESE7(^VL5nuwyn6*Z?r^iq(;v`c0TeeASC@jk?em$ldp=uI9nWt3I-ZqR zOAPJxI#Cy}OH-z46Lw@FbLz`jpFA`CAK9v*2*Q`9q-vpD^1ro8!B<)IOvS!qgArERvl>K-qkCk)HL_HHc z1$w~I>L%ox2W8ufmHuz$wpclf67E6pJ?Lyg7KywFS#&ZdTK;Knac6L!J7spMxOph! zHtdJEfd|09Wu@c@EBFb)&r`xDEIp6|W3s^gWAX;b>mg5G1V8CksE_x%ZS*8hR34#6 z`4dY1^0jx2wnid)ORCEw642-t}|Q z$!?;p{uvF&5yR85;Ff3)FY#6;s}dU>PU@&tY$LznA+MEI^8?-rNxH@V>0Oa9X8@UQ zQo4?CXGMP6*U%i4bUJVoB!?P%VgJR~SCr8pKTwfxSu7%t8jE_78TxMc06$-0vnetY z-dAI?cpda%qpTN|{tX|ksHMmF`pTtgKgO6U<3+{>jJK#E-cf0%Yxw!fY}&*7Av?IG z%1SeNZq-8DmsreDU8OeMP^Eoz*T@(@75;Qp@ytt-^;}4@qPEjqYv8&;BCU2p`llxF z531&v3Muh%y{cf@_$0w4d3;W~tN-TxIUS>hFF1|#C!j2EHd7|PL}rkd8m_FI<#SGQ z{wovl)ymduW*AygkKRcRKb_=ojeOtkwq&cL2rxMjeqv<~J;VRDGC%Kql&9!McqYQc zL>|pj32XXk_i}+X%lAG=LQnIoYPaQ?d#G$0L$~ny>R0GxPFK}h{&GWHKXIc{(I?dC zRlarAQhJehta4M0f3qr+e#jG6f0qvP@2$35en-guH#XxHj(x18g#(cnR$Hkh3@76b z#M88nYY*i|^oJajx92(Dez@39zv3lz??nddOz}ZS*f2gq*Dih=yMb=5cN#_xnyK=R zVJrsU0{$@g32;9+dDk#nz#ZU+z>C38igEC(;L3Z3k&K>r0elho!`PZhqU(Mi30S5zmx4DPToH#zKk($4_7iXcn+#;Ns^1`9PKYiH7X7MvTCD?3aqa zFN0Lh8ZxAxck+3S^Aa2YJ_nuRy$#v4h{rYN)BSvIV;TL7Z)}`Tm+=FQJ7=0s8HTVV zo@tKPO-JP%h&fnw@+t1xmPh^kv29i~ppDzACG#Koo7+~>&-uW%+9mB+^hqp-c&A<1 zO+Bygu^fHL;ytj-a_|L91>bGi|Dwg))?zu-Z1KL@WH}tNRK2EJjug_q{KS=!avrqo<_m98$avJd@tv%dqeznECkvDH&PS-|yw%1asbLWnXw5$C% zW|1g;iI_GnVz52KSR1v2ZP=r`iVtH4=tE%Z-_hCrVeHQECh=J-DGsOD9#16e zs8>3WO*aUFtejKT?-&b2MgIwy%Ar{S13qcMM|F5TU46@|#7jp{QOM?S!;}jq5)1pL zWtmJFlf&=aPDhd!I(Npi0qt5j!oPHJ)UIYiCWc5DRV2KTe8iWx*N} zqn{Pr<(*o{N8B$(6S&+e^^6e&rBbqdE$`eXn9#mgN|8fv>b&x?tXwQq3Zz*i06;Mi2Hw0xyb ztw0`}GoAB<`7=v@np5r>xtvQ3-2UlL{tLMub3P+_vtDPy8GwGnn&zfek7 zA4W$G-Ycccfta|_1JzD{3O5sEOu-hJ#>2jdP-I@sd79s+Oyau^lOO$Gpx=r=;me60 z&(Fn7zaXTbCuR!byG2CybrG(WoIu}pWGu_$!{=yz{)IZop-L&I6*EO9zS7fDK2i`o z;-x639Bql*)3id#p}vkFdD?QoYl(Ud0dXL6B(*sw+)jWygU{Tce53rJesE6b`bKD= zMFZCx;D_{4=Om(1#W`_A9{12q-*S%gZiTLvN;#B&{~8PuRp($#WFV#0iZCK05DcQv z0@3<+p1!Ns;VHmh1rCF$*AeL;6rHeWPBhl3bOK+?@k%Kr=<-TgN*CN)l8!6=fYOut z3zPbju8FlVGoP_?LOl&-#9P}AfQJ>~Sy$T(bf=D{M|jTE9R|7?XoRmjzLe_?u$qHN zzXp1B&|AA&cVLM?3hHAT{M1Byw3VA{>TMd+_BSmIWd(3e7Yj*!|B5#AJOsK&TXJdx z93aZsT6yK*9;vhtZcP_-G!U)2Ne=ooQ1;ChM0;Q(ANFBb$sxbgjy*WIoPf*91=)9) zbeaS?_i)u^u5pr<&6Cp4L*E{=5AYaPp}Ky*o3FWJJ+@XY zMb|J79$>L@Sly5I;ILw*~PZbN^cP(bwX7^W`ll`4Kr} z4y+ag7osjju0682#@h{00T{^w;!>`9#PW7d^Z?%kTvj$n%R!0L{8A$l$GJ1y zogj2L7Dv00>xWWrEe9l)ALOtKcpG4D&Tk8j^>gzI-Q4?lneuyjaxV9LE zQNyr@8#-WE913g#CM$j^2eCnQEmVtZ4sSva9y&NP8*P`;3$o1c(*qms!pz5phJy%R zZhXH!?qQC}y1ylcN!&0wyID7JgSJO8L?@q7zrh{`3#^M{sF*g$J{@c4nAO1E(y@aa zvjJ0Y@Seqitn_Nxw@3LEQ>Ptcsb4+L2Yea@?fn=_N%<&}jza^dG7@|q9QJw-C$Qjb zveL_KQ$GhDu|ZAU%c-BPdQaW(b?@d2e=?7rpR;&m++Myz`o=hfjS*Za**wMKnoi_N z{Vm3giDPl7@DiN?v{wtWUaEsBKUdwfWNw<*t!-m_lq02bXoJ)UnrFVL9;7)`)aAYpm?nm|c zOS&iRxKnEedziGKK56uNJ^HwL`boQTP)R$D7oj&~&x@HfQ+|fj(U)bL_$W3SU$ZX0 zlwxP-QF+Ro+dgMZITtwllv%y|j5c;ofT1%T9l8Bmo54-ixtSU_79FRLu}TX+^(=H& z@!h8AYPzoY*Ca?|#m%PXPwDUGk0-Ts^ZZHUJ^K5G|K=ZfZuU#xV!=jg_=H^aL}Cd!ZfWZdqpHgsIdC3V~>J%SkegO^9u2MCeht_Qe9(0uGsZuqp66A>(y|0Y2# zpt}~{p0*o{b;Sctr2_^VuYRM|Z6@-0d0}zJZX~idtYcR7N({Xqdc0I$#OQsCzC0oFBadw6Elri8~(= zgs!6U3BbhUksk30^9~?la*@@}4teOMlAnzU>MFJ$RTNt0eu7M+r`*+eTVA#}mmH_h zFAkX06Ljnn2N^-{TT(^p!pD~65b|L7houjgE&LXAg)$b0{VTT3pYirDc{(dc+oQ-A zJ7fgqT!xyROU4;-@3N(`ug}tDu7RSqTN>!HraOsDLru?3YQIa*``TTM6)PQI?$({( zvBSVu(3#C{lVuluuK7;l4M&6 z6!H|L<5k8~Na0V6@y`WEE0zO)={>?&EtlBOM!t9-efK1gXX!)xr4jL3lJG(z&HVGYq?H)NjU98IzP0~B za|aUTrO5|gyzPw_7%KqB9~|;RDV&@df|UZR1SU`#JMf!n<@JmqvD(Ac`#`M%h8VOb zi~om08NEM}U68otdhJ4P3M^gI)dyS->;_<>fpM-Jm}LunKXP;aCn)7ZL)5o|nvJqP zI0S0}<_Cs10PW#{bpVUgNo`~DvrzP*Odpt9V5^2`3n>2iAy^@}6ku+;wyh|&`)%Zi zpY3=Y_^SYOg&u30Ln>%;bdD+O1pQU?k0eSjNAEV}d`73Y7m#YYwtYF7BN*+$9f?4-umbd$z#?&(_- F{sX8JB~${X99Z)W!pM*)qdg5a3?}pxt>}zS@z6=IaLbi7+$~m%%y!XjPk#oR z!Fb`QbhX@$$3<7lJ*zf3T1V31Aqdo3qLoX!t!!=iCJAwC)h+Ap@4oj);xy;L_wMia z{kz}2_q*@)hU0p}abYV-T)tM&f6F7uWW#kNpKc$WmR=Y(!0;JES`8tT(-UC-V6y5k zb~7bnTKayI?HjXho&T86K#%fhDcpRp3X!;W-e8Pl}8E`dyc=8S+($P~C5{hh@^0|e)9SLOTSS))c zMP9p!jwHM29r@3Yqmi!s0z!UAEz_5iY+5ya9I;30re|1*K${BRB_3Kma|L;uelqiR zzr)KT9+JXwTyB-r6hTm?Nngr;Jt$c4SSDHJ&~AfPPRmNaSMf@tI!5Y7Tp5}ed6mmr z+5hxO)Sem1T|FsfMFqhsXdiuI!0muF+rJs}D4m#V-8qa!f9>J~L8ynamhdrSE;X8n zxwa7e(+&WDVr?c?PTK`90gZ^kWQ<&wTufVS`b7*ui}f<*3+$=TbHKUySMb7{f#u^E~`WWCJpgLqTaEUl5uZ>flwFtaQkjs7>1rJALex9NGgI z$n|BCL#{n3uD=&{9l=y?CdinAEt<{KzJOF@zL>Y_ejm4pue&WljBmua8SaJinU$3{ ziW)5+=LuFz&St)cwO76{>cZf!QoNvY zDz9Y(C^6eBf!MOZ8ZwmEC@u9 z3%X^IR(Y-dEHP8?gzo${aK*AZ<&J{O)uRzgNtLut_^3~nJb@14DLA$2%$MTR@qgZe z61^pd;-@|$2)@vMsT8?Y%HoPHU@l^^I`*9A5RYy3i0lpJsZr^Cy4{-ZQhWQ-R{0j7D8Fp>lj^{Pg zwBxOiW8K{DKQ9VGaj3Sz4^UQuk`u{6HyyggYmPQxgwXAr$-=WwiGdttI#OZ7eY^o5 zTqDo0jW^;R<<`p%GZK)M8IraY8nW_X)M4YZIR`2P%B4=$0-V9l zv&l+~+uGNIh&rLCE#nj#%4@W>XsoaG2A-XM+PTxya-fZLSg@7aR$Y|q%OzW7vACiG zb!vPRo==KnM@YQfJwU4TG&V^!m=r(TZfYw^^SkxU>QXsUDu>qTJA)VY!546_4+=;D z>0EgTdw4cxiORHb)e5dM7b<*(IIvP%r_ZiuNOniKKbsdG0;-W^*D9>xCP_n=+vrnz zivzT&D8>K&Mg3IgxGpLC^~R~~IHMz3dXJLpvB%*>8+Fb=Yt>a_Hc*xN*21D`<;{P$56W<{OPnCzbdcm;Mo0rx7{l;l>)ZnJ++*%Ag0miDW`r6Q@;#rog zD*9RRV)6~0J-5kX|AfZojziS=c@xK-GVb}{^_T%agWuo`M%VDk9ipCjX_nM4=r87t zxBa(I@A$#DzJ*$S{LA#&dE>{a%Xz(xjIq7GLgI>7O2s|$E^V1-w`BFuQ}YTf1*fRp zcLzB_7yG7@-L%1%K{n96zJi5zX3v}RhabP zwRi|pdFuPunKUT-luJsd)=FpG<+eLfdF*;ejQ2F3(vR-0`DnvkX=Eq;%U$CaC13;L z0t;TiMET}S#--Zq#K5Oq(3JO$3{GM^xWzwx4=M61?=ZKhkDtDb8$K!JMFp$%`yyz2 zY3BT!(k5ZcFIwVJ+F`K%^4I#hO+!7G&MzK63GWZFh=JL(2jDhVj}c$0d*7p-^W~&M zGt~MSG`*yf?4dO!CFCjEU6N~=dV-SDj4?xYliAv6X6Xd+D=eXyS%U=8BV(UYU+Lrt zKdBUi&Z4ReU}EyfO7YS7L?B{vk0DrrgSq?GfhDpZ`;RARKf^iiVFX$xvd7Co?_ zfb`P~3u-NLH(h9`jf_bEdS75H2)S|#V@_<9 zFyu7I0Z18AeTlK%klm0fWH03Vkm0S2jli3>3h99Ch0K8*fOPZmHpa>!OaH{!O2{x| z5YpMm*p`ua*Q&w8ZU6(2DrCiWIEEB2Spj(!awViKNf2rvCCH7Cw?T#>eMxA6-TjdX2WQ6-J6(E6O6aa5 zanFpTnQx{imLq5>PFXeGbLjr~vIfRB(B#7}{#1URv62fb9k*ni%J* zfbH5$-;Q2i_~eU>Egho16;uzf`-fm*U=o)0;UQQ%uo^nCC8hA#Fs2Vo)*n&lA=&~K zKR5*Qf=dNvqmQ)|+3WVAM*MET&yBy8VD{5@TIP^Enyk*T^u0@euD(QS=tXswW#D6a zOY2PHrjNCz6qe%)ZZtQ;k5_jF*7&|*SShfwVb}w}N@;6rN;>zDrrR4H)`3~ZwIgfn zI}`SP&A4}QRJHphdX?2df~KlqW}XYzxe<=g4SzvYsO$i!o3 Wtz-!$ALJ%hylj|ML<&B*G4B6`YxHpd diff --git a/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json b/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json index 5b2a269..f2bcf17 100644 --- a/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json +++ b/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json @@ -1 +1 @@ -{"version":"daimon.engine-broker-native-build.v1","architecture":"x64","target":"linux/amd64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:7b0a1da4add25abb2176bb5c882f247e03bf099f23fb295d72509253c7696beb","binary_sha256":"sha256:e7028d17c0aff843c37dd9393f9c30c6218329bae64f8ddf907296edaf2db349","install_path":"/opt/daimon/bin/daimon-engine-broker"} +{"version":"daimon.engine-broker-native-build.v1","architecture":"x64","target":"linux/amd64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:36f60689f0a8af0e3108f5f53d78ed52b7d4b6f934c75b6184606dfa82bc741e","binary_sha256":"sha256:36dc76b134eb59cf5a6720b6f94228eb279108e20ea3343fa6efd9ffcb60a4d3","install_path":"/opt/daimon/bin/daimon-engine-broker"} From 0878c976aa3d25081e39324f65f37f18e2481a5b Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 05:34:23 +0200 Subject: [PATCH 021/124] test: cover capability expiry on the proxy token lookups --- src/runtime/engineBrokerCapabilities.test.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/runtime/engineBrokerCapabilities.test.ts b/src/runtime/engineBrokerCapabilities.test.ts index af9835f..b29fc54 100644 --- a/src/runtime/engineBrokerCapabilities.test.ts +++ b/src/runtime/engineBrokerCapabilities.test.ts @@ -13,3 +13,11 @@ test("proxy resolves scope from opaque token without caller identity", () => { const capabilities = new EngineBrokerCapabilities(); const token = capabilities.issue("agent-a", "turn-a"); assert.deepEqual(capabilities.authorizeToken(token), { agentId: "agent-a", turnId: "turn-a" }); }); + +test("the live proxy lookups refuse an expired capability", async () => { + const capabilities = new EngineBrokerCapabilities(); const token = capabilities.issue("agent-a", "turn-a", 20, 64); + assert.deepEqual(capabilities.inspectToken(token), { agentId: "agent-a", turnId: "turn-a" }); + await new Promise((resolve) => setTimeout(resolve, 40)); + assert.equal(capabilities.inspectToken(token), undefined); + assert.equal(capabilities.authorizeToken(token), undefined); +}); From b1823ec974d4a59584b813a562b3c0f0135d45b9 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 05:50:41 +0200 Subject: [PATCH 022/124] fix: lock each Grok turn to the ProfileApplied event accepted before its first model request --- src/runtime/grokEngineBroker.ts | 6 +- src/runtime/grokWorkerAttestation.test.ts | 2 +- src/runtime/grokWorkerAttestation.ts | 73 +++++++++++++++--- src/runtime/grokWorkerIsolationGuard.test.ts | 79 ++++++++++++++++++++ 4 files changed, 147 insertions(+), 13 deletions(-) create mode 100644 src/runtime/grokWorkerIsolationGuard.test.ts diff --git a/src/runtime/grokEngineBroker.ts b/src/runtime/grokEngineBroker.ts index bfcc3ec..98a8384 100644 --- a/src/runtime/grokEngineBroker.ts +++ b/src/runtime/grokEngineBroker.ts @@ -9,7 +9,7 @@ import { startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; import { acquireGrokBrokerRealmLease } from "./grokBrokerRealmLease.js"; import { grokBrokerWorkerConfigSha256 } from "./grokBrokerWorkerConfig.js"; import { parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; -import { GrokWorkerAttestationFailure,prepareGrokWorkerAttestation,verifyGrokWorkerAttestation } from "./grokWorkerAttestation.js"; +import { createGrokWorkerIsolationGuard,GrokWorkerAttestationFailure,prepareGrokWorkerAttestation } from "./grokWorkerAttestation.js"; export type GrokEngineBrokerRegistration = Readonly<{ agentId:string;slot:number;workerUid:number;workspace:string;profilePath:string;eventsPath:string;profileSha256:string }>; export type GrokEngineBroker = Awaited>; @@ -45,9 +45,9 @@ export async function startGrokEngineBroker(options: Readonly<{ grokCommand: str if (closed) throw new Error("engine broker unavailable"); const registration = registrations.get(agentId); if (registration === undefined) throw new Error("engine broker unavailable"); const turnId = createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"); const request = { version: "noopolis.daimon.engine-broker.v1", kind: "start_turn", requestId: randomUUID(), turnId, agentId, wakeId, prompt,mcpEndpoint } as const; const begun = await turns.begin(request); if (begun !== "start") { if (begun.replay.kind === "completed") return {text:begun.replay.text,workerPid:begun.replay.workerPid,workerUid:begun.replay.workerUid,workerStartTime:begun.replay.workerStartTime};const code=begun.replay.code==="auth_stale"||begun.replay.code==="cancelled"?begun.replay.code:"engine_failed";throw new EngineBrokerTurnFailure(code,begun.replay.diagnostic as NativeBrokerDiagnostic|undefined); } - const attestation={...registration,brokerGid:2100,configSha256};const isolation=await prepareGrokWorkerAttestation(attestation);proxy.registerIsolationGuard(turnId,()=>verifyGrokWorkerAttestation(attestation,isolation));const providerCapability = proxy.capabilities.issue(agentId, turnId);const mcpCapability=mcp.register(agentId,turnId,mcpEndpoint); const controller = new AbortController();const onAbort=()=>controller.abort();signal?.addEventListener("abort",onAbort,{once:true});if(signal?.aborted)controller.abort(); let resolve!:()=>void;const done=new Promise((value)=>{resolve=value;});active.set(turnId,{controller,done,resolve}); + const attestation={...registration,brokerGid:2100,configSha256};const isolation=await prepareGrokWorkerAttestation(attestation);const isolationGuard=createGrokWorkerIsolationGuard(attestation,isolation);proxy.registerIsolationGuard(turnId,isolationGuard);const providerCapability = proxy.capabilities.issue(agentId, turnId);const mcpCapability=mcp.register(agentId,turnId,mcpEndpoint); const controller = new AbortController();const onAbort=()=>controller.abort();signal?.addEventListener("abort",onAbort,{once:true});if(signal?.aborted)controller.abort(); let resolve!:()=>void;const done=new Promise((value)=>{resolve=value;});active.set(turnId,{controller,done,resolve}); let nativeDiagnostic:NativeBrokerDiagnostic|undefined,attested=false; - try { const result = await runNativeBrokerTurn(options.nativeClient, { slot: registration.slot, requestId: request.requestId, turnId, agentId, wakeId, prompt, providerCapability,mcpCapability }, controller.signal);nativeDiagnostic=result.diagnostic;if(result.workerUid!==registration.workerUid)throw new Error();await verifyGrokWorkerAttestation(attestation,isolation);attested=true; const decoded = decodeGrokHeadlessTurn(result.text);const text = decoded.text;const completed={ version: request.version, kind: "completed", requestId: request.requestId, turnId, text, workerPid: result.workerPid, workerUid: result.workerUid, workerStartTime: result.startTicks.toString() } as const; await finishBrokerTurnWithUsage(turns,request,completed,usageLedgerPath,decoded.usage,agentId,wakeId); return {text,workerPid:result.workerPid,workerUid:result.workerUid,workerStartTime:result.startTicks.toString()}; } + try { const result = await runNativeBrokerTurn(options.nativeClient, { slot: registration.slot, requestId: request.requestId, turnId, agentId, wakeId, prompt, providerCapability,mcpCapability }, controller.signal);nativeDiagnostic=result.diagnostic;if(result.workerUid!==registration.workerUid)throw new Error();await isolationGuard();attested=true; const decoded = decodeGrokHeadlessTurn(result.text);const text = decoded.text;const completed={ version: request.version, kind: "completed", requestId: request.requestId, turnId, text, workerPid: result.workerPid, workerUid: result.workerUid, workerStartTime: result.startTicks.toString() } as const; await finishBrokerTurnWithUsage(turns,request,completed,usageLedgerPath,decoded.usage,agentId,wakeId); return {text,workerPid:result.workerPid,workerUid:result.workerUid,workerStartTime:result.startTicks.toString()}; } catch(error) { const code=authority.isStale()?"auth_stale":controller.signal.aborted?"cancelled":"engine_failed";const diagnostic=error instanceof NativeBrokerTurnFailure?error.diagnostic:nativeDiagnostic&&!attested?{...nativeDiagnostic,status:"worker_failed" as const,stage:"attestation" as const,failureClass:error instanceof GrokWorkerAttestationFailure?error.failureClass:"profile_invalid" as const,profileApplied:false}:undefined;await turns.finish(request, { version: request.version, kind: "failed", requestId: request.requestId, turnId, code,...(diagnostic?{diagnostic}:{}) }); throw new EngineBrokerTurnFailure(code,diagnostic); } finally { signal?.removeEventListener("abort",onAbort);active.get(turnId)?.resolve(); active.delete(turnId); proxy.revokeIsolationGuard(turnId);proxy.capabilities.revoke(turnId);mcp.revoke(turnId); } }, diff --git a/src/runtime/grokWorkerAttestation.test.ts b/src/runtime/grokWorkerAttestation.test.ts index 7c1b30a..92920e3 100644 --- a/src/runtime/grokWorkerAttestation.test.ts +++ b/src/runtime/grokWorkerAttestation.test.ts @@ -102,7 +102,7 @@ const eventsFile = async (dir: string, initial = ""): Promise => { }; const watermark = async (file: string, denyPaths: readonly string[] = []): Promise => { const info = await stat(file); - return { dev: Number(info.dev), ino: Number(info.ino), size: Number(info.size), mtimeMs: Number(info.mtimeMs), denyPaths }; + return { dev: Number(info.dev), ino: Number(info.ino), size: Number(info.size), denyPaths }; }; const verify = (eventsPath: string, before: GrokWorkerAttestationSnapshot, brokerGid = self.gid): Promise => verifyGrokWorkerAttestation({ eventsPath, workerUid: self.uid, brokerGid, workspace }, before); diff --git a/src/runtime/grokWorkerAttestation.ts b/src/runtime/grokWorkerAttestation.ts index 70eae1b..adfa3f5 100644 --- a/src/runtime/grokWorkerAttestation.ts +++ b/src/runtime/grokWorkerAttestation.ts @@ -12,7 +12,7 @@ import { grokWorkerEventsPathFor } from "./grokWorkerSandboxProfile.js"; * `size` is pre-turn history and is never read back — a `ProfileApplied` down * there is a replay, not evidence about this turn. */ -export type GrokWorkerAttestationSnapshot=Readonly<{dev:number;ino:number;size:number;mtimeMs:number;denyPaths:readonly string[]}>; +export type GrokWorkerAttestationSnapshot=Readonly<{dev:number;ino:number;size:number;denyPaths:readonly string[]}>; type Snapshot=GrokWorkerAttestationSnapshot; export class GrokWorkerAttestationFailure extends Error { constructor(readonly failureClass:"profile_missing"|"profile_invalid"){super("Grok worker isolation attestation unavailable");} } /** @@ -61,14 +61,65 @@ export function parseGrokWorkerSandboxProfile(bytes:Uint8Array,profileSha256:str * 1.0.34), and a root-owned read-only worker home whose `config.toml` hashes to * the declared renderer output (`grokWorkerHomeAttestation.ts`). */ -export async function prepareGrokWorkerAttestation(input:Readonly<{profilePath:string;eventsPath:string;profileSha256:string;workerUid:number;brokerGid:number;configSha256:string}>):Promise{ +export async function prepareGrokWorkerAttestation(input:Readonly<{profilePath:string;eventsPath:string;profileSha256:string;workerUid:number;brokerGid:number;configSha256:string}>,profileOwner:Readonly<{uid:number;gid:number}>={uid:0,gid:0}):Promise{ if(input.eventsPath!==grokWorkerEventsPathFor(input.profilePath))throw new Error("Grok worker sandbox events must be read from $GROK_HOME/sessions/sandbox-events.jsonl"); - const profile=await secureOpen(input.profilePath,0,0,0o444,65_536);let bytes:Buffer|undefined;let denyPaths:readonly string[]=[];try{bytes=await profile.readFile();denyPaths=parseGrokWorkerSandboxProfile(bytes,input.profileSha256);}catch{throw new Error("Grok worker isolation attestation unavailable");}finally{bytes?.fill(0);await profile.close();} + const profile=await secureOpen(input.profilePath,profileOwner.uid,profileOwner.gid,0o444,65_536);let bytes:Buffer|undefined;let denyPaths:readonly string[]=[];try{bytes=await profile.readFile();denyPaths=parseGrokWorkerSandboxProfile(bytes,input.profileSha256);}catch{throw new Error("Grok worker isolation attestation unavailable");}finally{bytes?.fill(0);await profile.close();} await verifyGrokWorkerHome(path.dirname(input.profilePath),input.configSha256); - const events=await secureOpen(input.eventsPath,input.workerUid,input.brokerGid,0o640,16*1024*1024);try{const stat=await events.stat();return{dev:Number(stat.dev),ino:Number(stat.ino),size:Number(stat.size),mtimeMs:Number(stat.mtimeMs),denyPaths};}finally{await events.close();} + const events=await secureOpen(input.eventsPath,input.workerUid,input.brokerGid,0o640,16*1024*1024);try{const stat=await events.stat();return{dev:Number(stat.dev),ino:Number(stat.ino),size:Number(stat.size),denyPaths};}finally{await events.close();} } -export async function verifyGrokWorkerAttestation(input:Readonly<{eventsPath:string;workerUid:number;brokerGid:number;workspace:string}>,before:Snapshot):Promise{ - let handle:Awaited>;try{handle=await secureOpen(input.eventsPath,input.workerUid,input.brokerGid,0o640,16*1024*1024);}catch{throw new GrokWorkerAttestationFailure("profile_invalid");}let bytes:Buffer|undefined;try{const stat=await handle.stat();if(Number(stat.dev)!==before.dev||Number(stat.ino)!==before.ino)throw new GrokWorkerAttestationFailure("profile_invalid");if(Number(stat.size)<=before.size)throw new GrokWorkerAttestationFailure("profile_missing");bytes=Buffer.alloc(Number(stat.size)-before.size);const read=await handle.read(bytes,0,bytes.length,before.size);if(read.bytesRead!==bytes.length)throw new GrokWorkerAttestationFailure("profile_invalid");const after=await handle.stat();if(Number(after.size)!==Number(stat.size)||Number(after.mtimeMs)!==Number(stat.mtimeMs))throw new GrokWorkerAttestationFailure("profile_invalid");parseGrokWorkerProfileApplied(bytes,input.workspace,before.denyPaths);}catch(error){if(error instanceof GrokWorkerAttestationFailure)throw error;throw new GrokWorkerAttestationFailure("profile_invalid");}finally{bytes?.fill(0);await handle.close();} +/** + * The accepted `ProfileApplied` line of one turn, as an absolute byte range of + * the events file plus its digest. + */ +export type GrokWorkerAttestationLock={accepted?:Readonly<{offset:number;length:number;digest:string}>;refused?:GrokWorkerAttestationFailure["failureClass"]}; + +/** + * The per-turn isolation guard: every model request of a turn, and the + * post-turn check, go through the same lock. + * + * Why the first verification is trustworthy: the proxy awaits this guard before + * *every* upstream request, including the first, and the only worker-uid + * process that exists before request 1 is the Grok the launcher started under + * the pinned profile — tool children are created only after a model response. + * So the event accepted at request 1 was written before any tool child could + * write to the (worker-owned) events file. Later requests must find that exact + * line, byte for byte, at the same offset: a `ProfileApplied` appended later — + * which a tool child could forge — is never a substitute, and once a turn has + * been refused it stays refused. + */ +export function createGrokWorkerIsolationGuard(input:Parameters[0],before:Snapshot):()=>Promise{ + const lock:GrokWorkerAttestationLock={}; + return async()=>{ + if(lock.refused!==undefined)throw new GrokWorkerAttestationFailure(lock.refused); + try{await verifyGrokWorkerAttestation(input,before,lock);} + catch(error){const failure=error instanceof GrokWorkerAttestationFailure?error:new GrokWorkerAttestationFailure("profile_invalid");lock.refused=failure.failureClass;throw failure;} + }; +} +export async function verifyGrokWorkerAttestation(input:Readonly<{eventsPath:string;workerUid:number;brokerGid:number;workspace:string}>,before:Snapshot,lock?:GrokWorkerAttestationLock):Promise{ + let handle:Awaited>;try{handle=await secureOpen(input.eventsPath,input.workerUid,input.brokerGid,0o640,16*1024*1024);}catch{throw new GrokWorkerAttestationFailure("profile_invalid");} + let bytes:Buffer|undefined; + try{ + const stat=await handle.stat(); + if(Number(stat.dev)!==before.dev||Number(stat.ino)!==before.ino)throw new GrokWorkerAttestationFailure("profile_invalid"); + if(Number(stat.size)<=before.size)throw new GrokWorkerAttestationFailure("profile_missing"); + const accepted=lock?.accepted; + const start=accepted===undefined?before.size:before.size+accepted.offset; + const length=accepted===undefined?Number(stat.size)-before.size:accepted.length; + if(start+length>Number(stat.size))throw new GrokWorkerAttestationFailure("profile_invalid"); + bytes=Buffer.alloc(length); + const read=await handle.read(bytes,0,bytes.length,start); + if(read.bytesRead!==bytes.length)throw new GrokWorkerAttestationFailure("profile_invalid"); + // The file must not change while it is read: a concurrent writer could + // otherwise show this check bytes that no single state of the file held. + const after=await handle.stat(); + if(Number(after.size)!==Number(stat.size)||Number(after.mtimeMs)!==Number(stat.mtimeMs))throw new GrokWorkerAttestationFailure("profile_invalid"); + if(accepted!==undefined){ + if(createHash("sha256").update(bytes).digest("hex")!==accepted.digest)throw new GrokWorkerAttestationFailure("profile_invalid"); + return; + } + const event=locateGrokWorkerProfileApplied(bytes,input.workspace,before.denyPaths); + if(lock!==undefined)lock.accepted={offset:event.offset,length:event.length,digest:createHash("sha256").update(bytes.subarray(event.offset,event.offset+event.length)).digest("hex")}; + }catch(error){if(error instanceof GrokWorkerAttestationFailure)throw error;throw new GrokWorkerAttestationFailure("profile_invalid");}finally{bytes?.fill(0);await handle.close();} } /** * Requires one fully-conforming `ProfileApplied` event *somewhere* in the @@ -101,15 +152,19 @@ export async function verifyGrokWorkerAttestation(input:Readonly<{eventsPath:str * guard exists for: a Grok that came up without kernel enforcement, which * emits no conforming `ProfileApplied` at all. */ -export function parseGrokWorkerProfileApplied(bytes:Uint8Array,workspace:string,denyPaths:readonly string[]=[]):void{ +export function parseGrokWorkerProfileApplied(bytes:Uint8Array,workspace:string,denyPaths:readonly string[]=[]):void{locateGrokWorkerProfileApplied(bytes,workspace,denyPaths);} +/** Byte range (within `bytes`) of the first fully-conforming `ProfileApplied` line. */ +export function locateGrokWorkerProfileApplied(bytes:Uint8Array,workspace:string,denyPaths:readonly string[]=[]):Readonly<{offset:number;length:number}>{ const expected=JSON.stringify([...denyPaths].sort()); - for(const line of Buffer.from(bytes).toString("utf8").split("\n")){ + const buffer=Buffer.from(bytes.buffer,bytes.byteOffset,bytes.byteLength); + for(let offset=0;offset; try{const parsed=JSON.parse(line) as unknown;if(parsed===null||typeof parsed!=="object"||Array.isArray(parsed))continue;event=parsed as Record;}catch{continue;} if(event.event_type!=="ProfileApplied")continue; const observed=Array.isArray(event.deny_paths)?event.deny_paths.filter((entry):entry is string=>typeof entry==="string").sort():[]; - if(event.profile==="daimon-strict"&&event.enforced===true&&event.restrict_network===true&&event.platform==="linux/landlock"&&event.workspace===workspace&&JSON.stringify(observed)===expected)return; + if(event.profile==="daimon-strict"&&event.enforced===true&&event.restrict_network===true&&event.platform==="linux/landlock"&&event.workspace===workspace&&JSON.stringify(observed)===expected)return{offset:lineOffset,length:end-lineOffset}; } throw new Error("Grok worker isolation attestation unavailable"); } diff --git a/src/runtime/grokWorkerIsolationGuard.test.ts b/src/runtime/grokWorkerIsolationGuard.test.ts new file mode 100644 index 0000000..98fa729 --- /dev/null +++ b/src/runtime/grokWorkerIsolationGuard.test.ts @@ -0,0 +1,79 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { appendFile, chmod, link, mkdir, mkdtemp, open, readFile, rm, stat, truncate, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test, { mock } from "node:test"; + +import { + createGrokWorkerIsolationGuard, + GrokWorkerAttestationFailure, + prepareGrokWorkerAttestation, + verifyGrokWorkerAttestation, + type GrokWorkerAttestationSnapshot +} from "./grokWorkerAttestation.js"; + +const workspace = "/var/lib/daimon-workers/2200/workspace"; +const self = { uid: process.getuid?.() ?? 0, gid: process.getgid?.() ?? 0 }; +const applied = (overrides: Record = {}): string => + `${JSON.stringify({ event_type: "ProfileApplied", profile: "daimon-strict", enforced: true, restrict_network: true, platform: "linux/landlock", workspace, deny_paths: [], ...overrides })}\n`; +const violation = `${JSON.stringify({ event_type: "FsViolation", profile: "daimon-strict", operation: "read", target: "/run/paideia/context.json" })}\n`; + +const fixture = async (t: { after(fn: () => Promise): void }, initial = "") => { + const dir = await mkdtemp(path.join(os.tmpdir(), "daimon-guard-")); + t.after(() => rm(dir, { recursive: true, force: true })); + const file = path.join(dir, "sandbox-events.jsonl"); + await writeFile(file, initial); + await chmod(file, 0o640); + const info = await stat(file); + const before: GrokWorkerAttestationSnapshot = { dev: Number(info.dev), ino: Number(info.ino), size: Number(info.size), denyPaths: [] }; + const input = { eventsPath: file, workerUid: self.uid, brokerGid: self.gid, workspace }; + return { dir, file, before, input }; +}; +const refusal = async (run: Promise): Promise => { + try { await run; } catch (error) { + assert.ok(error instanceof GrokWorkerAttestationFailure, `expected a GrokWorkerAttestationFailure, got ${String(error)}`); + return error.failureClass; + } + throw new Error("expected the attestation to be refused"); +}; + +test("a turn refused at its first request stays refused after a conforming ProfileApplied is appended", async (t) => { + // Before request 1 only the launcher-started Grok can have written events; a + // line appended after it can come from a tool child and must never repair the turn. + const { file, before, input } = await fixture(t); + const guard = createGrokWorkerIsolationGuard(input, before); + await appendFile(file, applied({ enforced: false })); + assert.equal(await refusal(guard()), "profile_invalid"); + await appendFile(file, applied()); + assert.equal(await refusal(guard()), "profile_invalid"); + // Without the lock the same bytes would be accepted, so the refusal above is the lock's doing. + await verifyGrokWorkerAttestation(input, before); +}); + +test("later requests need the accepted event at the same offset, not any newly appended one", async (t) => { + const { file, before, input } = await fixture(t, "{\"event_type\":\"stale\"}\n"); + const guard = createGrokWorkerIsolationGuard(input, before); + await appendFile(file, `${violation}${applied()}`); + await guard(); + await appendFile(file, `${violation}${applied()}`); + await guard(); + + // Rewrite the accepted line in place (same length, different bytes) and append a fresh conforming one. + const bytes = await readFile(file, "utf8"); + const acceptedAt = bytes.indexOf(applied()); + const handle = await open(file, "r+"); + try { await handle.write(Buffer.from(applied({ workspace: workspace.replace("workspace", "workspacX") })), 0, undefined, acceptedAt); } finally { await handle.close(); } + await appendFile(file, applied()); + assert.equal(await refusal(guard()), "profile_invalid"); +}); + +test("a truncated accepted event region is refused on the next request", async (t) => { + const { file, before, input } = await fixture(t); + const guard = createGrokWorkerIsolationGuard(input, before); + await appendFile(file, applied()); + await guard(); + await truncate(file, 10); + await appendFile(file, `\n${applied()}`); + assert.equal(await refusal(guard()), "profile_invalid"); +}); From fdbc7a0ce52bbee0ecfef60626c884c0d50dff97 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 05:50:41 +0200 Subject: [PATCH 023/124] test: cover worker home, hard link and read stability checks in Grok attestation --- .../grokWorkerAttestationChecks.test.ts | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 src/runtime/grokWorkerAttestationChecks.test.ts diff --git a/src/runtime/grokWorkerAttestationChecks.test.ts b/src/runtime/grokWorkerAttestationChecks.test.ts new file mode 100644 index 0000000..8a6480e --- /dev/null +++ b/src/runtime/grokWorkerAttestationChecks.test.ts @@ -0,0 +1,77 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { appendFile, chmod, link, mkdir, mkdtemp, open, readFile, rm, stat, truncate, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test, { mock } from "node:test"; + +import { + GrokWorkerAttestationFailure, + prepareGrokWorkerAttestation, + verifyGrokWorkerAttestation, + type GrokWorkerAttestationSnapshot +} from "./grokWorkerAttestation.js"; + +const workspace = "/var/lib/daimon-workers/2200/workspace"; +const self = { uid: process.getuid?.() ?? 0, gid: process.getgid?.() ?? 0 }; +const applied = (overrides: Record = {}): string => + `${JSON.stringify({ event_type: "ProfileApplied", profile: "daimon-strict", enforced: true, restrict_network: true, platform: "linux/landlock", workspace, deny_paths: [], ...overrides })}\n`; +const violation = `${JSON.stringify({ event_type: "FsViolation", profile: "daimon-strict", operation: "read", target: "/run/paideia/context.json" })}\n`; + +const fixture = async (t: { after(fn: () => Promise): void }, initial = "") => { + const dir = await mkdtemp(path.join(os.tmpdir(), "daimon-guard-")); + t.after(() => rm(dir, { recursive: true, force: true })); + const file = path.join(dir, "sandbox-events.jsonl"); + await writeFile(file, initial); + await chmod(file, 0o640); + const info = await stat(file); + const before: GrokWorkerAttestationSnapshot = { dev: Number(info.dev), ino: Number(info.ino), size: Number(info.size), denyPaths: [] }; + const input = { eventsPath: file, workerUid: self.uid, brokerGid: self.gid, workspace }; + return { dir, file, before, input }; +}; +const refusal = async (run: Promise): Promise => { + try { await run; } catch (error) { + assert.ok(error instanceof GrokWorkerAttestationFailure, `expected a GrokWorkerAttestationFailure, got ${String(error)}`); + return error.failureClass; + } + throw new Error("expected the attestation to be refused"); +}; + +test("refuses an events file with a second hard link", async (t) => { + const { dir, file, before, input } = await fixture(t); + await appendFile(file, applied()); + await verifyGrokWorkerAttestation(input, before); + await link(file, path.join(dir, "second-name.jsonl")); + assert.equal(await refusal(verifyGrokWorkerAttestation(input, before)), "profile_invalid"); +}); + +test("refuses events that change while they are being read", async (t) => { + const { file, before, input } = await fixture(t); + await appendFile(file, applied()); + const probe = await open(file, "r"); + const prototype = Object.getPrototypeOf(probe) as { read: (...args: unknown[]) => Promise }; + await probe.close(); + const original = prototype.read; + const reading = mock.method(prototype, "read", async function (this: unknown, ...args: unknown[]) { + const result = await original.apply(this, args); + await appendFile(file, violation); + return result; + }); + try { assert.equal(await refusal(verifyGrokWorkerAttestation(input, before)), "profile_invalid"); } finally { reading.mock.restore(); } + assert.ok(reading.mock.callCount() >= 1); +}); + +test("prepare refuses a worker home that fails attestation even when profile and events are valid", async (t) => { + // Run as a non-root owner so the profile and events legs pass; the home leg + // (root-owned, read-only config) cannot, and must be what refuses. + const home = await mkdtemp(path.join(os.tmpdir(), "daimon-guard-home-")); + t.after(() => rm(home, { recursive: true, force: true })); + const profile = path.join(home, "sandbox.toml"); + const text = '[profiles.daimon-strict]\nextends = "strict"\nrestrict_network = true\ndeny = []\n'; + await writeFile(profile, text); await chmod(profile, 0o444); + await mkdir(path.join(home, "sessions")); + const events = path.join(home, "sessions", "sandbox-events.jsonl"); + await writeFile(events, ""); await chmod(events, 0o640); + const input = { profilePath: profile, eventsPath: events, profileSha256: createHash("sha256").update(text).digest("hex"), workerUid: self.uid, brokerGid: self.gid, configSha256: "0".repeat(64) }; + await assert.rejects(prepareGrokWorkerAttestation(input, { uid: self.uid, gid: Number((await stat(profile)).gid) }), /attestation unavailable/u); +}); From a78a9937d509d14d08ddaffa173ab39930cf28f2 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 05:50:56 +0200 Subject: [PATCH 024/124] test: prove the Grok isolation guard gates the first upstream call of a turn --- src/runtime/grokBrokerProxy.test.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index 0a935d5..6c541b6 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -67,3 +67,21 @@ test("the session-title sink is refused before capability, guard, credential, or assert.equal(calls, 1); } finally { await proxy.close(); } }); + +test("the isolation guard is awaited before the first upstream call, and a failing guard makes no upstream call", async () => { + const order: string[] = []; let upstreamCalls = 0; let fail = true; + const proxy = await startGrokBrokerProxy({ accessToken: async () => { order.push("credential"); return "provider-token"; }, markRejected: async () => undefined }, async () => { upstreamCalls++; order.push("upstream"); return { status: 200, headers: { "content-type": "application/json" }, body: Buffer.from("{}") }; }); + try { + const token = proxy.capabilities.issue("agent", "turn"); + proxy.registerIsolationGuard("turn", async () => { + order.push("guard-start"); await new Promise((resolve) => setTimeout(resolve, 30)); order.push("guard-end"); + if (fail) throw new Error("no enforcement evidence"); + }); + assert.equal(await post(proxy.port, token, leanBody()), 503); + assert.equal(upstreamCalls, 0); + assert.deepEqual(order, ["guard-start", "guard-end"]); + fail = false; order.length = 0; + assert.equal(await post(proxy.port, token, leanBody()), 200); + assert.deepEqual(order, ["guard-start", "guard-end", "credential", "upstream"]); + } finally { await proxy.close(); } +}); From ad67499fec421f9dcf4c82621a83a58647cfd4c3 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 05:51:28 +0200 Subject: [PATCH 025/124] fix: require the opened Grok worker config to be the inode attested by lstat --- src/runtime/grokWorkerHomeAttestation.test.ts | 34 +++++++++++++++++-- src/runtime/grokWorkerHomeAttestation.ts | 15 ++++---- 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/src/runtime/grokWorkerHomeAttestation.test.ts b/src/runtime/grokWorkerHomeAttestation.test.ts index 1c2eceb..22f591e 100644 --- a/src/runtime/grokWorkerHomeAttestation.test.ts +++ b/src/runtime/grokWorkerHomeAttestation.test.ts @@ -1,8 +1,8 @@ import assert from "node:assert/strict"; -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { chmod, mkdir, mkdtemp, open, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import test from "node:test"; +import test, { mock } from "node:test"; import { grokBrokerWorkerConfigSha256, renderGrokBrokerWorkerConfig } from "./grokBrokerWorkerConfig.js"; import { assertGrokWorkerConfigBytes, assertGrokWorkerHomeEntries, verifyGrokWorkerHome } from "./grokWorkerHomeAttestation.js"; @@ -60,3 +60,33 @@ test("accepts only the renderer's exact config bytes for the declared model poli `${renderGrokBrokerWorkerConfig(declared)}\n[mcp_servers.extra]\nurl = "http://127.0.0.1:1/mcp"\n` ]) assert.throws(() => assertGrokWorkerConfigBytes(Buffer.from(tampered), grokBrokerWorkerConfigSha256(declared)), /attestation unavailable/u); }); + +const ownHome = async (t: { after(fn: () => Promise): void }) => { + const home = await mkdtemp(path.join(os.tmpdir(), "daimon-grok-home-owned-")); + t.after(async () => { await chmod(home, 0o700); await rm(home, { recursive: true, force: true }); }); + await mkdir(path.join(home, "sessions")); + const declared = { model: "grok-4.6", reasoningEffort: "low" } as const; + for (const name of files) await writeFile(path.join(home, name), name === "config.toml" ? renderGrokBrokerWorkerConfig(declared) : "", { mode: 0o444 }); + await chmod(path.join(home, "sessions"), 0o1771); await chmod(home, 0o1771); + return { home, sha: grokBrokerWorkerConfigSha256(declared), uid: process.getuid?.() ?? 0 }; +}; + +test("attests a correctly laid out home whose config is the declared renderer output", async (t) => { + const { home, sha, uid } = await ownHome(t); + await verifyGrokWorkerHome(home, sha, uid); + await assert.rejects(verifyGrokWorkerHome(home, grokBrokerWorkerConfigSha256({ model: "grok-4.6", reasoningEffort: "high" }), uid), /attestation unavailable/u); +}); + +test("refuses a config.toml whose opened inode is not the one lstat saw", async (t) => { + const { home, sha, uid } = await ownHome(t); + const probe = await open(path.join(home, "config.toml"), "r"); + const prototype = Object.getPrototypeOf(probe) as { stat: (...args: unknown[]) => Promise<{ ino: number }> }; + await probe.close(); + const original = prototype.stat; + const swapped = mock.method(prototype, "stat", async function (this: unknown, ...args: unknown[]) { + const real = await original.apply(this, args); + return Object.assign(Object.create(Object.getPrototypeOf(real)), real, { ino: Number(real.ino) + 1 }); + }); + try { await assert.rejects(verifyGrokWorkerHome(home, sha, uid), /attestation unavailable/u); } finally { swapped.mock.restore(); } + await verifyGrokWorkerHome(home, sha, uid); +}); diff --git a/src/runtime/grokWorkerHomeAttestation.ts b/src/runtime/grokWorkerHomeAttestation.ts index f029757..47b15ba 100644 --- a/src/runtime/grokWorkerHomeAttestation.ts +++ b/src/runtime/grokWorkerHomeAttestation.ts @@ -5,7 +5,7 @@ import path from "node:path"; import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; -type Entry = Pick & Readonly<{ isFile(): boolean; isDirectory(): boolean; isSymbolicLink(): boolean }>; +type Entry = Pick & Partial> & Readonly<{ isFile(): boolean; isDirectory(): boolean; isSymbolicLink(): boolean }>; const HOME = GROK_ENGINE_BROKER.worker.home; /** @@ -22,14 +22,14 @@ const HOME = GROK_ENGINE_BROKER.worker.home; * * Pure so every refusal is testable without root. */ -export function assertGrokWorkerHomeEntries(entries: Readonly>): void { +export function assertGrokWorkerHomeEntries(entries: Readonly>, rootUid: number = HOME.directory.uid): void { const directory = (entry: Entry | undefined): boolean => - entry !== undefined && entry.isDirectory() && !entry.isSymbolicLink() && entry.uid === HOME.directory.uid + entry !== undefined && entry.isDirectory() && !entry.isSymbolicLink() && entry.uid === rootUid && (Number(entry.mode) & 0o002) === 0 && ((Number(entry.mode) & 0o020) === 0 || (Number(entry.mode) & 0o1000) !== 0); if (!directory(entries["."]) || !directory(entries[HOME.sessionsDirectory.relativePath])) throw unavailable(); for (const name of HOME.readOnlyFiles.names) { const entry = entries[name]; - if (entry === undefined || !entry.isFile() || entry.isSymbolicLink() || entry.uid !== HOME.readOnlyFiles.uid || entry.nlink !== 1 || (Number(entry.mode) & 0o7222) !== 0) throw unavailable(); + if (entry === undefined || !entry.isFile() || entry.isSymbolicLink() || entry.uid !== rootUid || entry.nlink !== 1 || (Number(entry.mode) & 0o7222) !== 0) throw unavailable(); } } @@ -42,18 +42,19 @@ export function assertGrokWorkerConfigBytes(bytes: Uint8Array, configSha256: str * Attests the worker home layout and that `config.toml` is exactly the * renderer's bytes for the declared model policy (`configSha256`). */ -export async function verifyGrokWorkerHome(grokHome: string, configSha256: string): Promise { +export async function verifyGrokWorkerHome(grokHome: string, configSha256: string, rootUid: number = HOME.directory.uid): Promise { const entries: Record = {}; for (const name of [".", HOME.sessionsDirectory.relativePath, ...HOME.readOnlyFiles.names]) { try { entries[name] = await lstat(path.join(grokHome, name)); } catch { entries[name] = undefined; } } - assertGrokWorkerHomeEntries(entries); + assertGrokWorkerHomeEntries(entries, rootUid); let handle: Awaited> | undefined; try { handle = await open(path.join(grokHome, "config.toml"), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); const opened = await handle.stat(); const before = entries["config.toml"]!; - if (!opened.isFile() || opened.size > 65_536 || opened.uid !== before.uid || opened.mode !== before.mode || opened.nlink !== 1) throw unavailable(); + // Same inode as the lstat above, as in `secureOpen`: a rename between the two cannot swap in other bytes. + if (!opened.isFile() || opened.size > 65_536 || opened.dev !== before.dev || opened.ino !== before.ino || opened.uid !== before.uid || opened.mode !== before.mode || opened.nlink !== 1) throw unavailable(); assertGrokWorkerConfigBytes(await handle.readFile(), configSha256); } catch { throw unavailable(); } finally { await handle?.close().catch(() => undefined); } } From f4fe7f3253779cfb505af5d3ceed991242a4ad67 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 05:52:04 +0200 Subject: [PATCH 026/124] fix: refuse __proto__ members and extra tool members in Grok broker requests --- src/runtime/grokBrokerProxyRequest.test.ts | 36 ++++++++++++++++++++++ src/runtime/grokBrokerProxyRequest.ts | 11 +++++-- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/runtime/grokBrokerProxyRequest.test.ts b/src/runtime/grokBrokerProxyRequest.test.ts index 576ff0a..7cdcdfb 100644 --- a/src/runtime/grokBrokerProxyRequest.test.ts +++ b/src/runtime/grokBrokerProxyRequest.test.ts @@ -71,3 +71,39 @@ test("proxy refuses top-level members a lean Grok 1.0.34 worker never sends", () const { caps, input } = request(leanBody({ stream_options: { include_usage: true } })); assert.equal(Buffer.from(authorizeGrokBrokerProxyRequest(input, caps, "real-bearer").body).toString("utf8"), Buffer.from(leanBody({ stream_options: { include_usage: true } })).toString("utf8")); }); + +test("nested duplicate keys in tools and messages are forwarded only as the parsed values", () => { + const tools = leanTools.map((name) => `{"type":"function","function":{"name":${JSON.stringify(name === "read_file" ? "write" : name)},"name":${JSON.stringify(name)},"parameters":{"type":"object"}}}`).join(","); + const raw = `{"model":"grok-4.6","reasoning_effort":"low","stream":true,"messages":[{"role":"system","role":"user","content":"a","content":"b"}],"tools":[${tools}]}`; + const { caps, input } = request(Buffer.from(raw)); + const forwarded = Buffer.from(authorizeGrokBrokerProxyRequest(input, caps, "real-bearer").body).toString("utf8"); + assert.equal(forwarded, JSON.stringify(JSON.parse(raw))); + assert.doesNotMatch(forwarded, /"write"|"system"|"content":"a"/u); + assert.equal(forwarded.split('"role"').length, 2); + // The reverse order puts the forbidden name last, so the parsed (and gated) value is refused. + const hostile = raw.replace('"name":"write","name":"read_file"', '"name":"read_file","name":"write"'); + const second = request(Buffer.from(hostile)); + assert.throws(() => authorizeGrokBrokerProxyRequest(second.input, second.caps, "real-bearer"), /rejected/u); +}); + +test("__proto__ members are refused anywhere in the body", () => { + const lean = JSON.stringify(leanTools.map(tool)); + const base = `"model":"grok-4.6","reasoning_effort":"low","stream":true,"messages":[]`; + for (const raw of [ + `{${base},"tools":${lean},"__proto__":{"tools":[]}}`, + `{${base},"tools":${lean.replace('{"type":"function"', '{"__proto__":{"type":"function"},"type":"function"')}}`, + `{${base},"tools":${lean.replace('"parameters":{"type":"object"}', '"parameters":{"type":"object","__proto__":{"x":1}}')}}`, + `{"model":"grok-4.6","reasoning_effort":"low","stream":true,"messages":[{"role":"user","content":"hi","__proto__":{"role":"system"}}],"tools":${lean}}` + ]) { + const { caps, input } = request(Buffer.from(raw)); + assert.throws(() => authorizeGrokBrokerProxyRequest(input, caps, "real-bearer"), /rejected/u, raw.slice(0, 120)); + } +}); + +test("tool entries carry only the members a lean worker sends", () => { + for (const extra of [{ strict: true }, { function: { name: "read_file", parameters: {}, x: 1 } }]) { + const tools = leanTools.map((name) => name === "read_file" ? { ...tool(name), ...extra } : tool(name)); + const { caps, input } = request(leanBody({ tools })); + assert.throws(() => authorizeGrokBrokerProxyRequest(input, caps, "real-bearer"), /rejected/u, JSON.stringify(extra)); + } +}); diff --git a/src/runtime/grokBrokerProxyRequest.ts b/src/runtime/grokBrokerProxyRequest.ts index 24dcbc6..79956ac 100644 --- a/src/runtime/grokBrokerProxyRequest.ts +++ b/src/runtime/grokBrokerProxyRequest.ts @@ -30,7 +30,9 @@ export function authorizeGrokBrokerProxyRequest(input: GrokBrokerProxyInput, cap const clientVersion = input.headers["x-grok-client-version"]; if (clientVersion !== GROK_ENGINE_BROKER.grokCliVersion) throw new Error("broker proxy request rejected"); let parsed: Record; - try { parsed = JSON.parse(Buffer.from(input.body).toString("utf8")) as Record; } catch { throw new Error("broker proxy request rejected"); } + // `__proto__` is refused at any depth: JSON.parse makes it an ordinary own + // member, but an upstream JavaScript parser may treat it as a prototype. + try { parsed = JSON.parse(Buffer.from(input.body).toString("utf8"), (key, value: unknown) => { if (key === "__proto__") throw new Error(); return value; }) as Record; } catch { throw new Error("broker proxy request rejected"); } if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed) || parsed.stream !== true || !Array.isArray(parsed.messages)) throw new Error("broker proxy request rejected"); if (parsed.model !== declared.model || parsed.reasoning_effort !== declared.reasoningEffort) throw new Error("broker proxy request rejected"); if (!exactLeanTools(parsed.tools)) throw new Error("broker proxy request rejected"); @@ -43,15 +45,18 @@ export function authorizeGrokBrokerProxyRequest(input: GrokBrokerProxyInput, cap /** Top-level members of a Grok 1.0.34 lean worker chat-completions body (live stub capture). */ const LEAN_BODY_MEMBERS: ReadonlySet = new Set(["messages", "model", "reasoning_effort", "stream", "stream_options", "tools"]); +/** Members of each lean tool `function` entry (live stub capture). */ +const LEAN_FUNCTION_MEMBERS: ReadonlySet = new Set(["description", "name", "parameters"]); const validStreamOptions = (value: unknown): boolean => value === undefined || (value !== null && typeof value === "object" && !Array.isArray(value) && Object.keys(value).every((key) => key === "include_usage") && typeof (value as { include_usage?: unknown }).include_usage === "boolean"); export function exactLeanTools(tools: unknown): boolean { if (!Array.isArray(tools) || tools.length !== GROK_WORKER_VISIBLE_TOOLS.length) return false; const names = tools.map((tool) => { - if (tool === null || typeof tool !== "object" || (tool as { type?: unknown }).type !== "function") return undefined; + if (tool === null || typeof tool !== "object" || Array.isArray(tool) || (tool as { type?: unknown }).type !== "function" || Object.keys(tool).some((key) => key !== "type" && key !== "function")) return undefined; const fn = (tool as { function?: unknown }).function; - return fn !== null && typeof fn === "object" ? (fn as { name?: unknown }).name : undefined; + if (fn === null || typeof fn !== "object" || Array.isArray(fn) || Object.keys(fn).some((key) => !LEAN_FUNCTION_MEMBERS.has(key))) return undefined; + return (fn as { name?: unknown }).name; }); return JSON.stringify([...names].sort()) === JSON.stringify(GROK_WORKER_VISIBLE_TOOLS); } From b55f678ba93fc03290aee44a92ec21362daf27f4 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 05:53:18 +0200 Subject: [PATCH 027/124] test: prove a Grok worker that fails at exec runs nothing --- .../engineBrokerLauncherIntegrationMain.inc | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/runtime/native/engineBrokerLauncherIntegrationMain.inc b/src/runtime/native/engineBrokerLauncherIntegrationMain.inc index f5ea29b..1191e2d 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationMain.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationMain.inc @@ -1,3 +1,52 @@ +/* A worker that fails before or at exec must not run anything: the launcher + child exits instead. The registered "executable" is a #! script; fd 5 is + close-on-exec, so execveat(AT_EMPTY_PATH) cannot run it (a script needs + /dev/fd/5 after exec) and the turn fails at the exec stage. */ +static void exec_failure_case(struct dbl_registration r) { + const char *marker = "/tmp/exec-failure-script-ran"; + unlink(marker); + check(rename("/usr/local/bin/grok", "/usr/local/bin/grok.fixture") == 0, + "exec failure fixture move"); + int script = open("/usr/local/bin/grok", O_CREAT | O_EXCL | O_WRONLY, 0755); + const char body[] = "#!/bin/sh\ntouch /tmp/exec-failure-script-ran\n"; + check(script >= 0 && write(script, body, sizeof(body) - 1) == + (ssize_t)(sizeof(body) - 1), + "exec failure script"); + close(script); + digest("/usr/local/bin/grok", r.executable_sha256); + int f = open(DBL_REGISTRY, O_TRUNC | O_WRONLY, 0600); + check(f >= 0 && write(f, &r, sizeof(r)) == sizeof(r), "exec failure registry"); + close(f); + pid_t child = fork(); + if (!child) { + setgid(DBL_BROKER_UID); + setuid(DBL_BROKER_UID); + int s = connect_socket(), p = sealed("prompt"), + c = sealed_bundle("provider.Exec-1", "mcp.Exec-2"); + struct dbl_request q = request(); + struct dbl_result result; + send_request(s, &q, p, c); + _exit(read_all(s, &result, sizeof(result)) && + result.status == DBL_STATUS_PRELAUNCH_FAILED && + result.stage == DBL_STAGE_EXEC && + result.failure_class == DBL_FAILURE_EXEC && + result.worker_pid == 0 + ? 0 + : 1); + } + int status; + waitpid(child, &status, 0); + check(WIFEXITED(status) && WEXITSTATUS(status) == 0, "exec failure result"); + sleep(1); + check(access(marker, F_OK) != 0, "exec failure ran nothing"); + unlink("/usr/local/bin/grok"); + check(rename("/usr/local/bin/grok.fixture", "/usr/local/bin/grok") == 0, + "exec failure fixture restore"); + digest("/usr/local/bin/grok", r.executable_sha256); + f = open(DBL_REGISTRY, O_TRUNC | O_WRONLY, 0600); + check(f >= 0 && write(f, &r, sizeof(r)) == sizeof(r), "registry restore"); + close(f); +} int main(void) { setbuf(stdout, NULL); alarm(40); @@ -61,6 +110,9 @@ int main(void) { } int status; waitpid(org, &status, 0); + check(WIFEXITED(status) && WEXITSTATUS(status) == 0, "org cases"); + exec_failure_case(r); + puts("native-stage exec failure complete"); kill(broker, SIGKILL); waitpid(broker, 0, 0); check(WIFEXITED(status) && WEXITSTATUS(status) == 0, "org cases"); From 85bb35745a7584aff765eecbb0c50a0e62f16bb3 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 05:54:19 +0200 Subject: [PATCH 028/124] test: drop unused imports from Grok attestation tests --- src/runtime/grokWorkerAttestationChecks.test.ts | 2 +- src/runtime/grokWorkerIsolationGuard.test.ts | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/runtime/grokWorkerAttestationChecks.test.ts b/src/runtime/grokWorkerAttestationChecks.test.ts index 8a6480e..694391c 100644 --- a/src/runtime/grokWorkerAttestationChecks.test.ts +++ b/src/runtime/grokWorkerAttestationChecks.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; -import { appendFile, chmod, link, mkdir, mkdtemp, open, readFile, rm, stat, truncate, writeFile } from "node:fs/promises"; +import { appendFile, chmod, link, mkdir, mkdtemp, open, rm, stat, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import test, { mock } from "node:test"; diff --git a/src/runtime/grokWorkerIsolationGuard.test.ts b/src/runtime/grokWorkerIsolationGuard.test.ts index 98fa729..cae9d81 100644 --- a/src/runtime/grokWorkerIsolationGuard.test.ts +++ b/src/runtime/grokWorkerIsolationGuard.test.ts @@ -1,14 +1,12 @@ import assert from "node:assert/strict"; -import { createHash } from "node:crypto"; -import { appendFile, chmod, link, mkdir, mkdtemp, open, readFile, rm, stat, truncate, writeFile } from "node:fs/promises"; +import { appendFile, chmod, mkdtemp, open, readFile, rm, stat, truncate, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import test, { mock } from "node:test"; +import test from "node:test"; import { createGrokWorkerIsolationGuard, GrokWorkerAttestationFailure, - prepareGrokWorkerAttestation, verifyGrokWorkerAttestation, type GrokWorkerAttestationSnapshot } from "./grokWorkerAttestation.js"; From 99571606577c81c62408f595610dbddf64c5319c Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:13:24 +0200 Subject: [PATCH 029/124] feat: add engine-neutral wake limit env names with Codex aliases --- src/pi/cliSession.ts | 34 +++---------------- src/pi/engineWakeLimits.test.ts | 20 ++++++++++++ src/pi/engineWakeLimits.ts | 58 +++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 30 deletions(-) create mode 100644 src/pi/engineWakeLimits.test.ts create mode 100644 src/pi/engineWakeLimits.ts diff --git a/src/pi/cliSession.ts b/src/pi/cliSession.ts index 07b964c..49f8af8 100644 --- a/src/pi/cliSession.ts +++ b/src/pi/cliSession.ts @@ -59,36 +59,10 @@ export type CliEngineKind = "agy" | "codex" | "grok"; */ export const AGY_MAX_TOOL_TURNS = 16; -/** - * Codex's per-wake bounds, and the one place they are decided. - * - * `maxToolTurns` mediates only daimon-MCP tool calls; Codex's own shell - * (`exec_command`) is never routed through that gate, so a single Codex turn - * previously had no ceiling at all — one production wake ran 23:32→23:42 - * (unbounded wall clock) making 51 shell calls. Codex's `--json` stream - * reports token usage exactly once, on `turn.completed` — there is no - * incremental total to watch mid-turn (verified against a live multi-tool-call - * turn: `item.completed` fires once per tool call, but usage is reported only - * on the single terminal `turn.completed`) — so the token ceiling is the best - * bound obtainable from that wire shape: it converts an over-budget turn into - * an explicit, killed, named failure instead of a silent success, and the - * wall-clock timeout is what actually interrupts a runaway turn in progress. - */ -export const DEFAULT_CODEX_WAKE_TIMEOUT_MS = 240_000; -export const DEFAULT_CODEX_WAKE_TOKEN_CEILING = 300_000; -export const DAIMON_CODEX_WAKE_TIMEOUT_MS_ENV = "DAIMON_CODEX_WAKE_TIMEOUT_MS"; -export const DAIMON_CODEX_WAKE_TOKEN_CEILING_ENV = "DAIMON_CODEX_WAKE_TOKEN_CEILING"; - -const positiveInteger = (value: string | undefined, fallback: number, name: string): number => { - if (value === undefined) return fallback; - const parsed = Number(value); - if (!Number.isSafeInteger(parsed) || parsed <= 0) throw new Error(`${name} must be a positive integer`); - return parsed; -}; -export const resolveCodexWakeTimeoutMs = (environment: NodeJS.ProcessEnv = process.env): number => - positiveInteger(environment[DAIMON_CODEX_WAKE_TIMEOUT_MS_ENV], DEFAULT_CODEX_WAKE_TIMEOUT_MS, DAIMON_CODEX_WAKE_TIMEOUT_MS_ENV); -export const resolveCodexWakeTokenCeiling = (environment: NodeJS.ProcessEnv = process.env): number => - positiveInteger(environment[DAIMON_CODEX_WAKE_TOKEN_CEILING_ENV], DEFAULT_CODEX_WAKE_TOKEN_CEILING, DAIMON_CODEX_WAKE_TOKEN_CEILING_ENV); +export { + DAIMON_CODEX_WAKE_TIMEOUT_MS_ENV, DAIMON_CODEX_WAKE_TOKEN_CEILING_ENV, DAIMON_ENGINE_WAKE_TIMEOUT_MS_ENV, DAIMON_ENGINE_WAKE_TOKEN_CEILING_ENV, + DEFAULT_CODEX_WAKE_TIMEOUT_MS, DEFAULT_CODEX_WAKE_TOKEN_CEILING, resolveCodexWakeTimeoutMs, resolveCodexWakeTokenCeiling, resolveEngineWakeLimitOverrides +} from "./engineWakeLimits.js"; export type CliEngineOptions = { readonly commandArgs?: readonly string[]; diff --git a/src/pi/engineWakeLimits.test.ts b/src/pi/engineWakeLimits.test.ts new file mode 100644 index 0000000..e514af1 --- /dev/null +++ b/src/pi/engineWakeLimits.test.ts @@ -0,0 +1,20 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { resolveCodexWakeTimeoutMs, resolveCodexWakeTokenCeiling, resolveEngineWakeLimitOverrides } from "./engineWakeLimits.js"; + +test("engine-neutral wake bounds drive Codex, with the Codex names kept as aliases", () => { + assert.equal(resolveCodexWakeTimeoutMs({}), 240_000); + assert.equal(resolveCodexWakeTokenCeiling({}), 300_000); + assert.equal(resolveCodexWakeTimeoutMs({ DAIMON_ENGINE_WAKE_TIMEOUT_MS: "5000" }), 5_000); + assert.equal(resolveCodexWakeTokenCeiling({ DAIMON_CODEX_WAKE_TOKEN_CEILING: "7000" }), 7_000); + assert.equal(resolveCodexWakeTokenCeiling({ DAIMON_ENGINE_WAKE_TOKEN_CEILING: "7000", DAIMON_CODEX_WAKE_TOKEN_CEILING: "7000" }), 7_000); + assert.throws(() => resolveCodexWakeTokenCeiling({ DAIMON_ENGINE_WAKE_TOKEN_CEILING: "7000", DAIMON_CODEX_WAKE_TOKEN_CEILING: "8000" }), /disagree/u); + assert.throws(() => resolveCodexWakeTimeoutMs({ DAIMON_ENGINE_WAKE_TIMEOUT_MS: "0" }), /positive integer/u); +}); + +test("the broker receives only the bounds an operator actually set, as lowering limits", () => { + assert.equal(resolveEngineWakeLimitOverrides({}), undefined); + assert.deepEqual(resolveEngineWakeLimitOverrides({ DAIMON_ENGINE_WAKE_TOKEN_CEILING: "400000" }), { maxTokens: 400_000 }); + assert.deepEqual(resolveEngineWakeLimitOverrides({ DAIMON_CODEX_WAKE_TIMEOUT_MS: "480000", DAIMON_ENGINE_WAKE_TOKEN_CEILING: "1" }), { timeoutMs: 480_000, maxTokens: 1 }); +}); diff --git a/src/pi/engineWakeLimits.ts b/src/pi/engineWakeLimits.ts new file mode 100644 index 0000000..a4a0299 --- /dev/null +++ b/src/pi/engineWakeLimits.ts @@ -0,0 +1,58 @@ +/** + * Per-wake engine bounds, and the one place they are decided. + * + * `maxToolTurns` mediates only daimon-MCP tool calls; Codex's own shell + * (`exec_command`) is never routed through that gate, so a single Codex turn + * previously had no ceiling at all — one production wake ran 23:32→23:42 + * (unbounded wall clock) making 51 shell calls. Codex's `--json` stream + * reports token usage exactly once, on `turn.completed` — there is no + * incremental total to watch mid-turn (verified against a live multi-tool-call + * turn: `item.completed` fires once per tool call, but usage is reported only + * on the single terminal `turn.completed`) — so the token ceiling is the best + * bound obtainable from that wire shape: it converts an over-budget turn into + * an explicit, killed, named failure instead of a silent success, and the + * wall-clock timeout is what actually interrupts a runaway turn in progress. + * + * The names are engine-neutral: `DAIMON_ENGINE_WAKE_TIMEOUT_MS` and + * `DAIMON_ENGINE_WAKE_TOKEN_CEILING` bound Codex locally and are passed to the + * Grok broker as the wake's *lowering* limits (the broker refuses a value above + * its registration). The `DAIMON_CODEX_*` names remain aliases; setting both + * names of one bound to different values is refused rather than guessed. + */ +export const DEFAULT_CODEX_WAKE_TIMEOUT_MS = 240_000; +export const DEFAULT_CODEX_WAKE_TOKEN_CEILING = 300_000; +export const DAIMON_ENGINE_WAKE_TIMEOUT_MS_ENV = "DAIMON_ENGINE_WAKE_TIMEOUT_MS"; +export const DAIMON_ENGINE_WAKE_TOKEN_CEILING_ENV = "DAIMON_ENGINE_WAKE_TOKEN_CEILING"; +export const DAIMON_CODEX_WAKE_TIMEOUT_MS_ENV = "DAIMON_CODEX_WAKE_TIMEOUT_MS"; +export const DAIMON_CODEX_WAKE_TOKEN_CEILING_ENV = "DAIMON_CODEX_WAKE_TOKEN_CEILING"; + +const positiveInteger = (value: string, name: string): number => { + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed <= 0) throw new Error(`${name} must be a positive integer`); + return parsed; +}; + +const declared = (environment: NodeJS.ProcessEnv, neutral: string, alias: string): number | undefined => { + const primary = environment[neutral], legacy = environment[alias]; + const value = primary === undefined ? undefined : positiveInteger(primary, neutral); + const aliased = legacy === undefined ? undefined : positiveInteger(legacy, alias); + if (value !== undefined && aliased !== undefined && value !== aliased) throw new Error(`${neutral} and ${alias} disagree; set one`); + return value ?? aliased; +}; + +export const resolveCodexWakeTimeoutMs = (environment: NodeJS.ProcessEnv = process.env): number => + declared(environment, DAIMON_ENGINE_WAKE_TIMEOUT_MS_ENV, DAIMON_CODEX_WAKE_TIMEOUT_MS_ENV) ?? DEFAULT_CODEX_WAKE_TIMEOUT_MS; +export const resolveCodexWakeTokenCeiling = (environment: NodeJS.ProcessEnv = process.env): number => + declared(environment, DAIMON_ENGINE_WAKE_TOKEN_CEILING_ENV, DAIMON_CODEX_WAKE_TOKEN_CEILING_ENV) ?? DEFAULT_CODEX_WAKE_TOKEN_CEILING; + +/** + * The limits a wake asks a broker to lower to: only the bounds the operator + * actually set, never the Codex defaults (a broker registration's declared + * limits already are the defaults there). + */ +export const resolveEngineWakeLimitOverrides = (environment: NodeJS.ProcessEnv = process.env): Readonly<{ timeoutMs?: number; maxTokens?: number }> | undefined => { + const timeoutMs = declared(environment, DAIMON_ENGINE_WAKE_TIMEOUT_MS_ENV, DAIMON_CODEX_WAKE_TIMEOUT_MS_ENV); + const maxTokens = declared(environment, DAIMON_ENGINE_WAKE_TOKEN_CEILING_ENV, DAIMON_CODEX_WAKE_TOKEN_CEILING_ENV); + if (timeoutMs === undefined && maxTokens === undefined) return undefined; + return { ...(timeoutMs === undefined ? {} : { timeoutMs }), ...(maxTokens === undefined ? {} : { maxTokens }) }; +}; From 7e285428475d680329ea6ccd7c5db4939bcb4442 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:13:24 +0200 Subject: [PATCH 030/124] feat: record per-request timestamps, Grok stream usage and turn-keyed usage rows --- src/pi/codexRolloutUsage.test.ts | 8 ++ src/pi/codexRolloutUsage.ts | 34 ++++- src/pi/fixtures/README.md | 20 +++ .../grok-1.0.34-streaming-two-requests.jsonl | 5 + src/pi/grokStreamUsage.test.ts | 40 ++++++ src/pi/grokStreamUsage.ts | 52 ++++++++ src/runtime/engineBrokerTurnAccounting.ts | 125 ++++++++++++++++++ src/runtime/turnRequestLedger.test.ts | 14 ++ src/runtime/turnRequestLedger.ts | 52 +++++++- src/runtime/turnUsageLedger.test.ts | 13 +- src/runtime/turnUsageLedger.ts | 34 ++++- src/runtime/wakeFuse.test.ts | 12 ++ src/runtime/wakeFuse.ts | 5 +- 13 files changed, 408 insertions(+), 6 deletions(-) create mode 100644 src/pi/fixtures/grok-1.0.34-streaming-two-requests.jsonl create mode 100644 src/pi/grokStreamUsage.test.ts create mode 100644 src/pi/grokStreamUsage.ts create mode 100644 src/runtime/engineBrokerTurnAccounting.ts diff --git a/src/pi/codexRolloutUsage.test.ts b/src/pi/codexRolloutUsage.test.ts index e7fc317..ea41272 100644 --- a/src/pi/codexRolloutUsage.test.ts +++ b/src/pi/codexRolloutUsage.test.ts @@ -46,6 +46,14 @@ test("a real multi-request rollout yields one row per model request, cached and [288, 228, 50_292] ]); assert.deepEqual(requests.map((request) => request.cacheWrite), [0, 0, 0, 0]); + // End is the usage frame; start is the first non-usage frame after the previous + // request's usage frame, else the previous request's end. + assert.deepEqual(requests.map((request) => [request.startedAt, request.endedAt]), [ + ["2026-09-05T01:32:00.678Z", "2026-09-05T01:32:19.183Z"], + ["2026-09-05T01:34:00.679Z", "2026-09-05T01:52:22.056Z"], + ["2026-09-05T01:52:22.056Z", "2026-09-05T01:52:37.604Z"], + ["2026-09-05T01:52:37.604Z", "2026-09-05T01:52:51.112Z"] + ]); }); test("reasoning tokens, which the per-wake ledger drops entirely, survive per request", async () => { diff --git a/src/pi/codexRolloutUsage.ts b/src/pi/codexRolloutUsage.ts index b803f5e..0c872bf 100644 --- a/src/pi/codexRolloutUsage.ts +++ b/src/pi/codexRolloutUsage.ts @@ -44,6 +44,17 @@ export type CodexRequestUsage = Readonly<{ output: number; reasoning: number; total: number; + /** + * When the request began and ended, from the rollout's own frame + * `timestamp`s. `endedAt` is the usage frame's timestamp (Codex appends it at + * `response.completed`); `startedAt` is the first non-usage frame after the + * previous request's usage frame — the tool output or turn context that + * triggers the next request — falling back to the previous request's end. + * Either is absent when the frames carry no valid timestamp: a wake-end stamp + * substituted here would be indistinguishable from a measured one. + */ + startedAt?: string; + endedAt?: string; }>; /** @@ -177,17 +188,20 @@ export const parseCodexRolloutRequests = (text: string, threadId: string): reado const requests: CodexRequestUsage[] = []; const fallback: CodexRequestUsage[] = []; let previousFallback = ""; + const clocks = { record: requestClock(), fallback: requestClock() }; for (const line of text.split("\n")) { if (line.trim().length === 0) continue; let frame: unknown; try { frame = JSON.parse(line); } catch { continue; } if (!isRecord(frame)) continue; const block = usageBlock(frame, threadId); + const usageFrame = frame.type === "token_usage_record" || (frame.type === "event_msg" && isRecord(frame.payload) && frame.payload.type === "token_count"); + if (!usageFrame) { if (frame.type !== "session_meta") { clocks.record.observe(frame.timestamp); clocks.fallback.observe(frame.timestamp); } continue; } if (block === undefined) continue; if (frame.type === "token_usage_record") { const decoded = decodeRequestUsage(block.usage, requests.length); if (decoded === undefined) return []; - requests.push(decoded); + requests.push({ ...decoded, ...clocks.record.close(frame.timestamp) }); continue; } // `token_count` is NOT one frame per request: the captured fixture carries @@ -201,7 +215,7 @@ export const parseCodexRolloutRequests = (text: string, threadId: string): reado previousFallback = serialized; const decoded = decodeRequestUsage(block.usage, fallback.length); if (decoded === undefined) return []; - fallback.push(decoded); + fallback.push({ ...decoded, ...clocks.fallback.close(frame.timestamp) }); } // A Codex version that emits both shapes emits `token_usage_record` once per // request, so the richer one wins outright rather than being merged into a @@ -209,6 +223,22 @@ export const parseCodexRolloutRequests = (text: string, threadId: string): reado return requests.length > 0 ? requests : fallback; }; +const timestampOf = (value: unknown): string | undefined => + typeof value === "string" && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,9})?Z$/u.test(value) && !Number.isNaN(Date.parse(value)) ? value : undefined; + +/** Tracks one request stream's start/end stamps; see {@link CodexRequestUsage.startedAt}. */ +const requestClock = () => { + let start: string | undefined, previousEnd: string | undefined; + return { + observe(value: unknown): void { start ??= timestampOf(value); }, + close(value: unknown): { startedAt?: string; endedAt?: string } { + const endedAt = timestampOf(value), startedAt = start ?? previousEnd; + start = undefined; previousEnd = endedAt; + return { ...(startedAt === undefined ? {} : { startedAt }), ...(endedAt === undefined ? {} : { endedAt }) }; + } + }; +}; + /** * Read one turn's per-request usage. Never throws. * diff --git a/src/pi/fixtures/README.md b/src/pi/fixtures/README.md index f158a6d..0cf75ed 100644 --- a/src/pi/fixtures/README.md +++ b/src/pi/fixtures/README.md @@ -117,3 +117,23 @@ the previous block and why `token_usage_record` wins outright when both exist. The four requests also show exactly the shape the study predicted, in one wake: fresh input 15,742 → 248 → 4,276 → 10,068 against a context that only grows from 34,686 to 50,004 — most of every request after the first is cache-read replay. + + +# Grok 1.0.34 per-request stream fixture + +`grok-1.0.34-streaming-two-requests.jsonl` is a real two-request turn captured on +2026-09-17 from `grok 1.0.34` (macOS arm64) with the lean worker flags and +`--output-format streaming-messages-json` (P0 host matrix cell c14: one MCP +`use_tool` call, then the answer). Sanitization before commit: the capturing +scratchpad `cwd` was replaced with `/workspace`; every frame is otherwise +verbatim. + +It pins what `grokStreamUsage.ts` reads and the broker meters per request: + + assistant.message.id one request per distinct id + assistant.message.usage that request's own four buckets + result.modelUsage keys "grok-4.6-build" for grok-4.6 + +The two per-request totals (2,775 + 2,810) sum exactly to the terminal +`result.usage` (5,585), which is why a failed turn's frames are trusted as its +partial usage. diff --git a/src/pi/fixtures/grok-1.0.34-streaming-two-requests.jsonl b/src/pi/fixtures/grok-1.0.34-streaming-two-requests.jsonl new file mode 100644 index 0000000..3eaabd0 --- /dev/null +++ b/src/pi/fixtures/grok-1.0.34-streaming-two-requests.jsonl @@ -0,0 +1,5 @@ +{"type":"system","subtype":"init","session_id":"01a0ad21-a90f-7f71-8054-93fdb4334d6a","apiKeySource":"oauth","model":"grok-4.6","cwd":"/workspace","permissionMode":"bypassPermissions","tools":["run_terminal_command","read_file","list_dir","grep","search_tool","use_tool"],"slash_commands":["compact","always-approve","context","session-info","feedback"],"mcp_servers":[{"name":"probe","status":"connected"}],"skills":[],"uuid":"880b6dc2-0488-4678-9ec8-04b2c2fe9efd"} +{"type":"assistant","message":{"id":"msg_0","type":"message","role":"assistant","model":"grok-4.6","content":[{"type":"thinking","thinking":"The user wants me to use moltnet_read on room \"conference\" to get a code word. The instructions say I can call probe__moltnet_read directly with use_tool.","signature":"tl43Ieb3gy7AEz+YdNJWSx3rN7HTKi9cJKhytQwzBfudDDWXQ6UEWS4JxMYQZJRrKQeBXNtXluNa1UHRr5C3OMGlpHodcziT9fqhMn5tgr0xjUjCRq1/KL0fdIGj+tU+8sHeAKtHBCMC2bFFkbtdm4BrWR0kLcHzQKbla9tof1RpzzgFtwyIx3fsVCmp/AYHHNrm0BNZvQpfbb9M1yyN4AD0d7E0b1VZru3uMx9N47RtFoH7uWQ"},{"type":"text","text":"I'll read the conference room now."},{"type":"tool_use","id":"call-9582489e-555b-4f65-81ba-8a46439283ce-0","name":"use_tool","input":{"tool_name":"probe__moltnet_read","tool_input":{"room":"conference"}}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":2568,"output_tokens":79,"cache_read_input_tokens":128,"cache_creation_input_tokens":0}},"parent_tool_use_id":null,"session_id":"01a0ad21-a90f-7f71-8054-93fdb4334d6a","uuid":"6c8748c9-2ce6-437b-a0d2-24a01ca23b3b"} +{"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"call-9582489e-555b-4f65-81ba-8a46439283ce-0","content":"{\"type\":\"MCP\",\"tool_name\":\"moltnet_read\",\"server_name\":\"probe\",\"output\":{\"OkayOutput\":\"Brass: the code word is TANGERINE-7.\"}}","is_error":false}]},"parent_tool_use_id":null,"session_id":"01a0ad21-a90f-7f71-8054-93fdb4334d6a","uuid":"fcd581d3-b43f-4aa0-a9d7-9c25862b4bfc"} +{"type":"assistant","message":{"id":"msg_1","type":"message","role":"assistant","model":"grok-4.6","content":[{"type":"thinking","thinking":"Got the code word. Answer briefly.","signature":"KcaYjJIHN4fzMKF4FdR0695QoQUKP7Byln+CLEgdhkDCFs3y0wBDWWYCWDkmvXSZnjMYqtfHn79Tgkif73U"},{"type":"text","text":"TANGERINE-7"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":109,"output_tokens":13,"cache_read_input_tokens":2688,"cache_creation_input_tokens":0}},"parent_tool_use_id":null,"session_id":"01a0ad21-a90f-7f71-8054-93fdb4334d6a","uuid":"b1ddbf87-b115-4ee9-b773-0d81f65a8ef7"} +{"type":"result","subtype":"success","is_error":false,"duration_ms":3300,"duration_api_ms":3118,"num_turns":2,"result":"TANGERINE-7","stop_reason":"end_turn","total_cost_usd":0.00248676,"usage":{"input_tokens":2677,"output_tokens":92,"cache_read_input_tokens":2816,"cache_creation_input_tokens":0,"server_tool_use":{"web_search_requests":0}},"modelUsage":{"grok-4.6-build":{"inputTokens":2677,"outputTokens":92,"cacheReadInputTokens":2816,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.00248676}},"session_id":"01a0ad21-a90f-7f71-8054-93fdb4334d6a","uuid":"0dbd02b1-4721-4c52-8416-25bf2455bb8a"} diff --git a/src/pi/grokStreamUsage.test.ts b/src/pi/grokStreamUsage.test.ts new file mode 100644 index 0000000..37942fb --- /dev/null +++ b/src/pi/grokStreamUsage.test.ts @@ -0,0 +1,40 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +import { decodeGrokHeadlessTurn } from "./grokHeadlessResult.js"; +import { decodeGrokStreamUsage } from "./grokStreamUsage.js"; + +const fixture = (): Promise => readFile(fileURLToPath(new URL("./fixtures/grok-1.0.34-streaming-two-requests.jsonl", import.meta.url)), "utf8"); + +test("a real 1.0.34 two-request stream yields one row per request, summing exactly to the terminal result", async () => { + const output = await fixture(); + const stream = decodeGrokStreamUsage(output); + assert.deepEqual(stream.requests, [ + { index: 0, input: 2_568, cacheRead: 128, cacheWrite: 0, output: 79, total: 2_775 }, + { index: 1, input: 109, cacheRead: 2_688, cacheWrite: 0, output: 13, total: 2_810 } + ]); + assert.equal(stream.sessionId, "01a0ad21-a90f-7f71-8054-93fdb4334d6a"); + assert.deepEqual(stream.reportedModels, ["grok-4.6-build"]); + const terminal = decodeGrokHeadlessTurn(output).usage!; + assert.equal(stream.requests.reduce((sum, request) => sum + request.total, 0), terminal.total); +}); + +test("a malformed per-request usage block discards every request instead of reporting part of the turn", async () => { + const lines = (await fixture()).split("\n"); + const index = lines.findIndex((line) => line.includes('"msg_1"')); + lines[index] = lines[index]!.replace('"input_tokens":109', '"input_tokens":"109"'); + assert.deepEqual(decodeGrokStreamUsage(lines.join("\n")).requests, []); +}); + +test("frames repeating one message id are one request, and a torn line is skipped", async () => { + const lines = (await fixture()).split("\n").filter((line) => line.length > 0); + const repeated = lines.find((line) => line.includes('"msg_0"'))!; + const stream = decodeGrokStreamUsage([...lines.slice(0, 2), repeated, ...lines.slice(2), '{"type":"assist'].join("\n")); + assert.equal(stream.requests.length, 2); +}); + +test("the captured fixture carries no capturing machine's environment", async () => { + assert.doesNotMatch(await fixture(), /\/Users\/|\/private\/|scratchpad|\/home\//u); +}); diff --git a/src/pi/grokStreamUsage.ts b/src/pi/grokStreamUsage.ts new file mode 100644 index 0000000..79b4aef --- /dev/null +++ b/src/pi/grokStreamUsage.ts @@ -0,0 +1,52 @@ +/** + * Per-request token accounting read off a Grok `streaming-messages-json` + * stream. + * + * Grok 1.0.34 puts the usage of each model request on that request's + * top-level `assistant` frame (`message.usage`, the same four disjoint + * Messages API buckets as the terminal `result.usage`), before any tool result + * of that request, and the per-request frames sum exactly to the terminal + * result (P0 host matrix). That makes the stream usable in two places the + * terminal frame is not: a turn that failed before its `result` frame, and a + * per-request ledger row. + * + * Never throws, and never invents a number: a usage block that is present but + * does not decode discards *all* requests, because one fabricated zero is + * byte-identical to a measured one. + */ +export type GrokRequestUsage = Readonly<{ index: number; input: number; cacheRead: number; cacheWrite: number; output: number; total: number }>; +export type GrokStreamUsage = Readonly<{ requests: readonly GrokRequestUsage[]; sessionId?: string; reportedModels: readonly string[] }>; + +type JsonRecord = Readonly>; +const isRecord = (value: unknown): value is JsonRecord => typeof value === "object" && value !== null && !Array.isArray(value); +const tokenCount = (value: unknown): number | undefined => typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : undefined; +const SESSION_ID = /^[A-Za-z0-9][A-Za-z0-9-]{0,63}$/u; +const MODEL_KEY = /^[a-z0-9][a-z0-9.-]{0,63}$/u; + +const decodeUsage = (usage: unknown): Omit | undefined => { + if (!isRecord(usage)) return undefined; + const input = tokenCount(usage.input_tokens), output = tokenCount(usage.output_tokens), cacheRead = tokenCount(usage.cache_read_input_tokens), cacheWrite = tokenCount(usage.cache_creation_input_tokens); + if (input === undefined || output === undefined || cacheRead === undefined || cacheWrite === undefined) return undefined; + return { input, cacheRead, cacheWrite, output, total: input + cacheRead + cacheWrite + output }; +}; + +export const decodeGrokStreamUsage = (output: string): GrokStreamUsage => { + const byMessage = new Map>(); + const reportedModels = new Set(); + let sessionId: string | undefined, corrupt = false; + for (const line of output.split(/\r?\n/u)) { + if (line.trim().length === 0) continue; + let event: unknown; + try { event = JSON.parse(line); } catch { continue; } + if (!isRecord(event)) continue; + if (typeof event.session_id === "string" && SESSION_ID.test(event.session_id)) sessionId ??= event.session_id; + if (event.type === "result" && isRecord(event.modelUsage)) for (const key of Object.keys(event.modelUsage)) reportedModels.add(MODEL_KEY.test(key) ? key : "invalid"); + if (event.type !== "assistant" || event.parent_tool_use_id !== null || !isRecord(event.message) || event.message.usage === undefined) continue; + const decoded = decodeUsage(event.message.usage); + if (decoded === undefined || typeof event.message.id !== "string") { corrupt = true; continue; } + // One request can surface as more than one frame of the same message; it is one request. + byMessage.set(event.message.id, decoded); + } + const requests = corrupt ? [] : [...byMessage.values()].map((usage, index) => Object.freeze({ index, ...usage })); + return { requests, ...(sessionId === undefined ? {} : { sessionId }), reportedModels: [...reportedModels].sort() }; +}; diff --git a/src/runtime/engineBrokerTurnAccounting.ts b/src/runtime/engineBrokerTurnAccounting.ts new file mode 100644 index 0000000..951c4f4 --- /dev/null +++ b/src/runtime/engineBrokerTurnAccounting.ts @@ -0,0 +1,125 @@ +import { GROK_BROKER_MODELS, GROK_WORKER_MAX_TURNS } from "../contracts/grokWorkerContract.js"; +import type { GrokBrokerModel } from "./grokBrokerModelPolicy.js"; + +/** + * Numeric-only accounting the Grok engine broker seals beside every terminal + * turn response, and the per-turn limits it enforces. + * + * The broker is the single writer of this data (turn registry record, control + * response, usage ledger row). Nothing engine-controlled and non-numeric is + * persisted: `model` is a member of the closed declared list, never the + * provider's own string, and `limitReason` is a closed vocabulary. + * + * Buckets are disjoint and `total = input + cacheRead + cacheWrite + output`. + * `reasoning` is reported only when the source separates it; it is already + * inside `output` and never added to `total`. + */ +export type EngineBrokerTurnUsage = Readonly<{ input: number; cacheRead: number; cacheWrite: number; output: number; total: number; reasoning?: number }>; +export const ENGINE_BROKER_LIMIT_REASONS = ["tokens", "requests", "timeout", "none"] as const; +export type EngineBrokerLimitReason = (typeof ENGINE_BROKER_LIMIT_REASONS)[number]; +export type EngineBrokerTurnLimits = Readonly<{ maxRequests: number; maxTokens: number; timeoutMs: number }>; +export type EngineBrokerTurnLimitOverrides = Readonly>; +export type EngineBrokerTurnAccounting = Readonly<{ + outcome: "completed" | "failed"; + usage: EngineBrokerTurnUsage | null; + model: GrokBrokerModel; + requests: number; + limitReason: EngineBrokerLimitReason; +}>; + +/** + * Bounds every declared limit must sit inside. `maxRequests` stays at or below + * the launcher's compiled `--max-turns` backstop, so the broker ceiling is the + * one that fires first. + */ +export const ENGINE_BROKER_LIMIT_BOUNDS = Object.freeze({ + maxRequests: [1, GROK_WORKER_MAX_TURNS], + maxTokens: [1, 10_000_000], + timeoutMs: [1_000, 3_600_000] +} as const); + +/** What a v1 `service.json` registration gets; equal to the Codex per-wake defaults. */ +export const DEFAULT_GROK_BROKER_TURN_LIMITS: EngineBrokerTurnLimits = Object.freeze({ maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }); + +const LIMIT_KEYS = ["maxRequests", "maxTokens", "timeoutMs"] as const; +type JsonRecord = Record; +const plain = (value: unknown): value is JsonRecord => + value !== null && typeof value === "object" && !Array.isArray(value) && (Object.getPrototypeOf(value) === Object.prototype || Object.getPrototypeOf(value) === null); +const count = (value: unknown): value is number => typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +const invalid = (label: string): TypeError => new TypeError(`invalid ${label}`); + +const limitValue = (key: (typeof LIMIT_KEYS)[number], value: unknown, label: string): number => { + const [minimum, maximum] = ENGINE_BROKER_LIMIT_BOUNDS[key]; + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < minimum || value > maximum) throw invalid(label); + return value; +}; + +/** Exactly `{maxRequests, maxTokens, timeoutMs}`, each inside its bound. */ +export function parseEngineBrokerTurnLimits(value: unknown, label = "engine broker turn limits"): EngineBrokerTurnLimits { + if (!plain(value) || Object.keys(value).length !== LIMIT_KEYS.length || LIMIT_KEYS.some((key) => !Object.hasOwn(value, key))) throw invalid(label); + return Object.freeze({ maxRequests: limitValue("maxRequests", value.maxRequests, label), maxTokens: limitValue("maxTokens", value.maxTokens, label), timeoutMs: limitValue("timeoutMs", value.timeoutMs, label) }); +} + +/** A non-empty subset of the limit keys, each inside its bound. */ +export function parseEngineBrokerTurnLimitOverrides(value: unknown, label = "engine broker turn limits"): EngineBrokerTurnLimitOverrides { + if (!plain(value) || Object.keys(value).length === 0 || Object.keys(value).some((key) => !(LIMIT_KEYS as readonly string[]).includes(key))) throw invalid(label); + const result: Partial> = {}; + for (const key of LIMIT_KEYS) if (Object.hasOwn(value, key)) result[key] = limitValue(key, value[key], label); + return Object.freeze(result); +} + +/** + * The limits one turn runs under: the registration's, lowered by the wake. + * A wake can never raise a declared limit; asking to is refused rather than + * clamped, so a misconfigured caller learns it instead of silently getting less. + */ +export function lowerEngineBrokerTurnLimits(declared: EngineBrokerTurnLimits, overrides: EngineBrokerTurnLimitOverrides | undefined): EngineBrokerTurnLimits { + if (overrides === undefined) return declared; + for (const key of LIMIT_KEYS) { + const requested = overrides[key]; + if (requested !== undefined && requested > declared[key]) throw new RangeError(`engine broker turn limit ${key} may only be lowered`); + } + return Object.freeze({ maxRequests: overrides.maxRequests ?? declared.maxRequests, maxTokens: overrides.maxTokens ?? declared.maxTokens, timeoutMs: overrides.timeoutMs ?? declared.timeoutMs }); +} + +/** Four disjoint buckets plus an optional reasoning split; the total invariant is re-checked. */ +export function parseEngineBrokerTurnUsage(value: unknown, label = "engine broker turn usage"): EngineBrokerTurnUsage { + const required = ["input", "cacheRead", "cacheWrite", "output", "total"]; + if (!plain(value)) throw invalid(label); + const keys = Object.keys(value); + if (required.some((key) => !Object.hasOwn(value, key)) || keys.some((key) => !required.includes(key) && key !== "reasoning")) throw invalid(label); + if (![value.input, value.cacheRead, value.cacheWrite, value.output, value.total].every(count)) throw invalid(label); + const usage = value as { input: number; cacheRead: number; cacheWrite: number; output: number; total: number; reasoning?: unknown }; + if (usage.total !== usage.input + usage.cacheRead + usage.cacheWrite + usage.output) throw invalid(label); + if (usage.reasoning !== undefined && (!count(usage.reasoning) || usage.reasoning > usage.output)) throw invalid(label); + return Object.freeze({ input: usage.input, cacheRead: usage.cacheRead, cacheWrite: usage.cacheWrite, output: usage.output, total: usage.total, ...(usage.reasoning === undefined ? {} : { reasoning: usage.reasoning as number }) }); +} + +export const sumEngineBrokerTurnUsage = (items: readonly EngineBrokerTurnUsage[]): EngineBrokerTurnUsage | null => { + if (items.length === 0) return null; + const sum = (pick: (usage: EngineBrokerTurnUsage) => number): number => items.reduce((total, usage) => total + pick(usage), 0); + const reasoning = items.every((usage) => usage.reasoning !== undefined) ? { reasoning: sum((usage) => usage.reasoning ?? 0) } : {}; + return Object.freeze({ input: sum((usage) => usage.input), cacheRead: sum((usage) => usage.cacheRead), cacheWrite: sum((usage) => usage.cacheWrite), output: sum((usage) => usage.output), total: sum((usage) => usage.total), ...reasoning }); +}; + +/** Validates the accounting members of a v2 terminal response against its kind. */ +export function parseEngineBrokerTurnAccounting(value: JsonRecord, kind: "completed" | "failed"): EngineBrokerTurnAccounting { + if (value.outcome !== kind) throw invalid("broker frame"); + if (!(GROK_BROKER_MODELS as readonly unknown[]).includes(value.model)) throw invalid("broker frame"); + if (!count(value.requests) || value.requests > 1_024) throw invalid("broker frame"); + if (!(ENGINE_BROKER_LIMIT_REASONS as readonly unknown[]).includes(value.limitReason)) throw invalid("broker frame"); + if (kind === "completed" && value.limitReason !== "none") throw invalid("broker frame"); + let usage: EngineBrokerTurnUsage | null = null; + if (value.usage !== null) { try { usage = parseEngineBrokerTurnUsage(value.usage); } catch { throw invalid("broker frame"); } } + return { outcome: kind, usage, model: value.model as GrokBrokerModel, requests: value.requests, limitReason: value.limitReason as EngineBrokerLimitReason }; +} + +/** + * Maps a provider-reported model key onto the declared closed-list model. + * + * Grok 1.0.34 reports `grok-4.6` usage under `grok-4.6-build` (P0). The exact + * declared id and its `-build` alias are accepted; anything else is a + * different model and yields `undefined`. + */ +export const mapGrokReportedModel = (reported: string, declared: GrokBrokerModel): GrokBrokerModel | undefined => + reported === declared || reported === `${declared}-build` ? declared : undefined; diff --git a/src/runtime/turnRequestLedger.test.ts b/src/runtime/turnRequestLedger.test.ts index 954c069..5944103 100644 --- a/src/runtime/turnRequestLedger.test.ts +++ b/src/runtime/turnRequestLedger.test.ts @@ -96,9 +96,23 @@ test("a real rollout reaches the stream end to end, one line per request", async assert.equal(rows.length, 4); assert.equal(rows.every((row) => row.thread === FIXTURE_THREAD && row.wake === "wake-1" && row.requests === 4), true); assert.deepEqual(rows.map((row) => row.fresh_input), [15_742, 248, 4_276, 10_068]); + // Each request carries its own rollout-frame interval, not the wake's append time. + // Mutation guard: stamping every row with `at` collapses these to one value. + assert.deepEqual(rows.map((row) => [row.started_at, row.ended_at]), [ + ["2026-09-05T01:32:00.678Z", "2026-09-05T01:32:19.183Z"], + ["2026-09-05T01:34:00.679Z", "2026-09-05T01:52:22.056Z"], + ["2026-09-05T01:52:22.056Z", "2026-09-05T01:52:37.604Z"], + ["2026-09-05T01:52:37.604Z", "2026-09-05T01:52:51.112Z"] + ]); }); }); +test("a request without measured timestamps carries none rather than the wake end", () => { + const [row] = renderTurnRequestLines({ agent: "a", wake: "w", thread: FIXTURE_THREAD, at: "2026-09-05T02:00:00.000Z", requests: [request({ startedAt: "not-a-time" })] }).split("\n").filter((line) => line.length > 0).map((line) => JSON.parse(line)); + assert.equal("started_at" in row, false); + assert.equal("ended_at" in row, false); +}); + test("a missing or malformed rollout writes nothing and still resolves", async () => { await withDirectory(async (directory) => { const home = path.join(directory, "codex-home"); diff --git a/src/runtime/turnRequestLedger.ts b/src/runtime/turnRequestLedger.ts index 21d5fbb..a13bfdf 100644 --- a/src/runtime/turnRequestLedger.ts +++ b/src/runtime/turnRequestLedger.ts @@ -2,6 +2,7 @@ import { constants } from "node:fs"; import { open, rename, stat } from "node:fs/promises"; import { readCodexRolloutRequests, type CodexRequestUsage } from "../pi/codexRolloutUsage.js"; +import type { GrokBrokerModel } from "./grokBrokerModelPolicy.js"; import { TURN_USAGE_LEDGER, TURN_USAGE_MAX_IDENTIFIER_CHARS, TURN_USAGE_ROTATE_BYTES } from "./turnUsageLedger.js"; /** @@ -91,10 +92,59 @@ export const renderTurnRequestLines = (entry: TurnRequestEntry): string => { cache_write: request.cacheWrite, output: request.output, reasoning: request.reasoning, - total: request.total + total: request.total, + ...requestClockFields(request) })}\n`).join(""); }; +const TIMESTAMP = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,9})?Z$/u; +/** Per-request `started_at`/`ended_at`, each only when it was measured; `at` stays the append time. */ +const requestClockFields = (request: Readonly<{ startedAt?: string; endedAt?: string }>): Record => ({ + ...(request.startedAt !== undefined && TIMESTAMP.test(request.startedAt) ? { started_at: request.startedAt } : {}), + ...(request.endedAt !== undefined && TIMESTAMP.test(request.endedAt) ? { ended_at: request.endedAt } : {}) +}); + +/** One Grok broker model request: usage from the worker stream, timing from the proxy. */ +export type GrokTurnRequest = Readonly<{ index: number; input: number; cacheRead: number; cacheWrite: number; output: number; total: number; startedAt?: string; endedAt?: string }>; +export type GrokTurnRequestEntry = Readonly<{ agent: string; wake: string; turn: string; session?: string; model: GrokBrokerModel; requests: readonly GrokTurnRequest[]; at?: string }>; + +/** + * Grok rows share the Codex row's field meaning: `input` is the whole prompt + * side the request replayed (`input_tokens + cache_read`), `cached_input` the + * cache read, `fresh_input` the uncached remainder. Grok does not separate + * reasoning tokens, so `reasoning` is absent rather than zero. `turn` is the + * broker idempotency key and `thread` the Grok session id when the stream + * named one. + */ +export const renderGrokTurnRequestLines = (entry: GrokTurnRequestEntry): string => { + const at = entry.at ?? new Date().toISOString(); + return entry.requests.map((request) => `${JSON.stringify({ + v: TURN_REQUEST_LEDGER_VERSION, + agent: bounded(entry.agent), + wake: bounded(entry.wake), + engine: "grok", + at, + turn: entry.turn, + ...(entry.session === undefined ? {} : { thread: bounded(entry.session) }), + model: entry.model, + request: request.index, + requests: entry.requests.length, + input: request.input + request.cacheRead, + cached_input: request.cacheRead, + fresh_input: request.input, + cache_write: request.cacheWrite, + output: request.output, + total: request.total, + ...requestClockFields(request) + })}\n`).join(""); +}; + +/** Advisory and never rejects, like {@link recordTurnRequests}; an empty turn writes nothing. */ +export const recordGrokTurnRequests = async (file: string, entry: GrokTurnRequestEntry): Promise => { + if (entry.requests.length === 0) return false; + try { await rotate(file); await appendLines(file, renderGrokTurnRequestLines(entry)); return true; } catch { return false; } +}; + const rotate = async (file: string): Promise => { let size: number; try { size = (await stat(file)).size; } catch { return; } diff --git a/src/runtime/turnUsageLedger.test.ts b/src/runtime/turnUsageLedger.test.ts index e6be905..7ddefad 100644 --- a/src/runtime/turnUsageLedger.test.ts +++ b/src/runtime/turnUsageLedger.test.ts @@ -15,6 +15,7 @@ import { TURN_USAGE_LEDGER_VERSION, TURN_USAGE_MAX_IDENTIFIER_CHARS, TURN_USAGE_ROTATE_BYTES, + dedupeTurnUsageRows, type TurnUsageEntry } from "./turnUsageLedger.js"; @@ -148,7 +149,7 @@ test("a failed wake's row carries the outcome and a reason from the closed vocab assert.equal(failed.reason, "token_ceiling"); assert.equal(failed.total, measurement.total, "the numbers are the ones the engine reported, not the outcome's"); - assert.deepEqual([...TURN_USAGE_FAILURE_REASONS], ["token_ceiling", "wake_timeout", "output_limit", "engine_exit", "turn_rejected", "unknown"]); + assert.deepEqual([...TURN_USAGE_FAILURE_REASONS], ["token_ceiling", "request_ceiling", "wake_timeout", "output_limit", "engine_exit", "turn_rejected", "unknown"]); for (const reason of TURN_USAGE_FAILURE_REASONS) { assert.equal(JSON.parse(renderTurnUsageLine(entry({ outcome: { status: "failed", reason } }))).reason, reason); } @@ -183,3 +184,13 @@ test("the outcome survives an append and a read back of the ledger file", async for (const record of written) assert.equal(record.v, TURN_USAGE_LEDGER_VERSION, "an added field, not a version bump"); }); }); + +test("broker rows carry the turn key, closed limit reason and closed model, and readers dedupe on the turn", () => { + const turn = "c".repeat(64); + const row = JSON.parse(renderTurnUsageLine(entry({ turn, limitReason: "requests", model: "grok-4.6", outcome: { status: "failed", reason: "request_ceiling" } }))); + assert.deepEqual([row.turn, row.limit_reason, row.model, row.reason], [turn, "requests", "grok-4.6", "request_ceiling"]); + assert.equal(row.total, row.input + row.cache_read + row.cache_write + row.output); + const forged = JSON.parse(renderTurnUsageLine(entry({ turn: "not-a-turn", limitReason: "budget" as never, model: "gpt-5" as never }))); + assert.deepEqual([forged.turn, forged.limit_reason, forged.model], [undefined, undefined, undefined]); + assert.deepEqual(dedupeTurnUsageRows([{ turn, total: 1 }, { total: 2 }, { turn, total: 3 }, { total: 4 }]), [{ turn, total: 1 }, { total: 2 }, { total: 4 }]); +}); diff --git a/src/runtime/turnUsageLedger.ts b/src/runtime/turnUsageLedger.ts index 5ca205a..44e7621 100644 --- a/src/runtime/turnUsageLedger.ts +++ b/src/runtime/turnUsageLedger.ts @@ -1,6 +1,10 @@ import { constants } from "node:fs"; import { open, rename, stat } from "node:fs/promises"; +import { GROK_BROKER_MODELS } from "../contracts/grokWorkerContract.js"; +import { ENGINE_BROKER_LIMIT_REASONS, type EngineBrokerLimitReason } from "./engineBrokerTurnAccounting.js"; +import type { GrokBrokerModel } from "./grokBrokerModelPolicy.js"; + /** * Append-only per-turn token accounting for one container. * @@ -81,6 +85,7 @@ export const TURN_USAGE_OUTCOMES = ["completed", "failed"] as const; /** * - `token_ceiling` — the turn's own reported usage crossed the per-wake ceiling. + * - `request_ceiling` — the broker refused a model request past the turn's request limit. * - `wake_timeout` — the wall-clock bound fired before the child finished. * - `output_limit` — the retained-output bound was exceeded. * - `engine_exit` — the child exited non-zero (or died) after reporting usage. @@ -89,6 +94,7 @@ export const TURN_USAGE_OUTCOMES = ["completed", "failed"] as const; */ export const TURN_USAGE_FAILURE_REASONS = [ "token_ceiling", + "request_ceiling", "wake_timeout", "output_limit", "engine_exit", @@ -110,6 +116,15 @@ export type TurnUsageEntry = Readonly<{ usage: TurnUsageMeasurement; at?: string; outcome?: TurnUsageOutcome; + /** + * Broker rows only. `turn` is the idempotency key readers dedupe on (the + * broker turn id, a sha256 hex); `limitReason` and `model` come from the + * closed broker vocabularies and are dropped rather than written through + * when they are not members. + */ + turn?: string; + limitReason?: EngineBrokerLimitReason; + model?: GrokBrokerModel; }>; /** @@ -166,9 +181,26 @@ export const renderTurnUsageLine = (entry: TurnUsageEntry): string => `${JSON.st calls: entry.usage.calls, notional_usd: entry.usage.notionalUsd, complete: entry.usage.complete, - ...outcomeFields(entry.outcome) + ...outcomeFields(entry.outcome), + ...brokerFields(entry) })}\n`; +const brokerFields = (entry: TurnUsageEntry): Record => ({ + ...(entry.turn !== undefined && /^[a-f0-9]{64}$/u.test(entry.turn) ? { turn: entry.turn } : {}), + ...(entry.limitReason !== undefined && ENGINE_BROKER_LIMIT_REASONS.includes(entry.limitReason) ? { limit_reason: entry.limitReason } : {}), + ...(entry.model !== undefined && (GROK_BROKER_MODELS as readonly string[]).includes(entry.model) ? { model: entry.model } : {}) +}); + +/** + * Collapse rows that share a `turn` key to the first one, keeping rows without + * a key untouched. Every reader that sums the ledger applies this, because a + * turn key exists precisely so a re-appended turn can never be counted twice. + */ +export const dedupeTurnUsageRows = >(rows: readonly T[]): T[] => { + const seen = new Set(); + return rows.filter((row) => { if (typeof row.turn !== "string") return true; if (seen.has(row.turn)) return false; seen.add(row.turn); return true; }); +}; + const rotate = async (file: string): Promise => { let size: number; try { size = (await stat(file)).size; } catch { return; } diff --git a/src/runtime/wakeFuse.test.ts b/src/runtime/wakeFuse.test.ts index 9c6bf50..a526063 100644 --- a/src/runtime/wakeFuse.test.ts +++ b/src/runtime/wakeFuse.test.ts @@ -229,3 +229,15 @@ test("DAIMON_WAKE_FUSE=off never touches the usage ledger, missing or not", asyn const fuse = await WakeFuse.open({ organizationKey: "org", environment: environment(directory, { DAIMON_WAKE_FUSE: "off" }) }); assert.deepEqual(await fuse.admit("alpha", "one"), { state: "admitted" }); })); + +test("usage rows sharing a broker turn key count once toward the token ceiling", async () => await withDirectory(async (directory) => { + const turn = "b".repeat(64); + // 600 + 600 would trip a 1000-token ceiling; the duplicate turn row must not. + await writeFile(path.join(directory, "usage.jsonl"), [ + JSON.stringify({ at: "2026-08-30T00:00:00.000Z", total: 600, turn }), + JSON.stringify({ at: "2026-08-30T00:00:00.001Z", total: 600, turn }) + ].join("\n") + "\n"); + const now = () => new Date("2026-08-30T00:00:00.000Z"); + const fuse = await WakeFuse.open({ organizationKey: "org", environment: environment(directory), now }); + assert.deepEqual(await fuse.admit("alpha", "one"), { state: "admitted" }); +})); diff --git a/src/runtime/wakeFuse.ts b/src/runtime/wakeFuse.ts index 9786dff..d5d00f4 100644 --- a/src/runtime/wakeFuse.ts +++ b/src/runtime/wakeFuse.ts @@ -201,10 +201,13 @@ async function readFuseRecords(directory: string): Promise { let total = 0; + // Rows carrying a `turn` idempotency key count once per turn, whichever file holds them. + const turns = new Set(); for (const file of [`${ledgerPath}.1`, ledgerPath]) { for (const line of await lines(file)) { try { - const value = JSON.parse(line) as { at?: unknown; total?: unknown; agent?: unknown }; + const value = JSON.parse(line) as { at?: unknown; total?: unknown; agent?: unknown; turn?: unknown }; + if (typeof value.turn === "string") { if (turns.has(value.turn)) continue; turns.add(value.turn); } if ((agentId === undefined || value.agent === agentId) && typeof value.at === "string" && !Number.isNaN(Date.parse(value.at)) && value.at >= since && typeof value.total === "number" && Number.isFinite(value.total) && value.total >= 0) total += value.total; } catch { /* usage accounting is advisory input; malformed lines are skipped */ } } From 2bd8dedbcb795ff53b9152c71bfbcd9952c57096 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:13:24 +0200 Subject: [PATCH 031/124] feat: seal Grok broker usage in turn record v2, control protocol v2 and service config v2 with enforced turn limits --- src/runtime/engineBrokerControlClient.ts | 21 +- src/runtime/engineBrokerProtocol.test.ts | 45 +++- src/runtime/engineBrokerProtocol.ts | 74 ++++-- src/runtime/engineBrokerService.test.ts | 20 +- src/runtime/engineBrokerService.ts | 18 +- src/runtime/engineBrokerServiceCli.test.ts | 57 ++++- src/runtime/engineBrokerServiceCli.ts | 14 +- src/runtime/engineBrokerServiceConfig.ts | 69 ++++++ src/runtime/engineBrokerTurnRegistry.test.ts | 49 +++- src/runtime/engineBrokerTurnRegistry.ts | 63 ++++- src/runtime/grokBrokerProxy.test.ts | 22 +- src/runtime/grokBrokerProxy.ts | 34 ++- src/runtime/grokBrokerTurnMeter.test.ts | 103 +++++++++ src/runtime/grokBrokerTurnMeter.ts | 118 ++++++++++ src/runtime/grokEngineBroker.ts | 69 +++--- src/runtime/grokEngineBrokerMetering.ts | 43 ++++ src/runtime/grokEngineBrokerTurn.ts | 124 ++++++++++ src/runtime/grokEngineBrokerUsage.test.ts | 229 +++++++++++-------- 18 files changed, 952 insertions(+), 220 deletions(-) create mode 100644 src/runtime/engineBrokerServiceConfig.ts create mode 100644 src/runtime/grokBrokerTurnMeter.test.ts create mode 100644 src/runtime/grokBrokerTurnMeter.ts create mode 100644 src/runtime/grokEngineBrokerMetering.ts create mode 100644 src/runtime/grokEngineBrokerTurn.ts diff --git a/src/runtime/engineBrokerControlClient.ts b/src/runtime/engineBrokerControlClient.ts index 7136132..5054030 100644 --- a/src/runtime/engineBrokerControlClient.ts +++ b/src/runtime/engineBrokerControlClient.ts @@ -1,13 +1,22 @@ import { createHash, randomUUID } from "node:crypto"; import { createConnection } from "node:net"; -import { encodeEngineBrokerFrame,EngineBrokerFrameDecoder,parseEngineBrokerResponse } from "./engineBrokerProtocol.js"; +import { ENGINE_BROKER_VERSION, encodeEngineBrokerFrame,EngineBrokerFrameDecoder,parseEngineBrokerResponse } from "./engineBrokerProtocol.js"; +import type { EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; -export interface EngineBrokerTurnClient { turn(agentId:string,wakeId:string,prompt:string,mcpEndpoint:string,signal?:AbortSignal):Promise; } +/** + * What the organization runtime asks of a brokered turn beyond the prompt: + * `limits` may only lower the registration's declared limits, and `model`, + * when the agent declared one, must equal the model the broker sealed. + */ +export type EngineBrokerTurnOptions = Readonly<{ limits?: EngineBrokerTurnLimitOverrides; model?: string }>; +export interface EngineBrokerTurnClient { turn(agentId:string,wakeId:string,prompt:string,mcpEndpoint:string,signal?:AbortSignal,options?:EngineBrokerTurnOptions):Promise; } export class EngineBrokerControlClient implements EngineBrokerTurnClient { constructor(private readonly socketPath="/run/daimon-engine-broker/control.sock"){} - async ready():Promise{const requestId=randomUUID(),socket=createConnection({path:this.socketPath}),decoder=new EngineBrokerFrameDecoder();await new Promise((resolve,reject)=>{let settled=false;const fail=()=>{if(settled)return;settled=true;socket.destroy();reject(new Error("engine broker unavailable"));};socket.once("error",fail);socket.once("close",fail);socket.once("connect",()=>socket.write(encodeEngineBrokerFrame({version:"noopolis.daimon.engine-broker.v1",kind:"health",requestId})));socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const response=parseEngineBrokerResponse(value);if(response.kind!=="ready"||response.requestId!==requestId||settled)throw new Error();settled=true;socket.destroy();resolve();}}catch{fail();}});});} - async turn(agentId:string,wakeId:string,prompt:string,mcpEndpoint:string,signal?:AbortSignal):Promise{ - const turnId=createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"),requestId=randomUUID();const request={version:"noopolis.daimon.engine-broker.v1",kind:"start_turn",requestId,turnId,agentId,wakeId,prompt,mcpEndpoint} as const;const socket=createConnection({path:this.socketPath});const decoder=new EngineBrokerFrameDecoder(); - return new Promise((resolve,reject)=>{let accepted=false,settled=false;const fail=()=>{if(settled)return;settled=true;cleanup();reject(new Error("engine broker unavailable"));};const cleanup=()=>{signal?.removeEventListener("abort",abort);socket.destroy();};const abort=()=>fail();signal?.addEventListener("abort",abort,{once:true});if(signal?.aborted)return abort();socket.once("connect",()=>socket.write(encodeEngineBrokerFrame(request)));socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const response=parseEngineBrokerResponse(value);if(response.kind==="ready"||response.requestId!==requestId||response.turnId!==turnId)throw new Error();if(response.kind==="accepted"){if(accepted)throw new Error();accepted=true;continue;}if(!accepted||settled)throw new Error();settled=true;cleanup();if(response.kind==="completed")resolve(response.text);else reject(new Error(response.diagnostic ? `engine broker turn failed (${response.diagnostic.stage}/${response.diagnostic.failureClass}; exit=${response.diagnostic.exitCode}; signal=${response.diagnostic.termSignal})` : "engine broker turn failed"));}}catch{fail();}});socket.once("error",fail);socket.once("close",()=>{if(!settled)fail();});}); + async ready():Promise{const requestId=randomUUID(),socket=createConnection({path:this.socketPath}),decoder=new EngineBrokerFrameDecoder();await new Promise((resolve,reject)=>{let settled=false;const fail=()=>{if(settled)return;settled=true;socket.destroy();reject(new Error("engine broker unavailable"));};socket.once("error",fail);socket.once("close",fail);socket.once("connect",()=>socket.write(encodeEngineBrokerFrame({version:ENGINE_BROKER_VERSION,kind:"health",requestId})));socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const response=parseEngineBrokerResponse(value);if(response.kind!=="ready"||response.requestId!==requestId||settled)throw new Error();settled=true;socket.destroy();resolve();}}catch{fail();}});});} + async turn(agentId:string,wakeId:string,prompt:string,mcpEndpoint:string,signal?:AbortSignal,options:EngineBrokerTurnOptions={}):Promise{ + const turnId=createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"),requestId=randomUUID();const request={version:ENGINE_BROKER_VERSION,kind:"start_turn",requestId,turnId,agentId,wakeId,prompt,mcpEndpoint,...(options.limits===undefined?{}:{limits:options.limits})} as const;const socket=createConnection({path:this.socketPath});const decoder=new EngineBrokerFrameDecoder(); + return new Promise((resolve,reject)=>{let accepted=false,settled=false;const fail=()=>{if(settled)return;settled=true;cleanup();reject(new Error("engine broker unavailable"));};const cleanup=()=>{signal?.removeEventListener("abort",abort);socket.destroy();};const abort=()=>fail();signal?.addEventListener("abort",abort,{once:true});if(signal?.aborted)return abort();socket.once("connect",()=>socket.write(encodeEngineBrokerFrame(request)));socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const response=parseEngineBrokerResponse(value);if(response.kind==="ready"||response.requestId!==requestId||response.turnId!==turnId)throw new Error();if(response.kind==="accepted"){if(accepted)throw new Error();accepted=true;continue;}if(!accepted||settled)throw new Error();settled=true;cleanup(); + if(options.model!==undefined&&response.model!==options.model){reject(new Error(`engine broker turn used model ${response.model}, not the declared ${options.model}`));return;} + if(response.kind==="completed")resolve(response.text);else reject(new Error(`engine broker turn failed (${response.code}${response.limitReason==="none"?"":`; limit=${response.limitReason}`}${response.diagnostic ? `; ${response.diagnostic.stage}/${response.diagnostic.failureClass}; exit=${response.diagnostic.exitCode}; signal=${response.diagnostic.termSignal}` : ""})`));}}catch{fail();}});socket.once("error",fail);socket.once("close",()=>{if(!settled)fail();});}); } } diff --git a/src/runtime/engineBrokerProtocol.test.ts b/src/runtime/engineBrokerProtocol.test.ts index 1be3d53..ae762bc 100644 --- a/src/runtime/engineBrokerProtocol.test.ts +++ b/src/runtime/engineBrokerProtocol.test.ts @@ -1,8 +1,9 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { encodeEngineBrokerFrame, EngineBrokerFrameDecoder, parseEngineBrokerRequest, parseEngineBrokerResponse } from "./engineBrokerProtocol.js"; +import { encodeEngineBrokerFrame, EngineBrokerFrameDecoder, parseEngineBrokerRequest, parseEngineBrokerResponse, parseEngineBrokerV1TerminalResponse } from "./engineBrokerProtocol.js"; -const start = { version: "noopolis.daimon.engine-broker.v1", kind: "start_turn", requestId: "request-1", turnId: "turn-1", agentId: "agent-1", wakeId: "wake-1", prompt: "work",mcpEndpoint:"http://127.0.0.1:4567/mcp" } as const; +const accounting = { outcome: "completed", usage: { input: 20, cacheRead: 0, cacheWrite: 0, output: 10, total: 30 }, model: "grok-4.6", requests: 2, limitReason: "none" } as const; +const start = { version: "noopolis.daimon.engine-broker.v2", kind: "start_turn", requestId: "request-1", turnId: "turn-1", agentId: "agent-1", wakeId: "wake-1", prompt: "work",mcpEndpoint:"http://127.0.0.1:4567/mcp" } as const; test("broker frames survive arbitrary chunking and validate closed requests", () => { const encoded = encodeEngineBrokerFrame(start); const decoder = new EngineBrokerFrameDecoder(); const values: unknown[] = []; @@ -13,15 +14,51 @@ test("broker frames survive arbitrary chunking and validate closed requests", () }); test("broker response attestation is mandatory and bounded", () => { - const value = { version: start.version, kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 12, workerUid: 2200, workerStartTime: "12345" } as const; + const value = { version: start.version, kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 12, workerUid: 2200, workerStartTime: "12345", ...accounting } as const; assert.deepEqual(parseEngineBrokerResponse(value), value); assert.throws(() => parseEngineBrokerResponse({ ...value, workerUid: 0 }), /invalid broker frame/); const decoder = new EngineBrokerFrameDecoder(); assert.throws(() => decoder.push(Uint8Array.from([0, 16, 0, 1])), /invalid broker frame/); }); test("broker failure diagnostics are closed and contain no raw worker output",()=>{ - const value={version:start.version,kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",diagnostic:{status:"prelaunch_failed",stage:"executable",failureClass:"executable",profileApplied:false,exitCode:-1,termSignal:0,workerPid:0,workerUid:0,startTicks:"0"}} as const; + const value={version:start.version,kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",diagnostic:{status:"prelaunch_failed",stage:"executable",failureClass:"executable",profileApplied:false,exitCode:-1,termSignal:0,workerPid:0,workerUid:0,startTicks:"0"},outcome:"failed",usage:null,model:"grok-4.6",requests:0,limitReason:"none"} as const; assert.deepEqual(parseEngineBrokerResponse(value),value); assert.throws(()=>parseEngineBrokerResponse({...value,diagnostic:{...value.diagnostic,rawOutput:"secret"}}),/invalid broker frame/u); assert.throws(()=>parseEngineBrokerResponse({...value,diagnostic:{...value.diagnostic,failureClass:"secret"}}),/invalid broker frame/u); }); + +test("v2 terminal frames carry closed numeric accounting and refuse anything else", () => { + const completed = { version: start.version, kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 12, workerUid: 2200, workerStartTime: "12345", ...accounting } as const; + assert.deepEqual(parseEngineBrokerResponse(completed), completed); + for (const bad of [ + { ...completed, usage: { ...accounting.usage, total: 31 } }, + { ...completed, usage: { ...accounting.usage, note: "text" } }, + { ...completed, usage: { ...accounting.usage, input: "20" } }, + { ...completed, model: "grok-4.6-build" }, + { ...completed, outcome: "failed" }, + { ...completed, limitReason: "tokens" }, + { ...completed, limitReason: "budget" }, + { ...completed, requests: -1 }, + { ...completed, extra: 1 }, + (({ limitReason: _omit, ...rest }) => rest)(completed) + ]) assert.throws(() => parseEngineBrokerResponse(bad), /invalid broker frame/u); + const limit = { version: start.version, kind: "failed", requestId: "request-1", turnId: "turn-1", code: "limit_exceeded", outcome: "failed", usage: { input: 5, cacheRead: 5, cacheWrite: 0, output: 1, total: 11 }, model: "grok-4.6", requests: 3, limitReason: "requests" } as const; + assert.deepEqual(parseEngineBrokerResponse(limit), limit); + assert.throws(() => parseEngineBrokerResponse({ ...limit, limitReason: "none" }), /invalid broker frame/u); + assert.throws(() => parseEngineBrokerResponse({ ...limit, code: "engine_failed" }), /invalid broker frame/u); +}); + +test("v1 frames are refused on the wire but a v1 terminal record still parses", () => { + const v1 = { version: "noopolis.daimon.engine-broker.v1", kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 12, workerUid: 2200, workerStartTime: "12345" } as const; + assert.throws(() => parseEngineBrokerResponse(v1), /invalid broker frame/u); + assert.throws(() => parseEngineBrokerRequest({ ...start, version: "noopolis.daimon.engine-broker.v1" }), /invalid broker frame/u); + assert.deepEqual(parseEngineBrokerV1TerminalResponse(v1), v1); + assert.throws(() => parseEngineBrokerV1TerminalResponse({ ...v1, ...accounting }), /invalid broker frame/u); +}); + +test("start_turn limits are an optional closed subset inside their bounds", () => { + assert.deepEqual(parseEngineBrokerRequest({ ...start, limits: { maxTokens: 1_000 } }), { ...start, limits: { maxTokens: 1_000 } }); + for (const limits of [{}, { maxTokens: 0 }, { maxRequests: 49 }, { timeoutMs: 999 }, { maxTokens: 1, raise: true }, { maxTokens: 1.5 }]) { + assert.throws(() => parseEngineBrokerRequest({ ...start, limits }), /invalid broker frame/u); + } +}); diff --git a/src/runtime/engineBrokerProtocol.ts b/src/runtime/engineBrokerProtocol.ts index f8c79aa..d5de2f0 100644 --- a/src/runtime/engineBrokerProtocol.ts +++ b/src/runtime/engineBrokerProtocol.ts @@ -1,10 +1,21 @@ -const VERSION = "noopolis.daimon.engine-broker.v1" as const; +import { parseEngineBrokerTurnAccounting, parseEngineBrokerTurnLimitOverrides, type EngineBrokerTurnAccounting, type EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; + +/** + * Control protocol v2. Both ends ship in the same Daimon package and image + * (organization runtime client, broker service), so the wire moved to v2 in + * one step: v1 frames are refused. v1 survives only as a *durable record* + * shape, which {@link parseEngineBrokerV1TerminalResponse} still reads so + * turns sealed before the upgrade keep replaying. + */ +const VERSION = "noopolis.daimon.engine-broker.v2" as const; +export const ENGINE_BROKER_VERSION = VERSION; +const V1 = "noopolis.daimon.engine-broker.v1" as const; export const ENGINE_BROKER_MAX_FRAME_BYTES = 1_048_576; const ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; export type EngineBrokerRequest = | Readonly<{ version: typeof VERSION; kind: "health"; requestId: string }> - | Readonly<{ version: typeof VERSION; kind: "start_turn"; requestId: string; turnId: string; agentId: string; wakeId: string; prompt: string; mcpEndpoint: string }> + | Readonly<{ version: typeof VERSION; kind: "start_turn"; requestId: string; turnId: string; agentId: string; wakeId: string; prompt: string; mcpEndpoint: string; limits?: EngineBrokerTurnLimitOverrides }> | Readonly<{ version: typeof VERSION; kind: "cancel_turn"; requestId: string; turnId: string }>; export interface EngineBrokerFailureDiagnostic { status:string;stage:string;failureClass:string;profileApplied:boolean;exitCode:number;termSignal:number;workerPid:number;workerUid:number;startTicks:string } @@ -12,8 +23,15 @@ export interface EngineBrokerFailureDiagnostic { status:string;stage:string;fail export type EngineBrokerResponse = | Readonly<{ version: typeof VERSION; kind: "ready"; requestId: string; brokerUid: 2100; providerProxyPort: 43123; mcpFacadePort: 43124; registrations: number; credentialStale: false; realmLease: true; workerIsolation: true }> | Readonly<{ version: typeof VERSION; kind: "accepted"; requestId: string; turnId: string }> - | Readonly<{ version: typeof VERSION; kind: "completed"; requestId: string; turnId: string; text: string; workerPid: number; workerUid: number; workerStartTime: string }> - | Readonly<{ version: typeof VERSION; kind: "failed"; requestId: string; turnId: string; code: "auth_stale" | "cancelled" | "engine_failed" | "invalid_request" | "turn_conflict" | "unavailable"; diagnostic?: EngineBrokerFailureDiagnostic }>; + | (Readonly<{ version: typeof VERSION; kind: "completed"; requestId: string; turnId: string; text: string; workerPid: number; workerUid: number; workerStartTime: string }> & EngineBrokerTurnAccounting) + | (Readonly<{ version: typeof VERSION; kind: "failed"; requestId: string; turnId: string; code: EngineBrokerFailureCode; diagnostic?: EngineBrokerFailureDiagnostic }> & EngineBrokerTurnAccounting); +export const ENGINE_BROKER_FAILURE_CODES = ["auth_stale", "cancelled", "engine_failed", "invalid_request", "limit_exceeded", "turn_conflict", "unavailable"] as const; +export type EngineBrokerFailureCode = (typeof ENGINE_BROKER_FAILURE_CODES)[number]; +export type EngineBrokerTerminalResponse = Extract; +type V1Completed = Readonly<{ version: typeof V1; kind: "completed"; requestId: string; turnId: string; text: string; workerPid: number; workerUid: number; workerStartTime: string }>; +type V1Failed = Readonly<{ version: typeof V1; kind: "failed"; requestId: string; turnId: string; code: Exclude; diagnostic?: EngineBrokerFailureDiagnostic }>; +export type EngineBrokerV1TerminalResponse = V1Completed | V1Failed; +const ACCOUNTING = ["outcome", "usage", "model", "requests", "limitReason"] as const; type JsonRecord = Record; const record = (value: unknown): JsonRecord => { @@ -34,9 +52,12 @@ export function parseEngineBrokerRequest(value: unknown): EngineBrokerRequest { const input = record(value); version(input.version); if(input.kind==="health"){exact(input,["version","kind","requestId"]);return {version:VERSION,kind:"health",requestId:id(input.requestId)};} if (input.kind === "start_turn") { - exact(input, ["version", "kind", "requestId", "turnId", "agentId", "wakeId", "prompt", "mcpEndpoint"]); + const fields = ["version", "kind", "requestId", "turnId", "agentId", "wakeId", "prompt", "mcpEndpoint"]; + exact(input, input.limits === undefined ? fields : [...fields, "limits"]); const mcpEndpoint=text(input.mcpEndpoint,2048);const url=new URL(mcpEndpoint);if(url.protocol!=="http:"||url.hostname!=="127.0.0.1"||url.pathname!=="/mcp")throw new TypeError("invalid broker frame"); - return { version: VERSION, kind: "start_turn", requestId: id(input.requestId), turnId: id(input.turnId), agentId: id(input.agentId), wakeId: id(input.wakeId), prompt: text(input.prompt, 65_536),mcpEndpoint }; + let limits: EngineBrokerTurnLimitOverrides | undefined; + if (input.limits !== undefined) { try { limits = parseEngineBrokerTurnLimitOverrides(input.limits); } catch { throw new TypeError("invalid broker frame"); } } + return { version: VERSION, kind: "start_turn", requestId: id(input.requestId), turnId: id(input.turnId), agentId: id(input.agentId), wakeId: id(input.wakeId), prompt: text(input.prompt, 65_536),mcpEndpoint,...(limits === undefined ? {} : { limits }) }; } if (input.kind === "cancel_turn") { exact(input, ["version", "kind", "requestId", "turnId"]); @@ -52,20 +73,39 @@ export function parseEngineBrokerResponse(value: unknown): EngineBrokerResponse exact(input, ["version", "kind", "requestId", "turnId"]); return { version: VERSION, kind: "accepted", requestId: id(input.requestId), turnId: id(input.turnId) }; } + if (input.kind === "completed" || input.kind === "failed") return parseTerminal(input, VERSION) as EngineBrokerTerminalResponse; + throw new TypeError("invalid broker frame"); +} + +/** + * A terminal response persisted by a pre-v2 broker: the v1 field sets exactly, + * with no accounting. Accepted only from the durable turn registry, never from + * the wire. + */ +export function parseEngineBrokerV1TerminalResponse(value: unknown): EngineBrokerV1TerminalResponse { + const input = record(value); if (input.version !== V1 || (input.kind !== "completed" && input.kind !== "failed")) throw new TypeError("invalid broker frame"); + return parseTerminal(input, V1) as EngineBrokerV1TerminalResponse; +} + +function parseTerminal(input: JsonRecord, expected: typeof VERSION | typeof V1): EngineBrokerTerminalResponse | EngineBrokerV1TerminalResponse { + const accounting = expected === VERSION ? ACCOUNTING : []; if (input.kind === "completed") { - exact(input, ["version", "kind", "requestId", "turnId", "text", "workerPid", "workerUid", "workerStartTime"]); + exact(input, ["version", "kind", "requestId", "turnId", "text", "workerPid", "workerUid", "workerStartTime", ...accounting]); if (!Number.isSafeInteger(input.workerPid) || (input.workerPid as number) < 1 || !Number.isSafeInteger(input.workerUid) || (input.workerUid as number) < 1) throw new TypeError("invalid broker frame"); - return { version: VERSION, kind: "completed", requestId: id(input.requestId), turnId: id(input.turnId), text: text(input.text, 262_144), workerPid: input.workerPid as number, workerUid: input.workerUid as number, workerStartTime: id(input.workerStartTime) }; + const base = { kind: "completed", requestId: id(input.requestId), turnId: id(input.turnId), text: text(input.text, 262_144), workerPid: input.workerPid as number, workerUid: input.workerUid as number, workerStartTime: id(input.workerStartTime) } as const; + return expected === VERSION ? { version: VERSION, ...base, ...parseEngineBrokerTurnAccounting(input, "completed") } : { version: V1, ...base }; } - if (input.kind === "failed") { - exact(input, input.diagnostic === undefined ? ["version", "kind", "requestId", "turnId", "code"] : ["version", "kind", "requestId", "turnId", "code", "diagnostic"]); - const codes = ["auth_stale", "cancelled", "engine_failed", "invalid_request", "turn_conflict", "unavailable"] as const; - if (!codes.includes(input.code as typeof codes[number])) throw new TypeError("invalid broker frame"); - let diagnostic:EngineBrokerFailureDiagnostic|undefined; - if(input.diagnostic!==undefined){const value=record(input.diagnostic);exact(value,["status","stage","failureClass","profileApplied","exitCode","termSignal","workerPid","workerUid","startTicks"]);const status=["prelaunch_failed","worker_failed","output_failed","cancelled"],stage=["peer","request","registration","executable","exec","wait","output","attestation"],failureClass=["peer","protocol","registration","executable","exec","wait","output_limit","cancelled","profile_missing","profile_invalid"];if(!status.includes(value.status as string)||!stage.includes(value.stage as string)||!failureClass.includes(value.failureClass as string)||typeof value.profileApplied!=="boolean"||![value.exitCode,value.termSignal,value.workerPid,value.workerUid].every(Number.isSafeInteger)||typeof value.startTicks!=="string"||!/^(0|[1-9][0-9]*)$/u.test(value.startTicks)||!closedDiagnostic(value))throw new TypeError("invalid broker frame");diagnostic=value as unknown as EngineBrokerFailureDiagnostic;} - return { version: VERSION, kind: "failed", requestId: id(input.requestId), turnId: id(input.turnId), code: input.code as typeof codes[number],...(diagnostic?{diagnostic}:{}) }; - } - throw new TypeError("invalid broker frame"); + const fields = ["version", "kind", "requestId", "turnId", "code", ...accounting]; + exact(input, input.diagnostic === undefined ? fields : [...fields, "diagnostic"]); + const codes: readonly string[] = expected === VERSION ? ENGINE_BROKER_FAILURE_CODES : ENGINE_BROKER_FAILURE_CODES.filter((code) => code !== "limit_exceeded"); + if (!codes.includes(input.code as string)) throw new TypeError("invalid broker frame"); + let diagnostic:EngineBrokerFailureDiagnostic|undefined; + if(input.diagnostic!==undefined){const value=record(input.diagnostic);exact(value,["status","stage","failureClass","profileApplied","exitCode","termSignal","workerPid","workerUid","startTicks"]);const status=["prelaunch_failed","worker_failed","output_failed","cancelled"],stage=["peer","request","registration","executable","exec","wait","output","attestation"],failureClass=["peer","protocol","registration","executable","exec","wait","output_limit","cancelled","profile_missing","profile_invalid"];if(!status.includes(value.status as string)||!stage.includes(value.stage as string)||!failureClass.includes(value.failureClass as string)||typeof value.profileApplied!=="boolean"||![value.exitCode,value.termSignal,value.workerPid,value.workerUid].every(Number.isSafeInteger)||typeof value.startTicks!=="string"||!/^(0|[1-9][0-9]*)$/u.test(value.startTicks)||!closedDiagnostic(value))throw new TypeError("invalid broker frame");diagnostic=value as unknown as EngineBrokerFailureDiagnostic;} + const base = { kind: "failed", requestId: id(input.requestId), turnId: id(input.turnId), code: input.code as EngineBrokerFailureCode, ...(diagnostic ? { diagnostic } : {}) } as const; + if (expected === V1) return { version: V1, ...base } as V1Failed; + const accountingValue = parseEngineBrokerTurnAccounting(input, "failed"); + if ((input.code === "limit_exceeded") !== (accountingValue.limitReason !== "none")) throw new TypeError("invalid broker frame"); + return { version: VERSION, ...base, ...accountingValue }; } function closedDiagnostic(value:JsonRecord):boolean{ diff --git a/src/runtime/engineBrokerService.test.ts b/src/runtime/engineBrokerService.test.ts index e4fe882..d33f2bf 100644 --- a/src/runtime/engineBrokerService.test.ts +++ b/src/runtime/engineBrokerService.test.ts @@ -5,6 +5,9 @@ import path from "node:path"; import test from "node:test"; import { EngineBrokerControlClient } from "./engineBrokerControlClient.js"; import { startEngineBrokerServiceWithIdentity, type EngineBrokerServiceEngine } from "./engineBrokerService.js"; +import { EngineBrokerTurnFailure } from "./grokEngineBroker.js"; + +const completedAccounting = { outcome: "completed", usage: { input: 8, cacheRead: 2, cacheWrite: 0, output: 1, total: 11 }, model: "grok-4.6", requests: 1, limitReason: "none" } as const; test("broker backend serves a turn and preserves worker attestation", async () => { await withService(async (client) => {await client.ready();assert.equal(await client.turn("agent-a","wake-a","hello","http://127.0.0.1:44001/mcp"),"answer");}); @@ -33,8 +36,23 @@ test("service shutdown aborts turns and closes connected clients", async () => { await started;await service.close();await rejected;assert.equal(aborted,true);await rm(directory,{recursive:true,force:true}); }); -async function withService(run:(client:EngineBrokerControlClient)=>Promise,turn:EngineBrokerServiceEngine["turn"]=async()=>({text:"answer",workerPid:22,workerUid:2200,workerStartTime:"123"}),readiness:EngineBrokerServiceEngine["readiness"]=()=>({providerProxyPort:43123,mcpFacadePort:43124,registrations:1,credentialStale:false,realmLease:true,workerIsolation:true})):Promise{ +async function withService(run:(client:EngineBrokerControlClient)=>Promise,turn:EngineBrokerServiceEngine["turn"]=async()=>({text:"answer",workerPid:22,workerUid:2200,workerStartTime:"123",...completedAccounting}),readiness:EngineBrokerServiceEngine["readiness"]=()=>({providerProxyPort:43123,mcpFacadePort:43124,registrations:1,credentialStale:false,realmLease:true,workerIsolation:true})):Promise{ const directory=await mkdtemp(path.join(tmpdir(),"daimon-broker-service-")),socketPath=path.join(directory,"broker.sock"),engine=makeEngine(turn,readiness);const service=await startEngineBrokerServiceWithIdentity(engine,socketPath,process.getuid!()); try{await run(new EngineBrokerControlClient(socketPath));}finally{await service.close();await rm(directory,{recursive:true,force:true});} } function makeEngine(turn:EngineBrokerServiceEngine["turn"],readiness:EngineBrokerServiceEngine["readiness"]=()=>({providerProxyPort:43123,mcpFacadePort:43124,registrations:1,credentialStale:false,realmLease:true,workerIsolation:true})):EngineBrokerServiceEngine{return {turn,readiness,close:async()=>undefined};} + +test("the wake's lowering limits reach the broker, and the client verifies the declared model", async () => { + let seen: unknown; + await withService(async (client) => { + assert.equal(await client.turn("agent-a","wake-a","hello","http://127.0.0.1:44001/mcp",undefined,{limits:{maxTokens:1_000,timeoutMs:5_000},model:"grok-4.6"}),"answer"); + assert.deepEqual(seen,{maxTokens:1_000,timeoutMs:5_000}); + await assert.rejects(client.turn("agent-a","wake-b","hello","http://127.0.0.1:44001/mcp",undefined,{model:"grok-4.5"}),/model grok-4.6, not the declared grok-4.5/u); + },async(_agent,_wake,_prompt,_endpoint,_signal,limits)=>{seen=limits;return {text:"answer",workerPid:22,workerUid:2200,workerStartTime:"123",...completedAccounting};}); +}); + +test("a limit failure reaches the client with its code and limit reason", async () => { + await withService(async (client) => { + await assert.rejects(client.turn("agent-a","wake-a","hello","http://127.0.0.1:44001/mcp"),/engine broker turn failed \(limit_exceeded; limit=requests\)/u); + },async()=>{throw new EngineBrokerTurnFailure("limit_exceeded",undefined,{outcome:"failed",usage:{input:1,cacheRead:0,cacheWrite:0,output:1,total:2},model:"grok-4.6",requests:3,limitReason:"requests"});}); +}); diff --git a/src/runtime/engineBrokerService.ts b/src/runtime/engineBrokerService.ts index 9faa5c4..2b3f4c5 100644 --- a/src/runtime/engineBrokerService.ts +++ b/src/runtime/engineBrokerService.ts @@ -1,9 +1,10 @@ import { chmod, lstat, unlink } from "node:fs/promises"; import { createServer, type Server, type Socket } from "node:net"; import { encodeEngineBrokerFrame,EngineBrokerFrameDecoder,parseEngineBrokerRequest,type EngineBrokerResponse } from "./engineBrokerProtocol.js"; +import type { EngineBrokerTurnAccounting, EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; import { EngineBrokerTurnFailure } from "./grokEngineBroker.js"; export interface EngineBrokerServiceEngine { - turn(agentId:string,wakeId:string,prompt:string,mcpEndpoint:string,signal?:AbortSignal):Promise>; + turn(agentId:string,wakeId:string,prompt:string,mcpEndpoint:string,signal?:AbortSignal,limits?:EngineBrokerTurnLimitOverrides):Promise&EngineBrokerTurnAccounting>; readiness():Readonly<{providerProxyPort:number;mcpFacadePort:number;registrations:number;credentialStale:boolean;realmLease:boolean;workerIsolation:boolean}>; close():Promise; } @@ -36,7 +37,20 @@ export async function startEngineBrokerServiceWithIdentity(broker:EngineBrokerSe * reader has disconnected. */ function handleSocketError(socket:Socket,owned:()=>Readonly<{turnId:string;controller:AbortController}>|undefined):void{socket.on("error",()=>{owned()?.controller.abort();socket.destroy();});} -function handle(socket:Socket,broker:EngineBrokerServiceEngine,active:Map):void{const decoder=new EngineBrokerFrameDecoder();let started=false,owned:Readonly<{turnId:string;controller:AbortController}>|undefined;handleSocketError(socket,()=>owned);socket.once("close",()=>owned?.controller.abort());socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const request=parseEngineBrokerRequest(value);if(request.kind==="health"){if(started)throw new Error();started=true;const ready=broker.readiness();if(ready.providerProxyPort!==43123||ready.mcpFacadePort!==43124||ready.registrations<1||ready.credentialStale||!ready.realmLease||!ready.workerIsolation)throw new Error();return send(socket,{version:request.version,kind:"ready",requestId:request.requestId,brokerUid:2100,providerProxyPort:43123,mcpFacadePort:43124,registrations:ready.registrations,credentialStale:false,realmLease:true,workerIsolation:true});}if(request.kind==="cancel_turn"){active.get(request.turnId)?.abort();continue;}if(started||active.has(request.turnId))throw new Error();started=true;const controller=new AbortController();owned={turnId:request.turnId,controller};active.set(request.turnId,controller);socket.write(encodeEngineBrokerFrame({version:request.version,kind:"accepted",requestId:request.requestId,turnId:request.turnId}));void broker.turn(request.agentId,request.wakeId,request.prompt,request.mcpEndpoint,controller.signal).then((result)=>send(socket,{version:request.version,kind:"completed",requestId:request.requestId,turnId:request.turnId,...result}),(error:unknown)=>send(socket,{version:request.version,kind:"failed",requestId:request.requestId,turnId:request.turnId,code:error instanceof EngineBrokerTurnFailure?error.code:controller.signal.aborted?"cancelled":"engine_failed",...(error instanceof EngineBrokerTurnFailure&&error.diagnostic?{diagnostic:error.diagnostic}:{})})).finally(()=>{if(active.get(request.turnId)===controller)active.delete(request.turnId);owned=undefined;});}}catch{socket.destroy();}});} +function handle(socket:Socket,broker:EngineBrokerServiceEngine,active:Map):void{const decoder=new EngineBrokerFrameDecoder();let started=false,owned:Readonly<{turnId:string;controller:AbortController}>|undefined;handleSocketError(socket,()=>owned);socket.once("close",()=>owned?.controller.abort());socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const request=parseEngineBrokerRequest(value);if(request.kind==="health"){if(started)throw new Error();started=true;const ready=broker.readiness();if(ready.providerProxyPort!==43123||ready.mcpFacadePort!==43124||ready.registrations<1||ready.credentialStale||!ready.realmLease||!ready.workerIsolation)throw new Error();return send(socket,{version:request.version,kind:"ready",requestId:request.requestId,brokerUid:2100,providerProxyPort:43123,mcpFacadePort:43124,registrations:ready.registrations,credentialStale:false,realmLease:true,workerIsolation:true});}if(request.kind==="cancel_turn"){active.get(request.turnId)?.abort();continue;}if(started||active.has(request.turnId))throw new Error();started=true;const controller=new AbortController();owned={turnId:request.turnId,controller};active.set(request.turnId,controller);socket.write(encodeEngineBrokerFrame({version:request.version,kind:"accepted",requestId:request.requestId,turnId:request.turnId}));void broker.turn(request.agentId,request.wakeId,request.prompt,request.mcpEndpoint,controller.signal,request.limits).then((result)=>send(socket,{version:request.version,kind:"completed",requestId:request.requestId,turnId:request.turnId,text:result.text,workerPid:result.workerPid,workerUid:result.workerUid,workerStartTime:result.workerStartTime,outcome:"completed",usage:result.usage,model:result.model,requests:result.requests,limitReason:"none"}),(error:unknown)=>failed(socket,request,error,controller.signal.aborted)).finally(()=>{if(active.get(request.turnId)===controller)active.delete(request.turnId);owned=undefined;});}}catch{socket.destroy();}});} +/** + * Every failure the broker raises for a known registration carries its + * accounting (a wake that tried to raise a limit: `usage: null`, zero + * requests). A failure with no registration behind it (unknown agent, closed + * broker) has no declared model to report, so it is refused as a bare + * connection close and the client reports the broker unavailable. + */ +function failed(socket:Socket,request:Extract,{kind:"start_turn"}>,error:unknown,aborted:boolean):void{ + const accounting=error instanceof EngineBrokerTurnFailure?error.accounting:undefined; + if(accounting===undefined){socket.destroy();return;} + const code=error instanceof EngineBrokerTurnFailure?error.code:aborted?"cancelled":"engine_failed"; + send(socket,{version:request.version,kind:"failed",requestId:request.requestId,turnId:request.turnId,code,...(error instanceof EngineBrokerTurnFailure&&error.diagnostic?{diagnostic:error.diagnostic}:{}),outcome:"failed",usage:accounting.usage,model:accounting.model,requests:accounting.requests,limitReason:accounting.limitReason}); +} function send(socket:Socket,response:EngineBrokerResponse):void{if(!socket.destroyed)socket.end(encodeEngineBrokerFrame(response));} async function removeOwnedSocket(file:string,uid:number):Promise{try{const entry=await lstat(file);if(!entry.isSocket()||Number(entry.uid)!==uid)throw new Error("unsafe broker socket");await unlink(file);}catch(error){if((error as NodeJS.ErrnoException).code!=="ENOENT")throw error;}} async function verifySocket(file:string,uid:number):Promise{const entry=await lstat(file);if(!entry.isSocket()||Number(entry.uid)!==uid||(Number(entry.mode)&0o777)!==0o600)throw new Error("unsafe broker socket");} diff --git a/src/runtime/engineBrokerServiceCli.test.ts b/src/runtime/engineBrokerServiceCli.test.ts index 5d11cf6..db7771d 100644 --- a/src/runtime/engineBrokerServiceCli.test.ts +++ b/src/runtime/engineBrokerServiceCli.test.ts @@ -1,18 +1,55 @@ import assert from "node:assert/strict"; import test from "node:test"; import { parseEngineBrokerServiceConfig } from "./engineBrokerServiceCli.js"; +import { engineBrokerRequestLedgerPathFor } from "./engineBrokerServiceConfig.js"; -test("parses the closed broker service configuration",()=>{ - const registration=reg("agent-a",0);assert.deepEqual(parseEngineBrokerServiceConfig({version:"noopolis.daimon.engine-broker-service.v1",credentialHome:"/var/lib/daimon-engine-broker/credential",turnStore:"/var/lib/daimon-engine-broker/turns",registrations:[registration]}),{credentialHome:"/var/lib/daimon-engine-broker/credential",turnStore:"/var/lib/daimon-engine-broker/turns",registrations:[registration]}); +const paths = { credentialHome: "/var/lib/daimon-engine-broker/credential", turnStore: "/var/lib/daimon-engine-broker/turns" }; +const reg = (agentId: string, slot: number) => ({ agentId, slot, workerUid: 2200 + slot, workspace: `/workspace/${slot}`, profilePath: `/workers/${slot}/.grok/sandbox.toml`, eventsPath: `/workers/${slot}/.grok/sessions/sandbox-events.jsonl`, profileSha256: "a".repeat(64) }); +const v2 = (agentId: string, slot: number) => ({ ...reg(agentId, slot), usageLedgerPath: `/run/slots/${slot}/usage/usage.jsonl`, limits: { maxRequests: 12, maxTokens: 400_000, timeoutMs: 480_000 }, model: { id: "grok-4.6", reasoningEffort: "low" } }); +const config = (version: string, registrations: readonly unknown[]) => ({ version: `noopolis.daimon.engine-broker-service.${version}`, ...paths, registrations }); + +test("v1 service config is still accepted and receives today's defaults", () => { + assert.deepEqual(parseEngineBrokerServiceConfig(config("v1", [reg("agent-a", 0)])), { ...paths, registrations: [{ + ...reg("agent-a", 0), usageLedgerPath: "/var/lib/spawnfile/daimon/usage/usage.jsonl", + limits: { maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }, model: { model: "grok-4.6", reasoningEffort: "low" } + }] }); +}); + +test("v2 declares a per-slot ledger, limits and a closed-list model per registration", () => { + const parsed = parseEngineBrokerServiceConfig(config("v2", [v2("agent-a", 0), v2("agent-b", 1)])); + assert.deepEqual(parsed.registrations[1], { ...reg("agent-b", 1), usageLedgerPath: "/run/slots/1/usage/usage.jsonl", limits: { maxRequests: 12, maxTokens: 400_000, timeoutMs: 480_000 }, model: { model: "grok-4.6", reasoningEffort: "low" } }); + assert.equal(engineBrokerRequestLedgerPathFor(parsed.registrations[0]!.usageLedgerPath), "/run/slots/0/usage/requests.jsonl"); +}); + +test("v2 rejects unknown keys at every level, off-list models, missing fields and out-of-bound limits", () => { + const base = v2("agent-a", 0); + // Mutation guard: loosening any exact-member check accepts one of these. + for (const registration of [ + { ...base, extra: true }, + { ...base, limits: { ...base.limits, maxWakes: 1 } }, + { ...base, model: { ...base.model, provider: "xai" } }, + { ...base, model: { id: "grok-4.6-build", reasoningEffort: "low" } }, + { ...base, model: { id: "grok-4.6", reasoningEffort: "xhigh" } }, + { ...base, limits: { ...base.limits, maxRequests: 49 } }, + { ...base, limits: { ...base.limits, maxTokens: 0 } }, + { ...base, usageLedgerPath: "relative/usage.jsonl" }, + { ...base, usageLedgerPath: "/run/slots/0/requests.jsonl" }, + { ...base, usageLedgerPath: "/run/slots/0/usage" }, + (({ model: _omit, ...rest }) => rest)(base), + reg("agent-a", 0) + ]) assert.throws(() => parseEngineBrokerServiceConfig(config("v2", [registration])), /invalid engine broker service config/u); + assert.throws(() => parseEngineBrokerServiceConfig({ ...config("v2", [base]), grokCommand: "evil" }), /invalid engine broker service config/u); + assert.throws(() => parseEngineBrokerServiceConfig(config("v1", [base])), /invalid engine broker service config/u, "v2 members are not accepted under v1"); + assert.throws(() => parseEngineBrokerServiceConfig(config("v3", [base])), /invalid engine broker service config/u); }); -test("rejects caller-selected commands, duplicate identities, and traversal",()=>{ - const base={version:"noopolis.daimon.engine-broker-service.v1",credentialHome:"/var/lib/daimon-engine-broker/credential",turnStore:"/var/lib/daimon-engine-broker/turns",registrations:[reg("agent-a",0)]}; - assert.throws(()=>parseEngineBrokerServiceConfig({...base,grokCommand:"evil"})); - assert.throws(()=>parseEngineBrokerServiceConfig({...base,turnStore:"/var/lib/../secret"})); - assert.throws(()=>parseEngineBrokerServiceConfig({...base,registrations:[reg("agent-a",0),reg("agent-a",1)]})); +test("rejects caller-selected commands, duplicate identities, and traversal", () => { + const base = config("v1", [reg("agent-a", 0)]); + assert.throws(() => parseEngineBrokerServiceConfig({ ...base, grokCommand: "evil" })); + assert.throws(() => parseEngineBrokerServiceConfig({ ...base, turnStore: "/var/lib/../secret" })); + assert.throws(() => parseEngineBrokerServiceConfig({ ...base, registrations: [reg("agent-a", 0), reg("agent-a", 1)] })); + assert.throws(() => parseEngineBrokerServiceConfig({ ...base, registrations: [reg("agent-a", 0), reg("agent-b", 0)] }), /invalid/u, "one slot is one worker"); // Grok 1.0.34 logs sandbox events under $GROK_HOME/sessions/; the 1.0.13 root path stays empty and must not be attested. - assert.throws(()=>parseEngineBrokerServiceConfig({...base,registrations:[{...reg("agent-a",0),eventsPath:"/workers/0/.grok/sandbox-events.jsonl"}]})); - assert.throws(()=>parseEngineBrokerServiceConfig({...base,registrations:[{...reg("agent-a",0),eventsPath:"/workers/1/.grok/sessions/sandbox-events.jsonl"}]})); + assert.throws(() => parseEngineBrokerServiceConfig({ ...base, registrations: [{ ...reg("agent-a", 0), eventsPath: "/workers/0/.grok/sandbox-events.jsonl" }] })); + assert.throws(() => parseEngineBrokerServiceConfig({ ...base, registrations: [{ ...reg("agent-a", 0), eventsPath: "/workers/1/.grok/sessions/sandbox-events.jsonl" }] })); }); -const reg=(agentId:string,slot:number)=>({agentId,slot,workerUid:2200+slot,workspace:`/workspace/${slot}`,profilePath:`/workers/${slot}/.grok/sandbox.toml`,eventsPath:`/workers/${slot}/.grok/sessions/sandbox-events.jsonl`,profileSha256:"a".repeat(64)}); diff --git a/src/runtime/engineBrokerServiceCli.ts b/src/runtime/engineBrokerServiceCli.ts index d70839c..2a3967a 100644 --- a/src/runtime/engineBrokerServiceCli.ts +++ b/src/runtime/engineBrokerServiceCli.ts @@ -1,8 +1,10 @@ import { constants } from "node:fs"; import { open } from "node:fs/promises"; import { startEngineBrokerService } from "./engineBrokerService.js"; -import { startGrokEngineBroker, type GrokEngineBrokerRegistration } from "./grokEngineBroker.js"; -import { grokWorkerEventsPathFor } from "./grokWorkerSandboxProfile.js"; +import { parseEngineBrokerServiceConfig } from "./engineBrokerServiceConfig.js"; +import { startGrokEngineBroker } from "./grokEngineBroker.js"; + +export { parseEngineBrokerServiceConfig }; export const ENGINE_BROKER_SERVICE_CONFIG = "/etc/daimon-engine-broker/service.json"; const MAX_CONFIG_BYTES=65_536; @@ -16,12 +18,4 @@ export async function runEngineBrokerServiceCli():Promise{ const onSignal=()=>{void stop().catch(()=>{process.exitCode=1;});};process.once("SIGINT",onSignal);process.once("SIGTERM",onSignal); } -export function parseEngineBrokerServiceConfig(value:unknown):Readonly<{credentialHome:string;turnStore:string;registrations:readonly GrokEngineBrokerRegistration[]}>{ - if(value===null||typeof value!=="object"||Array.isArray(value))throw new TypeError("invalid engine broker service config");const input=value as Record; - if(Object.keys(input).length!==4||input.version!=="noopolis.daimon.engine-broker-service.v1"||typeof input.credentialHome!=="string"||typeof input.turnStore!=="string"||!Array.isArray(input.registrations))throw new TypeError("invalid engine broker service config"); - const absolute=(item:string)=>item.startsWith("/")&&!item.includes("/../")&&!item.endsWith("/..");if(!absolute(input.credentialHome)||!absolute(input.turnStore))throw new TypeError("invalid engine broker service config"); - const seen=new Set();const registrations=input.registrations.map((entry)=>{if(entry===null||typeof entry!=="object"||Array.isArray(entry))throw new TypeError("invalid engine broker service config");const item=entry as Record;if(Object.keys(item).length!==7||typeof item.agentId!=="string"||!item.agentId.trim()||!Number.isSafeInteger(item.slot)||(item.slot as number)<0||!Number.isSafeInteger(item.workerUid)||(item.workerUid as number)<2200||typeof item.workspace!=="string"||!absolute(item.workspace)||typeof item.profilePath!=="string"||!absolute(item.profilePath)||typeof item.eventsPath!=="string"||!absolute(item.eventsPath)||typeof item.profileSha256!=="string"||!/^[a-f0-9]{64}$/u.test(item.profileSha256)||item.eventsPath!==grokWorkerEventsPathFor(item.profilePath)||!item.profilePath.endsWith("/sandbox.toml")||seen.has(item.agentId))throw new TypeError("invalid engine broker service config");seen.add(item.agentId);return {agentId:item.agentId,slot:item.slot as number,workerUid:item.workerUid as number,workspace:item.workspace,profilePath:item.profilePath,eventsPath:item.eventsPath,profileSha256:item.profileSha256};}); - if(registrations.length===0)throw new TypeError("invalid engine broker service config");return {credentialHome:input.credentialHome,turnStore:input.turnStore,registrations}; -} - async function readRootConfig(file:string):Promise{const handle=await open(file,constants.O_RDONLY|constants.O_NOFOLLOW);try{const stat=await handle.stat();if(!stat.isFile()||stat.uid!==0||stat.gid!==2100||(stat.mode&0o777)!==0o440||stat.size<2||stat.size>MAX_CONFIG_BYTES)throw new Error("unsafe engine broker service config");const bytes=await handle.readFile();if(bytes.length>MAX_CONFIG_BYTES)throw new Error("unsafe engine broker service config");return JSON.parse(bytes.toString("utf8"));}finally{await handle.close();}} diff --git a/src/runtime/engineBrokerServiceConfig.ts b/src/runtime/engineBrokerServiceConfig.ts new file mode 100644 index 0000000..836963c --- /dev/null +++ b/src/runtime/engineBrokerServiceConfig.ts @@ -0,0 +1,69 @@ +import path from "node:path"; + +import { DEFAULT_GROK_BROKER_TURN_LIMITS, parseEngineBrokerTurnLimits, type EngineBrokerTurnLimits } from "./engineBrokerTurnAccounting.js"; +import { DEFAULT_GROK_BROKER_MODEL_POLICY, GROK_BROKER_MODELS, GROK_BROKER_REASONING_EFFORTS, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; +import { grokWorkerEventsPathFor } from "./grokWorkerSandboxProfile.js"; +import { TURN_USAGE_LEDGER } from "./turnUsageLedger.js"; + +export const ENGINE_BROKER_SERVICE_V1 = "noopolis.daimon.engine-broker-service.v1" as const; +export const ENGINE_BROKER_SERVICE_V2 = "noopolis.daimon.engine-broker-service.v2" as const; + +/** One root-provisioned broker slot. Every field is fixed at provisioning time; a wake can only lower `limits`. */ +export type EngineBrokerServiceRegistration = Readonly<{ + agentId: string; slot: number; workerUid: number; workspace: string; profilePath: string; eventsPath: string; profileSha256: string; + /** Per-slot usage ledger the broker appends turn rows to; per-request rows go to `requests.jsonl` beside it. */ + usageLedgerPath: string; + limits: EngineBrokerTurnLimits; + model: GrokBrokerModelPolicy; +}>; +export type EngineBrokerServiceConfig = Readonly<{ credentialHome: string; turnStore: string; registrations: readonly EngineBrokerServiceRegistration[] }>; + +const V1_REGISTRATION = ["agentId", "slot", "workerUid", "workspace", "profilePath", "eventsPath", "profileSha256"] as const; +const V2_REGISTRATION = [...V1_REGISTRATION, "usageLedgerPath", "limits", "model"] as const; +const invalid = (): TypeError => new TypeError("invalid engine broker service config"); +const plain = (value: unknown): value is Record => value !== null && typeof value === "object" && !Array.isArray(value); +const exact = (value: Record, fields: readonly string[]): void => { if (Object.keys(value).length !== fields.length || fields.some((field) => !Object.hasOwn(value, field))) throw invalid(); }; +const absolute = (item: unknown): item is string => typeof item === "string" && item.startsWith("/") && !item.includes("/../") && !item.endsWith("/..") && !item.includes("\0"); + +/** The per-request stream written beside a registration's usage ledger. */ +export const engineBrokerRequestLedgerPathFor = (usageLedgerPath: string): string => path.posix.join(path.posix.dirname(usageLedgerPath), "requests.jsonl"); + +/** + * Strict `service.json` parser. + * + * v2 requires every registration to declare its usage ledger, limits and model + * (`model.id`/`model.reasoningEffort` from the closed lists); unknown keys at + * any level are refused. v1 is still accepted and receives today's defaults: + * the container ledger, {@link DEFAULT_GROK_BROKER_TURN_LIMITS}, and + * `grok-4.6`/`low`. + */ +export function parseEngineBrokerServiceConfig(value: unknown): EngineBrokerServiceConfig { + if (!plain(value)) throw invalid(); + const v2 = value.version === ENGINE_BROKER_SERVICE_V2; + if (!v2 && value.version !== ENGINE_BROKER_SERVICE_V1) throw invalid(); + exact(value, ["version", "credentialHome", "turnStore", "registrations"]); + if (!absolute(value.credentialHome) || !absolute(value.turnStore) || !Array.isArray(value.registrations) || value.registrations.length === 0) throw invalid(); + const seen = new Set(), slots = new Set(); + const registrations = value.registrations.map((entry: unknown): EngineBrokerServiceRegistration => { + if (!plain(entry)) throw invalid(); + exact(entry, v2 ? V2_REGISTRATION : V1_REGISTRATION); + const { agentId, slot, workerUid, workspace, profilePath, eventsPath, profileSha256 } = entry; + if (typeof agentId !== "string" || !agentId.trim() || seen.has(agentId) || !Number.isSafeInteger(slot) || (slot as number) < 0 || slots.has(slot as number) || !Number.isSafeInteger(workerUid) || (workerUid as number) < 2200 || !absolute(workspace) || !absolute(profilePath) || !absolute(eventsPath) || typeof profileSha256 !== "string" || !/^[a-f0-9]{64}$/u.test(profileSha256) || eventsPath !== grokWorkerEventsPathFor(profilePath) || !profilePath.endsWith("/sandbox.toml")) throw invalid(); + seen.add(agentId); slots.add(slot as number); + const base = { agentId, slot: slot as number, workerUid: workerUid as number, workspace, profilePath, eventsPath, profileSha256 }; + if (!v2) return { ...base, usageLedgerPath: TURN_USAGE_LEDGER.filePath, limits: DEFAULT_GROK_BROKER_TURN_LIMITS, model: DEFAULT_GROK_BROKER_MODEL_POLICY }; + const usageLedgerPath = entry.usageLedgerPath; + if (!absolute(usageLedgerPath) || !usageLedgerPath.endsWith(".jsonl") || usageLedgerPath === engineBrokerRequestLedgerPathFor(usageLedgerPath) || path.posix.normalize(usageLedgerPath) !== usageLedgerPath) throw invalid(); + let limits: EngineBrokerTurnLimits; + try { limits = parseEngineBrokerTurnLimits(entry.limits); } catch { throw invalid(); } + return { ...base, usageLedgerPath, limits, model: parseServiceModel(entry.model) }; + }); + return { credentialHome: value.credentialHome, turnStore: value.turnStore, registrations }; +} + +function parseServiceModel(value: unknown): GrokBrokerModelPolicy { + if (!plain(value)) throw invalid(); + exact(value, ["id", "reasoningEffort"]); + if (!(GROK_BROKER_MODELS as readonly unknown[]).includes(value.id) || !(GROK_BROKER_REASONING_EFFORTS as readonly unknown[]).includes(value.reasoningEffort)) throw invalid(); + return Object.freeze({ model: value.id as GrokBrokerModelPolicy["model"], reasoningEffort: value.reasoningEffort as GrokBrokerModelPolicy["reasoningEffort"] }); +} diff --git a/src/runtime/engineBrokerTurnRegistry.test.ts b/src/runtime/engineBrokerTurnRegistry.test.ts index 16172c7..3de4e20 100644 --- a/src/runtime/engineBrokerTurnRegistry.test.ts +++ b/src/runtime/engineBrokerTurnRegistry.test.ts @@ -5,18 +5,49 @@ import path from "node:path"; import test from "node:test"; import { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; -const start = (prompt = "work") => ({ version: "noopolis.daimon.engine-broker.v1", kind: "start_turn", requestId: "request-1", turnId: "turn-1", agentId: "agent-1", wakeId: "wake-1", prompt,mcpEndpoint:"http://127.0.0.1:4567/mcp" } as const); +const start = (prompt = "work") => ({ version: "noopolis.daimon.engine-broker.v2", kind: "start_turn", requestId: "request-1", turnId: "turn-1", agentId: "agent-1", wakeId: "wake-1", prompt,mcpEndpoint:"http://127.0.0.1:4567/mcp" } as const); test("turn registry replays terminal results across restart and rejects conflicts", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-turns-")); try { - const first = new EngineBrokerTurnRegistry(root,"boot-a"); assert.equal(await first.begin(start()), "start"); - await assert.rejects(first.begin(start()), /already active/); - const response = { version: "noopolis.daimon.engine-broker.v1", kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 11, workerUid: 2200, workerStartTime: "123" } as const; + const first = new EngineBrokerTurnRegistry(root,"boot-a"); assert.equal(await first.begin(start(),"grok-4.6"), "start"); + await assert.rejects(first.begin(start(),"grok-4.6"), /already active/); + const response = { version: "noopolis.daimon.engine-broker.v2", kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 11, workerUid: 2200, workerStartTime: "123", outcome: "completed", usage: { input: 3, cacheRead: 2, cacheWrite: 0, output: 1, total: 6 }, model: "grok-4.6", requests: 1, limitReason: "none" } as const; await first.finish(start(), response); - assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-b").begin(start()), { replay: response }); - await assert.rejects(first.begin(start("different")), /conflict/); + assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-b").begin(start(),"grok-4.6"), { replay: response }); + await assert.rejects(first.begin(start("different"),"grok-4.6"), /conflict/); } finally { await rm(root, { recursive: true, force: true }); } }); -test("turn registry fails an orphaned active turn once after broker restart",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{assert.equal(await new EngineBrokerTurnRegistry(root,"boot-a").begin(start()),"start");const replay=await new EngineBrokerTurnRegistry(root,"boot-b").begin(start());assert.equal(typeof replay,"object");if(typeof replay==="object")assert.equal(replay.replay.kind,"failed");assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-c").begin(start()),replay);}finally{await rm(root,{recursive:true,force:true});}}); -test("turn registry durably replays a sanitized pre-attestation failure",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{const registry=new EngineBrokerTurnRegistry(root,"boot-a");assert.equal(await registry.begin(start()),"start");const response={version:"noopolis.daimon.engine-broker.v1",kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",diagnostic:{status:"worker_failed",stage:"attestation",failureClass:"profile_missing",profileApplied:false,exitCode:0,termSignal:0,workerPid:42,workerUid:2200,startTicks:"123"}} as const;await registry.finish(start(),response);assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-b").begin(start()),{replay:response});}finally{await rm(root,{recursive:true,force:true});}}); -test("turn registry rejects a persisted diagnostic with undeclared secret-bearing fields",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{const registry=new EngineBrokerTurnRegistry(root,"boot-a");assert.equal(await registry.begin(start()),"start");const [name]=await readdir(root);const file=path.join(root,name!);const record=JSON.parse(await readFile(file,"utf8")) as Record;record.state="terminal";record.response={version:"noopolis.daimon.engine-broker.v1",kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",rawOutput:"secret"};await writeFile(file,JSON.stringify(record));await assert.rejects(new EngineBrokerTurnRegistry(root,"boot-b").begin(start()),/invalid broker frame/u);}finally{await rm(root,{recursive:true,force:true});}}); +test("turn registry fails an orphaned active turn once after broker restart",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{assert.equal(await new EngineBrokerTurnRegistry(root,"boot-a").begin(start(),"grok-4.6"),"start");const replay=await new EngineBrokerTurnRegistry(root,"boot-b").begin(start(),"grok-4.6");assert.equal(typeof replay,"object");if(typeof replay==="object")assert.equal(replay.replay.kind,"failed");assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-c").begin(start(),"grok-4.6"),replay);}finally{await rm(root,{recursive:true,force:true});}}); +test("turn registry durably replays a sanitized pre-attestation failure",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{const registry=new EngineBrokerTurnRegistry(root,"boot-a");assert.equal(await registry.begin(start(),"grok-4.6"),"start");const response={version:"noopolis.daimon.engine-broker.v2",kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",diagnostic:{status:"worker_failed",stage:"attestation",failureClass:"profile_missing",profileApplied:false,exitCode:0,termSignal:0,workerPid:42,workerUid:2200,startTicks:"123"},outcome:"failed",usage:null,model:"grok-4.6",requests:0,limitReason:"none"} as const;await registry.finish(start(),response);assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-b").begin(start(),"grok-4.6"),{replay:response});}finally{await rm(root,{recursive:true,force:true});}}); +test("turn registry rejects a persisted diagnostic with undeclared secret-bearing fields",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{const registry=new EngineBrokerTurnRegistry(root,"boot-a");assert.equal(await registry.begin(start(),"grok-4.6"),"start");const [name]=await readdir(root);const file=path.join(root,name!);const record=JSON.parse(await readFile(file,"utf8")) as Record;record.state="terminal";record.response={version:"noopolis.daimon.engine-broker.v2",kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",rawOutput:"secret",outcome:"failed",usage:null,model:"grok-4.6",requests:0,limitReason:"none"};await writeFile(file,JSON.stringify(record));await assert.rejects(new EngineBrokerTurnRegistry(root,"boot-b").begin(start(),"grok-4.6"),/registry unavailable/u);}finally{await rm(root,{recursive:true,force:true});}}); + +const withRoot = async (run: (root: string) => Promise): Promise => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-turns-")); try { await run(root); } finally { await rm(root, { recursive: true, force: true }); } }; +const recordFile = async (root: string): Promise => { const [name] = await readdir(root); return path.join(root, name!); }; + +test("a v1 record sealed before the upgrade still replays, upgraded with no usage and never re-metered", async () => { + await withRoot(async (root) => { + const registry = new EngineBrokerTurnRegistry(root, "boot-a"); + assert.equal(await registry.begin(start(), "grok-4.5"), "start"); + const file = await recordFile(root); const record = JSON.parse(await readFile(file, "utf8")) as Record; + const v1 = { version: "noopolis.daimon.engine-broker.v1", kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 11, workerUid: 2200, workerStartTime: "123" }; + await writeFile(file, JSON.stringify({ version: "noopolis.daimon.engine-broker-turn.v1", digest: record.digest, state: "terminal", bootId: "boot-a", response: v1 })); + assert.deepEqual(await new EngineBrokerTurnRegistry(root, "boot-b").begin(start(), "grok-4.5"), { replay: { ...v1, version: "noopolis.daimon.engine-broker.v2", outcome: "completed", usage: null, model: "grok-4.5", requests: 0, limitReason: "none" } }); + }); +}); + +test("the v2 record parser is strict: an unknown member or a v1 frame inside a v2 record is refused", async () => { + await withRoot(async (root) => { + const registry = new EngineBrokerTurnRegistry(root, "boot-a"); + assert.equal(await registry.begin(start(), "grok-4.6"), "start"); + const response = { version: "noopolis.daimon.engine-broker.v2", kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 11, workerUid: 2200, workerStartTime: "123", outcome: "completed", usage: null, model: "grok-4.6", requests: 0, limitReason: "none" } as const; + await registry.finish(start(), response); + const file = await recordFile(root); const record = JSON.parse(await readFile(file, "utf8")) as Record; + assert.equal(record.version, "noopolis.daimon.engine-broker-turn.v2"); + // Mutation guard: dropping the exact-member check accepts this record. + await writeFile(file, JSON.stringify({ ...record, usageRow: "extra" })); + await assert.rejects(new EngineBrokerTurnRegistry(root, "boot-b").begin(start(), "grok-4.6"), /registry unavailable/u); + const { outcome: _o, usage: _u, model: _m, requests: _r, limitReason: _l, ...legacy } = response; + await writeFile(file, JSON.stringify({ ...record, response: { ...legacy, version: "noopolis.daimon.engine-broker.v1" } })); + await assert.rejects(new EngineBrokerTurnRegistry(root, "boot-b").begin(start(), "grok-4.6"), /registry unavailable/u); + }); +}); diff --git a/src/runtime/engineBrokerTurnRegistry.ts b/src/runtime/engineBrokerTurnRegistry.ts index bdee3c3..1ddd01c 100644 --- a/src/runtime/engineBrokerTurnRegistry.ts +++ b/src/runtime/engineBrokerTurnRegistry.ts @@ -2,33 +2,76 @@ import { createHash, randomUUID } from "node:crypto"; import { constants } from "node:fs"; import { mkdir, open, readFile, rename, unlink } from "node:fs/promises"; import path from "node:path"; -import { parseEngineBrokerResponse,type EngineBrokerRequest, type EngineBrokerResponse } from "./engineBrokerProtocol.js"; +import { parseEngineBrokerResponse, parseEngineBrokerV1TerminalResponse, type EngineBrokerRequest, type EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; +import type { GrokBrokerModel } from "./grokBrokerModelPolicy.js"; type Start = Extract; -type Terminal = Extract; -type Record = { version: "noopolis.daimon.engine-broker-turn.v1"; digest: string; state: "active" | "terminal"; bootId: string; response?: Terminal }; +type Terminal = EngineBrokerTerminalResponse; +export const ENGINE_BROKER_TURN_RECORD_V1 = "noopolis.daimon.engine-broker-turn.v1" as const; +export const ENGINE_BROKER_TURN_RECORD_V2 = "noopolis.daimon.engine-broker-turn.v2" as const; +// The digest deliberately excludes `limits` and the protocol version, so a v1 +// record written before the upgrade still identifies the same turn. const digest = (request: Start): string => createHash("sha256").update(JSON.stringify([request.turnId, request.agentId, request.wakeId, request.prompt,request.mcpEndpoint])).digest("hex"); const safe = (turnId: string): string => `${createHash("sha256").update(turnId).digest("hex")}.json`; +type Observed = Readonly<{ version: typeof ENGINE_BROKER_TURN_RECORD_V1 | typeof ENGINE_BROKER_TURN_RECORD_V2; digest: string; state: "active" | "terminal"; bootId: string; response?: Terminal }>; +/** + * Durable per-turn state. Record v2 stores the terminal response *with* its + * sealed accounting (usage, outcome, model, requests, limitReason), so a replay + * returns exactly what was metered and never meters again: metering happens + * only on the path that returned `"start"`. + */ export class EngineBrokerTurnRegistry { constructor(private readonly root: string,private readonly bootId:string=randomUUID()) {} - async begin(request: Start): Promise<"start" | { replay: Terminal }> { + /** `model` is the registration's declared model, used only to upgrade a v1 record on replay. */ + async begin(request: Start, model: GrokBrokerModel): Promise<"start" | { replay: Terminal }> { await mkdir(this.root, { recursive: true, mode: 0o700 }); const file = path.join(this.root, safe(request.turnId)); const expected = digest(request); - try { const handle = await open(file, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); try { await handle.writeFile(`${JSON.stringify({ version: "noopolis.daimon.engine-broker-turn.v1", digest: expected, state: "active",bootId:this.bootId })}\n`); await handle.sync(); } finally { await handle.close(); } await syncDirectory(this.root); return "start"; } + try { const handle = await open(file, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); try { await handle.writeFile(`${JSON.stringify({ version: ENGINE_BROKER_TURN_RECORD_V2, digest: expected, state: "active",bootId:this.bootId })}\n`); await handle.sync(); } finally { await handle.close(); } await syncDirectory(this.root); return "start"; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw new Error("broker turn registry unavailable"); } - const observed = JSON.parse(await readFile(file, "utf8")) as Record; - if (observed.version !== "noopolis.daimon.engine-broker-turn.v1" || observed.digest !== expected) throw new Error("broker turn conflict"); - if (observed.state === "terminal" && observed.response !== undefined) { const response=parseEngineBrokerResponse(observed.response);if(response.kind!=="completed"&&response.kind!=="failed")throw new Error("broker turn registry unavailable");return { replay: response }; } - if(observed.state==="active"&&observed.bootId!==this.bootId){const response={version:request.version,kind:"failed",requestId:request.requestId,turnId:request.turnId,code:"engine_failed"} as const;await this.finish(request,response);return {replay:response};} + const observed = parseEngineBrokerTurnRecord(await readFile(file, "utf8"), model); + if (observed.digest !== expected) throw new Error("broker turn conflict"); + if (observed.state === "terminal" && observed.response !== undefined) return { replay: observed.response }; + if(observed.state==="active"&&observed.bootId!==this.bootId){const response={version:request.version,kind:"failed",requestId:request.requestId,turnId:request.turnId,code:"engine_failed",outcome:"failed",usage:null,model,requests:0,limitReason:"none"} as const;await this.finish(request,response);return {replay:response};} throw new Error("broker turn already active"); } async finish(request: Start, response: Terminal): Promise { const file = path.join(this.root, safe(request.turnId)); const temporary = `${file}.${randomUUID()}.tmp`; const handle = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); - try { await handle.writeFile(`${JSON.stringify({ version: "noopolis.daimon.engine-broker-turn.v1", digest: digest(request), state: "terminal",bootId:this.bootId, response })}\n`); await handle.sync(); } finally { await handle.close(); } + try { await handle.writeFile(`${JSON.stringify({ version: ENGINE_BROKER_TURN_RECORD_V2, digest: digest(request), state: "terminal",bootId:this.bootId, response })}\n`); await handle.sync(); } finally { await handle.close(); } try { await rename(temporary, file); await syncDirectory(this.root); } finally { await unlink(temporary).catch(() => undefined); } } } +/** + * Strict record parser. v2 accepts exactly `{version,digest,state,bootId}` + * plus `response` when terminal, and the response must be a v2 terminal frame. + * v1 records keep their historical looser shape and are upgraded on read: no + * usage (`null`), zero requests, `limitReason: "none"`, the declared model. + */ +export function parseEngineBrokerTurnRecord(text: string, model: GrokBrokerModel): Observed { + let value: unknown; + try { value = JSON.parse(text); } catch { throw new Error("broker turn registry unavailable"); } + if (value === null || typeof value !== "object" || Array.isArray(value)) throw new Error("broker turn registry unavailable"); + const input = value as Record; + if (typeof input.digest !== "string" || !/^[a-f0-9]{64}$/u.test(input.digest) || typeof input.bootId !== "string" || (input.state !== "active" && input.state !== "terminal")) throw new Error("broker turn registry unavailable"); + const base = { digest: input.digest, state: input.state, bootId: input.bootId } as const; + if (input.version === ENGINE_BROKER_TURN_RECORD_V1) { + if (input.state !== "terminal" || input.response === undefined) return { version: ENGINE_BROKER_TURN_RECORD_V1, ...base }; + let legacy; + try { legacy = parseEngineBrokerV1TerminalResponse(input.response); } catch { throw new Error("broker turn registry unavailable"); } + const accounting = { outcome: legacy.kind, usage: null, model, requests: 0, limitReason: "none" } as const; + const response: Terminal = { ...legacy, ...accounting, version: "noopolis.daimon.engine-broker.v2" } as Terminal; + return { version: ENGINE_BROKER_TURN_RECORD_V1, ...base, response }; + } + if (input.version !== ENGINE_BROKER_TURN_RECORD_V2) throw new Error("broker turn conflict"); + const fields = input.state === "terminal" ? ["version", "digest", "state", "bootId", "response"] : ["version", "digest", "state", "bootId"]; + if (Object.keys(input).length !== fields.length || fields.some((field) => !Object.hasOwn(input, field))) throw new Error("broker turn registry unavailable"); + if (input.state === "active") return { version: ENGINE_BROKER_TURN_RECORD_V2, ...base }; + let response; + try { response = parseEngineBrokerResponse(input.response); } catch { throw new Error("broker turn registry unavailable"); } + if (response.kind !== "completed" && response.kind !== "failed") throw new Error("broker turn registry unavailable"); + return { version: ENGINE_BROKER_TURN_RECORD_V2, ...base, response }; +} + async function syncDirectory(directory: string): Promise { const handle = await open(directory, constants.O_RDONLY); try { await handle.sync(); } finally { await handle.close(); } } diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index 6c541b6..0e6e6de 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -3,6 +3,14 @@ import { request as httpRequest } from "node:http"; import test from "node:test"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; import { GROK_SESSION_TITLE_SINK_KEY } from "./grokBrokerWorkerConfig.js"; +import { GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; + +type Proxy = Awaited>; +const arm = (proxy: Proxy, guard: () => Promise, meter = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 })): GrokBrokerTurnMeter => { + proxy.registerIsolationGuard("turn", guard); + proxy.registerTurn("turn", { policy: { model: "grok-4.6", reasoningEffort: "low" }, meter }); + return meter; +}; const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); const leanBody = (overrides: Record = {}): string => JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: lean, ...overrides }); @@ -13,29 +21,29 @@ test("proxy retries one 401 with refreshed broker bearer and shuts down", async calls.push(request.headers.authorization); return calls.length === 1 ? { status: 401, headers: { "content-type": "application/json" }, body: new Uint8Array() } : { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from("data: done\n\n") }; }); const token = proxy.capabilities.issue("agent", "turn"); - proxy.registerIsolationGuard("turn", async () => undefined); + arm(proxy, async () => undefined); const result = await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`, { method: "POST", headers: { authorization: `Bearer ${token}`, "content-type": "application/json", "x-grok-client-version": "1.0.34" }, body: leanBody() }); assert.equal(result.status, 200); assert.equal(await result.text(), "data: done\n\n"); assert.deepEqual(calls, ["Bearer first", "Bearer second"]); assert.equal(refreshes, 0); await proxy.close(); await assert.rejects(fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`)); }); -test("proxy stale-fences a refreshed credential rejected by upstream",async()=>{let rejected=0;const proxy=await startGrokBrokerProxy({accessToken:async(force)=>force?"second":"first",markRejected:async()=>{rejected++;throw new Error("stale");}},async()=>({status:401,headers:{"content-type":"application/json"},body:new Uint8Array()}));const token=proxy.capabilities.issue("agent","turn");proxy.registerIsolationGuard("turn",async()=>undefined);const response=await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`,{method:"POST",headers:{authorization:`Bearer ${token}`,"x-grok-client-version":"1.0.34"},body:leanBody()});assert.equal(response.status,503);assert.equal(rejected,1);await proxy.close();}); +test("proxy stale-fences a refreshed credential rejected by upstream",async()=>{let rejected=0;const proxy=await startGrokBrokerProxy({accessToken:async(force)=>force?"second":"first",markRejected:async()=>{rejected++;throw new Error("stale");}},async()=>({status:401,headers:{"content-type":"application/json"},body:new Uint8Array()}));const token=proxy.capabilities.issue("agent","turn");arm(proxy, async()=>undefined);const response=await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`,{method:"POST",headers:{authorization:`Bearer ${token}`,"x-grok-client-version":"1.0.34"},body:leanBody()});assert.equal(response.status,503);assert.equal(rejected,1);await proxy.close();}); test("proxy failures expose only a fixed diagnostic", async () => { const proxy = await startGrokBrokerProxy({ accessToken: async () => { throw new Error("secret-token"); }, markRejected: async () => undefined }, async () => { throw new Error("unreachable"); }); const token = proxy.capabilities.issue("agent", "turn"); - proxy.registerIsolationGuard("turn", async () => undefined); + arm(proxy, async () => undefined); const result = await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`, { method: "POST", headers: { authorization: `Bearer ${token}`, "x-grok-client-version": "1.0.34" }, body: leanBody() }); assert.equal(result.status, 503); const body = await result.text(); assert.equal(body, '{"error":"broker unavailable"}'); assert.doesNotMatch(body, /secret/u); await proxy.close(); }); -test("one turn capability supports multiple guarded cognition requests",async()=>{let guarded=0,calls=0;const proxy=await startGrokBrokerProxy({accessToken:async()=>"provider-token",markRejected:async()=>undefined},async()=>{calls++;return{status:200,headers:{"content-type":"application/json"},body:Buffer.from("{}")};});try{const token=proxy.capabilities.issue("agent","turn");proxy.registerIsolationGuard("turn",async()=>{guarded++;});for(let index=0;index<2;index++){const response=await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`,{method:"POST",headers:{authorization:`Bearer ${token}`,"x-grok-client-version":"1.0.34"},body:leanBody()});assert.equal(response.status,200);}assert.equal(calls,2);assert.equal(guarded,2);}finally{await proxy.close();}}); +test("one turn capability supports multiple guarded cognition requests",async()=>{let guarded=0,calls=0;const proxy=await startGrokBrokerProxy({accessToken:async()=>"provider-token",markRejected:async()=>undefined},async()=>{calls++;return{status:200,headers:{"content-type":"application/json"},body:Buffer.from("{}")};});try{const token=proxy.capabilities.issue("agent","turn");arm(proxy, async()=>{guarded++;});for(let index=0;index<2;index++){const response=await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`,{method:"POST",headers:{authorization:`Bearer ${token}`,"x-grok-client-version":"1.0.34"},body:leanBody()});assert.equal(response.status,200);}assert.equal(calls,2);assert.equal(guarded,2);}finally{await proxy.close();}}); test("proxy refuses a fail-open tool set or an undeclared effort without calling upstream", async () => { let calls = 0; let accessed = 0; const proxy = await startGrokBrokerProxy({ accessToken: async () => { accessed++; return "provider-token"; }, markRejected: async () => undefined }, async () => { calls++; return { status: 200, headers: { "content-type": "application/json" }, body: Buffer.from("{}") }; }, { model: "grok-4.6", reasoningEffort: "low" }); try { - const token = proxy.capabilities.issue("agent", "turn"); proxy.registerIsolationGuard("turn", async () => undefined); + const token = proxy.capabilities.issue("agent", "turn"); arm(proxy, async () => undefined); const full = [...lean, ...["search_replace", "todo_write", "write", "monitor"].map((name) => ({ type: "function", function: { name } }))]; for (const payload of [leanBody({ tools: full }), leanBody({ tools: [{ type: "function", function: { name: "session_title" } }] }), leanBody({ reasoning_effort: "high" }), leanBody({ reasoning_effort: undefined })]) { assert.equal(await post(proxy.port, token, payload), 503); @@ -58,7 +66,7 @@ test("the session-title sink is refused before capability, guard, credential, or let calls = 0, accessed = 0, guarded = 0; const proxy = await startGrokBrokerProxy({ accessToken: async () => { accessed++; return "provider-token"; }, markRejected: async () => undefined }, async () => { calls++; return { status: 200, headers: { "content-type": "application/json" }, body: Buffer.from("{}") }; }); try { - const token = proxy.capabilities.issue("agent", "turn", 60_000, 1); proxy.registerIsolationGuard("turn", async () => { guarded++; }); + const token = proxy.capabilities.issue("agent", "turn", 60_000, 1); arm(proxy, async () => { guarded++; }); const title = JSON.stringify({ model: "disabled", max_tokens: 100, temperature: 0, stream: true, messages: [{ role: "user", content: "prompt-derived" }], tool_choice: { type: "function", function: { name: "session_title" } }, tools: [{ type: "function", function: { name: "session_title" } }] }); assert.equal(await post(proxy.port, GROK_SESSION_TITLE_SINK_KEY, title), 503); assert.deepEqual({ calls, accessed, guarded }, { calls: 0, accessed: 0, guarded: 0 }); @@ -73,7 +81,7 @@ test("the isolation guard is awaited before the first upstream call, and a faili const proxy = await startGrokBrokerProxy({ accessToken: async () => { order.push("credential"); return "provider-token"; }, markRejected: async () => undefined }, async () => { upstreamCalls++; order.push("upstream"); return { status: 200, headers: { "content-type": "application/json" }, body: Buffer.from("{}") }; }); try { const token = proxy.capabilities.issue("agent", "turn"); - proxy.registerIsolationGuard("turn", async () => { + arm(proxy, async () => { order.push("guard-start"); await new Promise((resolve) => setTimeout(resolve, 30)); order.push("guard-end"); if (fail) throw new Error("no enforcement evidence"); }); diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index 023a474..4f53103 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -4,28 +4,44 @@ import type { AddressInfo } from "node:net"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; import { DEFAULT_GROK_BROKER_MODEL_POLICY, parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; import { authorizeGrokBrokerProxyRequest } from "./grokBrokerProxyRequest.js"; +import { parseGrokUpstreamUsage, type GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; + +/** One running turn as the proxy sees it: its declared model/effort and its spend gate. */ +export type GrokBrokerProxyTurn = Readonly<{ policy: GrokBrokerModelPolicy; meter: GrokBrokerTurnMeter }>; export type GrokBrokerCredentialAuthority = Readonly<{ accessToken(forceRefresh: boolean): Promise; refreshAfterRejection?(rejectedTokenDigest:string):Promise; markRejected(rejectedTokenDigest?:string): Promise }>; export type GrokBrokerUpstream = (request: ReturnType) => Promise>; body: Uint8Array }>>; -/** `policy` is the declared model/effort every forwarded body must carry (closed list; defaults grok-4.6/low). */ -export async function startGrokBrokerProxy(authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream = defaultUpstream, policy: GrokBrokerModelPolicy = DEFAULT_GROK_BROKER_MODEL_POLICY): PromisePromise):void; revokeIsolationGuard(turnId:string):void; close(): Promise }>> { +/** + * `policy` is the fallback declared model/effort (closed list); a registered + * turn's own policy wins. A request whose turn has no registered meter is + * refused like one without an isolation guard: nothing is forwarded unmetered. + * `listenPort` exists for tests that must not contend for the production port. + */ +export async function startGrokBrokerProxy(authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream = defaultUpstream, policy: GrokBrokerModelPolicy = DEFAULT_GROK_BROKER_MODEL_POLICY, listenPort = 43_123): PromisePromise):void; revokeIsolationGuard(turnId:string):void; registerTurn(turnId:string,turn:GrokBrokerProxyTurn):void; revokeTurn(turnId:string):void; close(): Promise }>> { const declared = parseGrokBrokerModelPolicy(policy); const capabilities = new EngineBrokerCapabilities(); - const guards=new MapPromise>();const server = createServer((request, response) => { void serve(request, response, authority, upstream, capabilities,guards,declared); }); - await new Promise((resolve, reject) => { server.once("error", reject); server.listen(43_123, "127.0.0.1", () => { server.off("error", reject); resolve(); }); }); + const guards=new MapPromise>();const turns=new Map();const server = createServer((request, response) => { void serve(request, response, authority, upstream, capabilities,guards,turns,declared); }); + await new Promise((resolve, reject) => { server.once("error", reject); server.listen(listenPort, "127.0.0.1", () => { server.off("error", reject); resolve(); }); }); const address = server.address() as AddressInfo; - return { port: address.port, capabilities,registerIsolationGuard(turnId,guard){guards.set(turnId,guard);},revokeIsolationGuard(turnId){guards.delete(turnId);}, close: () => new Promise((resolve, reject) => server.close((error) => error === undefined ? resolve() : reject(error))) }; + return { port: address.port, capabilities,registerIsolationGuard(turnId,guard){guards.set(turnId,guard);},revokeIsolationGuard(turnId){guards.delete(turnId);},registerTurn(turnId,turn){turns.set(turnId,{policy:parseGrokBrokerModelPolicy(turn.policy),meter:turn.meter});},revokeTurn(turnId){turns.delete(turnId);}, close: () => new Promise((resolve, reject) => server.close((error) => error === undefined ? resolve() : reject(error))) }; } -async function serve(request: IncomingMessage, response: ServerResponse, authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream, capabilities: EngineBrokerCapabilities,guards:MapPromise>,policy:GrokBrokerModelPolicy): Promise { +async function serve(request: IncomingMessage, response: ServerResponse, authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream, capabilities: EngineBrokerCapabilities,guards:MapPromise>,turns:Map,fallback:GrokBrokerModelPolicy): Promise { + let settle:((usage:ReturnType)=>void)|undefined; try { const body = await readBody(request); const headers = Object.fromEntries(Object.entries(request.headers).map(([key, value]) => [key, Array.isArray(value) ? value[0] : value])); - const match=headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u),scope=match?capabilities.inspectToken(match[1]!):undefined;if(!scope)throw new Error();const guard=guards.get(scope.turnId);if(!guard)throw new Error();await guard(); - let token = await authority.accessToken(false);const rejectedDigest=createHash("sha256").update(token).digest("hex"); let prepared = authorizeGrokBrokerProxyRequest({ method: request.method ?? "", pathname: new URL(request.url ?? "/", "http://127.0.0.1").pathname, headers, body }, capabilities, token, policy); token = ""; + const match=headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u),scope=match?capabilities.inspectToken(match[1]!):undefined;if(!scope)throw new Error();const guard=guards.get(scope.turnId),turn=turns.get(scope.turnId);if(!guard||!turn)throw new Error();await guard(); + let token = await authority.accessToken(false);const rejectedDigest=createHash("sha256").update(token).digest("hex"); let prepared = authorizeGrokBrokerProxyRequest({ method: request.method ?? "", pathname: new URL(request.url ?? "/", "http://127.0.0.1").pathname, headers, body }, capabilities, token, turn.policy ?? fallback); token = ""; + // The spend gate runs after the body is proven a real lean worker request + // (a refused session-title body never counts) and before any upstream call. + const admission=turn.meter.admit(); + if("refused" in admission){response.writeHead(429,{"content-type":"application/json","cache-control":"no-store"});response.end(JSON.stringify({error:"turn limit reached",limit:admission.refused}));return;} + settle=(usage)=>{turn.meter.settle(admission.index,usage);settle=undefined;}; let result = await upstream(prepared); if (result.status === 401) { token = authority.refreshAfterRejection?await authority.refreshAfterRejection(rejectedDigest):await authority.accessToken(true);const refreshedDigest=createHash("sha256").update(token).digest("hex"); prepared = { ...prepared, headers: { ...prepared.headers, authorization: `Bearer ${token}` } }; token = ""; result = await upstream(prepared);if(result.status===401)await authority.markRejected(refreshedDigest); } + settle?.(parseGrokUpstreamUsage(result.body,result.headers["content-type"])); response.writeHead(result.status, { "content-type": result.headers["content-type"] ?? "application/json", "cache-control": "no-store" }); response.end(result.body); - } catch { response.writeHead(503, { "content-type": "application/json", "cache-control": "no-store" }); response.end('{"error":"broker unavailable"}'); } + } catch { settle?.(undefined); response.writeHead(503, { "content-type": "application/json", "cache-control": "no-store" }); response.end('{"error":"broker unavailable"}'); } } async function readBody(request: IncomingMessage): Promise { const chunks: Buffer[] = []; let bytes = 0; for await (const chunk of request) { const value = Buffer.from(chunk); bytes += value.length; if (bytes > 2 * 1024 * 1024) throw new Error("too large"); chunks.push(value); } return Buffer.concat(chunks); } const defaultUpstream: GrokBrokerUpstream = async (request) => { const result = await fetch(request.url, { method: "POST", headers: request.headers, body: Buffer.from(request.body) }); return { status: result.status, headers: { "content-type": result.headers.get("content-type") ?? "application/json" }, body: new Uint8Array(await result.arrayBuffer()) }; }; diff --git a/src/runtime/grokBrokerTurnMeter.test.ts b/src/runtime/grokBrokerTurnMeter.test.ts new file mode 100644 index 0000000..04c0a6a --- /dev/null +++ b/src/runtime/grokBrokerTurnMeter.test.ts @@ -0,0 +1,103 @@ +import assert from "node:assert/strict"; +import { request as httpRequest } from "node:http"; +import test from "node:test"; + +import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; +import { GrokBrokerTurnMeter, parseGrokUpstreamUsage } from "./grokBrokerTurnMeter.js"; + +const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); +const body = JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: lean }); +const sse = (usage: Record): Uint8Array => Buffer.from([{ choices: [{ index: 0, delta: { content: "x" } }] }, { choices: [], usage }].map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("") + "data: [DONE]\n\n"); + +function post(port: number, token: string): Promise> { + return new Promise((resolve, reject) => { + const req = httpRequest({ host: "127.0.0.1", port, path: "/v1/chat/completions", method: "POST", agent: false, headers: { authorization: `Bearer ${token}`, "x-grok-client-version": "1.0.34", "content-type": "application/json" } }, (response) => { + const chunks: Buffer[] = []; response.on("data", (chunk: Buffer) => chunks.push(chunk)); response.on("end", () => resolve({ status: response.statusCode ?? 0, text: Buffer.concat(chunks).toString("utf8") })); + }); + req.on("error", reject); req.end(body); + }); +} + +const withProxy = async (usage: Record | undefined, meter: GrokBrokerTurnMeter, run: (post: () => Promise>, calls: () => number) => Promise): Promise => { + let calls = 0; + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; return { status: 200, headers: { "content-type": "text/event-stream" }, body: usage === undefined ? Buffer.from("data: [DONE]\n\n") : sse(usage) }; }, undefined, 0); + try { + const token = proxy.capabilities.issue("agent", "turn"); + proxy.registerIsolationGuard("turn", async () => undefined); + proxy.registerTurn("turn", { policy: { model: "grok-4.6", reasoningEffort: "low" }, meter }); + await run(() => post(proxy.port, token), () => calls); + } finally { await proxy.close(); } +}; + +test("maxRequests is hard: request N+1 is refused with 429 before any upstream call, and the limit trips once", async () => { + const tripped: string[] = []; + const meter = new GrokBrokerTurnMeter({ maxRequests: 2, maxTokens: 1_000_000, timeoutMs: 60_000 }, (reason) => tripped.push(reason)); + await withProxy({ prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, meter, async (send, calls) => { + assert.equal((await send()).status, 200); + assert.equal((await send()).status, 200); + const refused = await send(); + assert.equal(refused.status, 429); + assert.deepEqual(JSON.parse(refused.text), { error: "turn limit reached", limit: "requests" }); + assert.equal((await send()).status, 429); + assert.equal(calls(), 2, "upstream never sees a request past maxRequests"); + }); + assert.deepEqual(tripped, ["requests"]); + const snapshot = meter.snapshot(); + assert.equal(snapshot.requests, 2); + assert.equal(snapshot.limitReason, "requests"); + assert.deepEqual(snapshot.usage, { input: 20, cacheRead: 0, cacheWrite: 0, output: 10, total: 30 }); +}); + +test("the token ceiling overshoots by at most one request, counting cached input", async () => { + // 60 tokens per request (40 cached) against a 100-token ceiling: requests 1 and + // 2 are admitted (0 and 60 < 100 before each), request 3 is refused at 120. + // Mutation guard: checking the ceiling after forwarding, or ignoring cached + // tokens, admits a third request and this goes red. + const meter = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 100, timeoutMs: 60_000 }); + await withProxy({ prompt_tokens: 50, completion_tokens: 10, total_tokens: 60, prompt_tokens_details: { cached_tokens: 40 } }, meter, async (send, calls) => { + const statuses = []; + for (let index = 0; index < 5; index++) statuses.push((await send()).status); + assert.deepEqual(statuses, [200, 200, 429, 429, 429]); + assert.equal(calls(), 2); + }); + const snapshot = meter.snapshot(); + assert.equal(snapshot.limitReason, "tokens"); + assert.equal(snapshot.tokens, 120); + assert.ok(snapshot.tokens - meter.limits.maxTokens <= 60, "overshoot is bounded by the last admitted request"); + assert.deepEqual(snapshot.usage, { input: 20, cacheRead: 80, cacheWrite: 0, output: 20, total: 120 }); +}); + +test("a request after the elapsed deadline is refused, and every admitted request is timed", async () => { + let now = 1_000_000; + const meter = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 1_000_000, timeoutMs: 5_000 }, undefined, () => now); + await withProxy(undefined, meter, async (send, calls) => { + assert.equal((await send()).status, 200); + now += 5_000; + assert.equal((await send()).status, 429); + assert.equal(calls(), 1); + }); + const snapshot = meter.snapshot(); + assert.equal(snapshot.limitReason, "timeout"); + assert.equal(snapshot.usage, null, "a body without usage contributes no invented zero"); + assert.deepEqual(snapshot.timings, [{ startedAt: new Date(1_000_000).toISOString(), endedAt: new Date(1_000_000).toISOString() }]); +}); + +test("a turn without a registered meter is never forwarded", async () => { + let calls = 0; + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; return { status: 200, headers: {}, body: Buffer.from("{}") }; }, undefined, 0); + try { + const token = proxy.capabilities.issue("agent", "turn"); + proxy.registerIsolationGuard("turn", async () => undefined); + assert.equal((await post(proxy.port, token)).status, 503); + assert.equal(calls, 0); + } finally { await proxy.close(); } +}); + +test("upstream usage parsing takes the last usage block and never zero-fills", () => { + assert.deepEqual(parseGrokUpstreamUsage(sse({ prompt_tokens: 100, completion_tokens: 7, total_tokens: 120, prompt_tokens_details: { cached_tokens: 30 }, completion_tokens_details: { reasoning_tokens: 13 } }), "text/event-stream"), + { input: 70, cacheRead: 30, cacheWrite: 0, output: 20, total: 120, reasoning: 13 }); + assert.deepEqual(parseGrokUpstreamUsage(Buffer.from(JSON.stringify({ usage: { prompt_tokens: 4, completion_tokens: 1 } })), "application/json"), { input: 4, cacheRead: 0, cacheWrite: 0, output: 1, total: 5 }); + assert.equal(parseGrokUpstreamUsage(sse({ prompt_tokens: "4", completion_tokens: 1 }), "text/event-stream"), undefined); + assert.equal(parseGrokUpstreamUsage(sse({ prompt_tokens: 4, completion_tokens: 1, prompt_tokens_details: { cached_tokens: 9 } }), "text/event-stream"), undefined); + assert.equal(parseGrokUpstreamUsage(Buffer.from("not json"), "application/json"), undefined); +}); diff --git a/src/runtime/grokBrokerTurnMeter.ts b/src/runtime/grokBrokerTurnMeter.ts new file mode 100644 index 0000000..203a9be --- /dev/null +++ b/src/runtime/grokBrokerTurnMeter.ts @@ -0,0 +1,118 @@ +import { sumEngineBrokerTurnUsage, type EngineBrokerLimitReason, type EngineBrokerTurnLimits, type EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; + +export type GrokBrokerRequestTiming = Readonly<{ startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage }>; +export type GrokBrokerTurnMeterSnapshot = Readonly<{ requests: number; tokens: number; limitReason: EngineBrokerLimitReason; usage: EngineBrokerTurnUsage | null; timings: readonly GrokBrokerRequestTiming[] }>; + +/** + * The proxy's per-turn spend gate. + * + * Every forwarded model request of one turn passes through {@link admit} + * *before* a bearer is attached, so the checks are hard for requests and + * elapsed time and between-requests for tokens: + * + * - `maxRequests`: request `maxRequests + 1` is refused; upstream never sees it. + * - `timeoutMs`: a request arriving after the deadline is refused (the broker's + * own timer additionally kills a worker that is mid-request). + * - `maxTokens`: checked against the running total of upstream-reported usage + * of the requests already answered. A request is admitted while that total is + * still below the ceiling, so the overshoot is bounded by exactly one + * request's usage — the last admitted one. Usage counts total input + * *including* cached tokens (P0 observed an uncached replay at +55%). + * + * The first limit that fires is sticky: every later request is refused with + * the same reason, and `onLimit` runs once. + */ +export class GrokBrokerTurnMeter { + private readonly startedAt: number; + private readonly timings: { startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage }[] = []; + private tokens = 0; + private reason: EngineBrokerLimitReason = "none"; + constructor(readonly limits: EngineBrokerTurnLimits, private readonly onLimit: (reason: Exclude) => void = () => undefined, private readonly now: () => number = Date.now) { + this.startedAt = now(); + } + + /** Returns the request index when admitted, or the limit that refused it. */ + admit(): Readonly<{ index: number } | { refused: Exclude }> { + if (this.reason === "none") { + if (this.now() - this.startedAt >= this.limits.timeoutMs) this.trip("timeout"); + else if (this.timings.length >= this.limits.maxRequests) this.trip("requests"); + else if (this.tokens >= this.limits.maxTokens) this.trip("tokens"); + } + if (this.reason !== "none") return { refused: this.reason }; + this.timings.push({ startedAt: new Date(this.now()).toISOString() }); + return { index: this.timings.length - 1 }; + } + + /** Records one admitted request's end and its upstream-reported usage, when the body carried any. */ + settle(index: number, usage: EngineBrokerTurnUsage | undefined): void { + const timing = this.timings[index]; + if (timing === undefined || timing.endedAt !== undefined) return; + timing.endedAt = new Date(this.now()).toISOString(); + if (usage === undefined) return; + timing.usage = usage; + this.tokens += usage.total; + } + + /** Trips a limit from outside the request path (the broker's wall-clock timer). */ + trip(reason: Exclude): void { + if (this.reason !== "none") return; + this.reason = reason; + this.onLimit(reason); + } + + snapshot(): GrokBrokerTurnMeterSnapshot { + const measured = this.timings.flatMap((timing) => timing.usage === undefined ? [] : [timing.usage]); + return { requests: this.timings.length, tokens: this.tokens, limitReason: this.reason, usage: sumEngineBrokerTurnUsage(measured), timings: this.timings.map((timing) => Object.freeze({ ...timing })) }; + } +} + +type JsonRecord = Record; +const isRecord = (value: unknown): value is JsonRecord => value !== null && typeof value === "object" && !Array.isArray(value); +const count = (value: unknown): number | undefined => typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : undefined; + +/** + * Upstream-reported usage of one chat-completions response, or `undefined`. + * + * The proxy buffers the whole upstream body, so the last `usage` object of an + * SSE stream (`stream_options.include_usage`) or of a JSON body is available + * before the body is returned to the worker. OpenAI-shaped `prompt_tokens` + * include cached tokens; they are split into disjoint buckets here, and any + * reasoning tokens reported outside `completion_tokens` (visible as + * `total_tokens` above prompt + completion) are folded into `output` so the + * total invariant holds. A malformed block is ignored, never zero-filled. + */ +export function parseGrokUpstreamUsage(body: Uint8Array, contentType: string | undefined): EngineBrokerTurnUsage | undefined { + const text = Buffer.from(body.buffer, body.byteOffset, body.byteLength).toString("utf8"); + const candidates: unknown[] = []; + if (contentType?.includes("text/event-stream") === true || text.startsWith("data:")) { + for (const line of text.split(/\r?\n/u)) { + if (!line.startsWith("data:")) continue; + const payload = line.slice(5).trim(); + if (payload === "[DONE]" || payload.length === 0) continue; + try { candidates.push(JSON.parse(payload)); } catch { /* a non-JSON event carries no usage */ } + } + } else { + try { candidates.push(JSON.parse(text)); } catch { return undefined; } + } + let found: EngineBrokerTurnUsage | undefined; + for (const candidate of candidates) { + if (!isRecord(candidate) || !isRecord(candidate.usage)) continue; + const decoded = decodeOpenAiUsage(candidate.usage); + if (decoded !== undefined) found = decoded; + } + return found; +} + +function decodeOpenAiUsage(usage: JsonRecord): EngineBrokerTurnUsage | undefined { + const prompt = count(usage.prompt_tokens), completion = count(usage.completion_tokens); + if (prompt === undefined || completion === undefined) return undefined; + const details = isRecord(usage.prompt_tokens_details) ? usage.prompt_tokens_details : {}; + const cached = details.cached_tokens === undefined ? 0 : count(details.cached_tokens); + const reported = usage.total_tokens === undefined ? prompt + completion : count(usage.total_tokens); + if (cached === undefined || reported === undefined || cached > prompt) return undefined; + const total = Math.max(reported, prompt + completion); + const completionDetails = isRecord(usage.completion_tokens_details) ? usage.completion_tokens_details : {}; + const reasoning = count(completionDetails.reasoning_tokens); + const output = total - prompt; + return { input: prompt - cached, cacheRead: cached, cacheWrite: 0, output, total, ...(reasoning === undefined || reasoning > output ? {} : { reasoning }) }; +} diff --git a/src/runtime/grokEngineBroker.ts b/src/runtime/grokEngineBroker.ts index 98a8384..d6d5125 100644 --- a/src/runtime/grokEngineBroker.ts +++ b/src/runtime/grokEngineBroker.ts @@ -1,57 +1,46 @@ -import { createHash, randomUUID } from "node:crypto"; -import { decodeGrokHeadlessTurn } from "../pi/grokHeadlessResult.js"; -import { recordTurnUsage, TURN_USAGE_LEDGER, type TurnUsageEntry } from "./turnUsageLedger.js"; import { DurableGrokBrokerCredentialAuthority } from "./grokBrokerCredentialAuthority.js"; -import { NativeBrokerTurnFailure, runNativeBrokerTurn, type NativeBrokerDiagnostic } from "./engineBrokerNativeClient.js"; +import { runNativeBrokerTurn } from "./engineBrokerNativeClient.js"; +import type { EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +import type { EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; import { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; import { startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; import { acquireGrokBrokerRealmLease } from "./grokBrokerRealmLease.js"; import { grokBrokerWorkerConfigSha256 } from "./grokBrokerWorkerConfig.js"; -import { parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; -import { createGrokWorkerIsolationGuard,GrokWorkerAttestationFailure,prepareGrokWorkerAttestation } from "./grokWorkerAttestation.js"; +import { parseGrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; +import { runGrokEngineBrokerTurn, type GrokEngineBrokerTurnResult } from "./grokEngineBrokerTurn.js"; +import { createGrokWorkerIsolationGuard,prepareGrokWorkerAttestation } from "./grokWorkerAttestation.js"; -export type GrokEngineBrokerRegistration = Readonly<{ agentId:string;slot:number;workerUid:number;workspace:string;profilePath:string;eventsPath:string;profileSha256:string }>; +export { EngineBrokerTurnFailure, type GrokEngineBrokerTurnResult } from "./grokEngineBrokerTurn.js"; +export { finishBrokerTurnWithUsage } from "./grokEngineBrokerMetering.js"; +export type GrokEngineBrokerRegistration = EngineBrokerServiceRegistration; export type GrokEngineBroker = Awaited>; -export class EngineBrokerTurnFailure extends Error{constructor(readonly code:"auth_stale"|"cancelled"|"engine_failed",readonly diagnostic?:NativeBrokerDiagnostic){super("engine broker turn failed");}} /** - * Seal a completed turn, then meter it. - * - * Order is load-bearing. `turns.finish` publishes the durable *completed* - * record; only after that does the advisory usage line get appended. A replayed - * turn returns before the enclosing `try` block and never reaches here, so a - * crash-recovered turn cannot double-count. Usage is deliberately kept out of - * the `completed` frame itself: that record is re-validated by the strict wire - * parser on the next `begin()`, whose exact field set would reject an extra key - * and break crash-recovery replay permanently. - * - * `recordTurnUsage` never rejects, so an append failure cannot escape into the - * caller's `catch` and rewrite this already-completed turn as failed. + * The Grok engine broker: one credential realm, one provider proxy, one MCP + * facade, and the root-provisioned registrations. Each registration declares + * its own model/effort (whose worker config bytes are attested), usage ledger, + * and turn limits (`engineBrokerServiceConfig.ts`). */ -export async function finishBrokerTurnWithUsage(turns: EngineBrokerTurnRegistry, request: Parameters[0], completed: Parameters[1], usageLedgerPath: string, usage: TurnUsageEntry["usage"] | undefined, agentId: string, wakeId: string): Promise { - await turns.finish(request, completed); - if (usage === undefined) return; - await recordTurnUsage(usageLedgerPath, { agent: agentId, wake: wakeId, engine: "grok", usage }); -} -export async function startGrokEngineBroker(options: Readonly<{ grokCommand: string; nativeClient: string; credentialHome: string; turnStore: string; registrations: readonly GrokEngineBrokerRegistration[]; usageLedgerPath?: string; modelPolicy?: GrokBrokerModelPolicy }>) { - // One declared model/effort drives both the worker config bytes the broker attests and the bodies the proxy forwards. - const modelPolicy = parseGrokBrokerModelPolicy(options.modelPolicy ?? {}); const configSha256 = grokBrokerWorkerConfigSha256(modelPolicy); - const usageLedgerPath = options.usageLedgerPath ?? TURN_USAGE_LEDGER.filePath; - const registrations = new Map(options.registrations.map((entry) => [entry.agentId, entry])); if (registrations.size !== options.registrations.length) throw new Error("engine broker registration conflict"); - const lease=await acquireGrokBrokerRealmLease(options.credentialHome);const authority = new DurableGrokBrokerCredentialAuthority(options.grokCommand, options.credentialHome);try{await authority.initialize();}catch(error){await lease.close();throw error;} let proxy:Awaited>;try{proxy=await startGrokBrokerProxy(authority,undefined,modelPolicy);}catch(error){await lease.close();throw error;}let mcp:Awaited>|undefined;try{mcp=await startEngineBrokerMcpFacade();for(const registration of registrations.values())await prepareGrokWorkerAttestation({...registration,brokerGid:2100,configSha256});}catch(error){if(mcp)await mcp.close().catch(()=>undefined);await proxy.close();await lease.close();throw error;}if(!mcp)throw new Error("engine broker unavailable");const turns = new EngineBrokerTurnRegistry(options.turnStore); const active = new Map; resolve: () => void }>(); let closed = false; +export async function startGrokEngineBroker(options: Readonly<{ grokCommand: string; nativeClient: string; credentialHome: string; turnStore: string; registrations: readonly GrokEngineBrokerRegistration[] }>) { + const registrations = new Map(options.registrations.map((entry) => [entry.agentId, { ...entry, model: parseGrokBrokerModelPolicy(entry.model) }])); if (registrations.size !== options.registrations.length) throw new Error("engine broker registration conflict"); + const attestationFor = (registration: GrokEngineBrokerRegistration) => ({ ...registration, brokerGid: 2100, configSha256: grokBrokerWorkerConfigSha256(registration.model) }); + const lease=await acquireGrokBrokerRealmLease(options.credentialHome);const authority = new DurableGrokBrokerCredentialAuthority(options.grokCommand, options.credentialHome);try{await authority.initialize();}catch(error){await lease.close();throw error;} let proxy:Awaited>;try{proxy=await startGrokBrokerProxy(authority);}catch(error){await lease.close();throw error;}let mcp:Awaited>|undefined;try{mcp=await startEngineBrokerMcpFacade();for(const registration of registrations.values())await prepareGrokWorkerAttestation(attestationFor(registration));}catch(error){if(mcp)await mcp.close().catch(()=>undefined);await proxy.close();await lease.close();throw error;}if(!mcp)throw new Error("engine broker unavailable");const turns = new EngineBrokerTurnRegistry(options.turnStore); const active = new Map }>(); let closed = false; + const facade = mcp; + const deps = { + turns, proxy, mcp: facade, credentialStale: () => authority.isStale(), + prepareIsolation: async (registration: GrokEngineBrokerRegistration) => { const attestation = attestationFor(registration); return createGrokWorkerIsolationGuard(attestation, await prepareGrokWorkerAttestation(attestation)); }, + runNative: (input: Parameters[1], signal: AbortSignal) => runNativeBrokerTurn(options.nativeClient, input, signal) + }; return { - async turn(agentId: string, wakeId: string, prompt: string, mcpEndpoint: string, signal?: AbortSignal): Promise> { + async turn(agentId: string, wakeId: string, prompt: string, mcpEndpoint: string, signal?: AbortSignal, limits?: EngineBrokerTurnLimitOverrides): Promise { if (closed) throw new Error("engine broker unavailable"); const registration = registrations.get(agentId); if (registration === undefined) throw new Error("engine broker unavailable"); - const turnId = createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"); const request = { version: "noopolis.daimon.engine-broker.v1", kind: "start_turn", requestId: randomUUID(), turnId, agentId, wakeId, prompt,mcpEndpoint } as const; - const begun = await turns.begin(request); if (begun !== "start") { if (begun.replay.kind === "completed") return {text:begun.replay.text,workerPid:begun.replay.workerPid,workerUid:begun.replay.workerUid,workerStartTime:begun.replay.workerStartTime};const code=begun.replay.code==="auth_stale"||begun.replay.code==="cancelled"?begun.replay.code:"engine_failed";throw new EngineBrokerTurnFailure(code,begun.replay.diagnostic as NativeBrokerDiagnostic|undefined); } - const attestation={...registration,brokerGid:2100,configSha256};const isolation=await prepareGrokWorkerAttestation(attestation);const isolationGuard=createGrokWorkerIsolationGuard(attestation,isolation);proxy.registerIsolationGuard(turnId,isolationGuard);const providerCapability = proxy.capabilities.issue(agentId, turnId);const mcpCapability=mcp.register(agentId,turnId,mcpEndpoint); const controller = new AbortController();const onAbort=()=>controller.abort();signal?.addEventListener("abort",onAbort,{once:true});if(signal?.aborted)controller.abort(); let resolve!:()=>void;const done=new Promise((value)=>{resolve=value;});active.set(turnId,{controller,done,resolve}); - let nativeDiagnostic:NativeBrokerDiagnostic|undefined,attested=false; - try { const result = await runNativeBrokerTurn(options.nativeClient, { slot: registration.slot, requestId: request.requestId, turnId, agentId, wakeId, prompt, providerCapability,mcpCapability }, controller.signal);nativeDiagnostic=result.diagnostic;if(result.workerUid!==registration.workerUid)throw new Error();await isolationGuard();attested=true; const decoded = decodeGrokHeadlessTurn(result.text);const text = decoded.text;const completed={ version: request.version, kind: "completed", requestId: request.requestId, turnId, text, workerPid: result.workerPid, workerUid: result.workerUid, workerStartTime: result.startTicks.toString() } as const; await finishBrokerTurnWithUsage(turns,request,completed,usageLedgerPath,decoded.usage,agentId,wakeId); return {text,workerPid:result.workerPid,workerUid:result.workerUid,workerStartTime:result.startTicks.toString()}; } - catch(error) { const code=authority.isStale()?"auth_stale":controller.signal.aborted?"cancelled":"engine_failed";const diagnostic=error instanceof NativeBrokerTurnFailure?error.diagnostic:nativeDiagnostic&&!attested?{...nativeDiagnostic,status:"worker_failed" as const,stage:"attestation" as const,failureClass:error instanceof GrokWorkerAttestationFailure?error.failureClass:"profile_invalid" as const,profileApplied:false}:undefined;await turns.finish(request, { version: request.version, kind: "failed", requestId: request.requestId, turnId, code,...(diagnostic?{diagnostic}:{}) }); throw new EngineBrokerTurnFailure(code,diagnostic); } - finally { signal?.removeEventListener("abort",onAbort);active.get(turnId)?.resolve(); active.delete(turnId); proxy.revokeIsolationGuard(turnId);proxy.capabilities.revoke(turnId);mcp.revoke(turnId); } + const controller = new AbortController(); const onAbort = () => controller.abort(); signal?.addEventListener("abort", onAbort, { once: true }); if (signal?.aborted) controller.abort(); + const key = `${agentId}\0${wakeId}`; const done = runGrokEngineBrokerTurn(deps, registration, wakeId, prompt, mcpEndpoint, controller.signal, limits); + active.set(key, { controller, done: done.then(() => undefined, () => undefined) }); + try { return await done; } finally { signal?.removeEventListener("abort", onAbort); active.delete(key); } }, - async close(): Promise { if (closed) return; closed = true; const running=[...active.values()];for (const entry of running) entry.controller.abort();await Promise.allSettled(running.map((entry)=>entry.done));const results=await Promise.allSettled([mcp.close(),proxy.close(),lease.close()]);const failures=results.flatMap((entry)=>entry.status==="rejected"?[entry.reason]:[]);if(failures.length)throw new AggregateError(failures,"engine broker shutdown failed"); }, + async close(): Promise { if (closed) return; closed = true; const running=[...active.values()];for (const entry of running) entry.controller.abort();await Promise.allSettled(running.map((entry)=>entry.done));const results=await Promise.allSettled([facade.close(),proxy.close(),lease.close()]);const failures=results.flatMap((entry)=>entry.status==="rejected"?[entry.reason]:[]);if(failures.length)throw new AggregateError(failures,"engine broker shutdown failed"); }, readiness: () => ({ providerProxyPort: proxy.port, mcpFacadePort:43_124, registrations: registrations.size,credentialStale:authority.isStale(),realmLease:true,workerIsolation:true }) }; } diff --git a/src/runtime/grokEngineBrokerMetering.ts b/src/runtime/grokEngineBrokerMetering.ts new file mode 100644 index 0000000..f254554 --- /dev/null +++ b/src/runtime/grokEngineBrokerMetering.ts @@ -0,0 +1,43 @@ +import type { EngineBrokerRequest, EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; +import type { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; +import { recordGrokTurnRequests, type GrokTurnRequest } from "./turnRequestLedger.js"; +import { recordTurnUsage, type TurnUsageFailureReason } from "./turnUsageLedger.js"; + +export type BrokerTurnMetering = Readonly<{ + usageLedgerPath: string; + requestLedgerPath: string; + agentId: string; + wakeId: string; +}>; +export type BrokerTurnMeteringDetail = Readonly<{ notionalUsd: number; complete: boolean; reason?: TurnUsageFailureReason; requests: readonly GrokTurnRequest[]; session?: string }>; + +/** + * Seal a terminal turn, then meter it. The broker is the single writer. + * + * Order is load-bearing. `turns.finish` publishes the durable terminal record + * *with* its accounting; only after that are the advisory ledger rows appended. + * A replayed turn returns before the broker's `try` block and never reaches + * here, so a crash-recovered or repeated turn cannot double-count; every row + * also carries the turn id as `turn`, so a reader that sees one twice counts + * it once. + * + * Both terminal kinds meter: a failed turn spent real tokens, so its partial + * usage is written with `outcome: failed` and its closed `limitReason`. A turn + * with no usage at all (`usage: null`) writes nothing — a zero row is + * byte-identical to a measured zero. + * + * `recordTurnUsage`/`recordGrokTurnRequests` never reject, so an append failure + * cannot escape into the caller's `catch` and rewrite a completed turn as failed. + */ +export async function finishBrokerTurnWithUsage(turns: EngineBrokerTurnRegistry, request: Extract, terminal: EngineBrokerTerminalResponse, metering: BrokerTurnMetering, detail: BrokerTurnMeteringDetail): Promise { + await turns.finish(request, terminal); + if (terminal.usage === null) return; + const { usage } = terminal; + await recordTurnUsage(metering.usageLedgerPath, { + agent: metering.agentId, wake: metering.wakeId, engine: "grok", + usage: { input: usage.input, output: usage.output, cacheRead: usage.cacheRead, cacheWrite: usage.cacheWrite, total: usage.total, calls: terminal.requests, notionalUsd: detail.notionalUsd, complete: detail.complete }, + outcome: terminal.kind === "completed" ? { status: "completed" } : { status: "failed", reason: detail.reason ?? "unknown" }, + turn: terminal.turnId, limitReason: terminal.limitReason, model: terminal.model + }); + await recordGrokTurnRequests(metering.requestLedgerPath, { agent: metering.agentId, wake: metering.wakeId, turn: terminal.turnId, model: terminal.model, requests: detail.requests, ...(detail.session === undefined ? {} : { session: detail.session }) }); +} diff --git a/src/runtime/grokEngineBrokerTurn.ts b/src/runtime/grokEngineBrokerTurn.ts new file mode 100644 index 0000000..105ff02 --- /dev/null +++ b/src/runtime/grokEngineBrokerTurn.ts @@ -0,0 +1,124 @@ +import { createHash, randomUUID } from "node:crypto"; + +import { decodeGrokHeadlessTurn } from "../pi/grokHeadlessResult.js"; +import { decodeGrokStreamUsage, type GrokStreamUsage } from "../pi/grokStreamUsage.js"; +import { ENGINE_BROKER_VERSION, type EngineBrokerFailureCode, type EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; +import { lowerEngineBrokerTurnLimits, mapGrokReportedModel, type EngineBrokerTurnAccounting, type EngineBrokerTurnLimitOverrides, type EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; +import { NativeBrokerTurnFailure, type NativeBrokerDiagnostic, type NativeBrokerTurn, type NativeBrokerTurnResult } from "./engineBrokerNativeClient.js"; +import type { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; +import { engineBrokerRequestLedgerPathFor, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +import { finishBrokerTurnWithUsage, type BrokerTurnMetering, type BrokerTurnMeteringDetail } from "./grokEngineBrokerMetering.js"; +import type { GrokBrokerProxyTurn } from "./grokBrokerProxy.js"; +import { GrokBrokerTurnMeter, type GrokBrokerTurnMeterSnapshot } from "./grokBrokerTurnMeter.js"; +import { GrokWorkerAttestationFailure } from "./grokWorkerAttestation.js"; + +export type GrokEngineBrokerTurnResult = Readonly<{ text: string; workerPid: number; workerUid: number; workerStartTime: string }> & EngineBrokerTurnAccounting; +export class EngineBrokerTurnFailure extends Error { + constructor(readonly code: Exclude, readonly diagnostic?: NativeBrokerDiagnostic, readonly accounting?: EngineBrokerTurnAccounting) { super("engine broker turn failed"); } +} + +/** Everything one broker turn touches, injected so the accounting and limit paths run under test without a native launcher. */ +export type GrokEngineBrokerTurnDependencies = Readonly<{ + turns: EngineBrokerTurnRegistry; + proxy: Readonly<{ capabilities: Readonly<{ issue(agentId: string, turnId: string): string; revoke(turnId: string): void }>; registerIsolationGuard(turnId: string, guard: () => Promise): void; revokeIsolationGuard(turnId: string): void; registerTurn(turnId: string, turn: GrokBrokerProxyTurn): void; revokeTurn(turnId: string): void }>; + mcp: Readonly<{ register(agentId: string, turnId: string, endpoint: string): string; revoke(turnId: string): void }>; + credentialStale(): boolean; + prepareIsolation(registration: EngineBrokerServiceRegistration): Promise<() => Promise>; + runNative(input: NativeBrokerTurn, signal: AbortSignal): Promise>; +}>; + +const limitReasonFor = { tokens: "token_ceiling", requests: "request_ceiling", timeout: "wake_timeout" } as const; +const usageOf = (usage: Readonly<{ input: number; cacheRead: number; cacheWrite: number; output: number; total: number }>): EngineBrokerTurnUsage => ({ input: usage.input, cacheRead: usage.cacheRead, cacheWrite: usage.cacheWrite, output: usage.output, total: usage.total }); + +/** + * One brokered Grok turn under its declared limits. + * + * The limits are the registration's, lowered (never raised) by the wake. The + * proxy meter refuses model requests past `maxRequests`, past the elapsed + * deadline, or once the upstream-reported running total reached `maxTokens`, + * and a tripped limit aborts the worker through the same cancel/kill path a + * client cancellation uses. A wall-clock timer trips `timeout` for a worker + * that is mid-request. + * + * Every terminal path — completed, failed, limit, cancelled — is sealed and + * metered through {@link finishBrokerTurnWithUsage}; a replayed turn returns its + * sealed accounting before any of this runs and never meters again. + */ +export async function runGrokEngineBrokerTurn(deps: GrokEngineBrokerTurnDependencies, registration: EngineBrokerServiceRegistration, wakeId: string, prompt: string, mcpEndpoint: string, signal?: AbortSignal, overrides?: EngineBrokerTurnLimitOverrides): Promise { + const { agentId } = registration, declared = registration.model.model; + let limits; + try { limits = lowerEngineBrokerTurnLimits(registration.limits, overrides); } catch { throw new EngineBrokerTurnFailure("invalid_request", undefined, { outcome: "failed", usage: null, model: declared, requests: 0, limitReason: "none" }); } + const turnId = createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"); + const request = { version: ENGINE_BROKER_VERSION, kind: "start_turn", requestId: randomUUID(), turnId, agentId, wakeId, prompt, mcpEndpoint, ...(overrides === undefined ? {} : { limits: overrides }) } as const; + const begun = await deps.turns.begin(request, declared); + if (begun !== "start") return replay(begun.replay); + const controller = new AbortController(); + const meter = new GrokBrokerTurnMeter(limits, () => controller.abort()); + const onAbort = () => controller.abort(); signal?.addEventListener("abort", onAbort, { once: true }); if (signal?.aborted) controller.abort(); + const timer = setTimeout(() => meter.trip("timeout"), limits.timeoutMs); timer.unref?.(); + const metering: BrokerTurnMetering = { usageLedgerPath: registration.usageLedgerPath, requestLedgerPath: engineBrokerRequestLedgerPathFor(registration.usageLedgerPath), agentId, wakeId }; + let nativeDiagnostic: NativeBrokerDiagnostic | undefined, attested = false, output: string | undefined, rejected = false; + try { + const isolationGuard = await deps.prepareIsolation(registration); + deps.proxy.registerIsolationGuard(turnId, isolationGuard); + deps.proxy.registerTurn(turnId, { policy: registration.model, meter }); + const providerCapability = deps.proxy.capabilities.issue(agentId, turnId), mcpCapability = deps.mcp.register(agentId, turnId, mcpEndpoint); + const result = await deps.runNative({ slot: registration.slot, requestId: request.requestId, turnId, agentId, wakeId, prompt, providerCapability, mcpCapability }, controller.signal); + nativeDiagnostic = result.diagnostic; output = result.text; + if (result.workerUid !== registration.workerUid) throw new Error("engine broker worker identity mismatch"); + await isolationGuard(); attested = true; + rejected = true; + const decoded = decodeGrokHeadlessTurn(result.text), stream = decodeGrokStreamUsage(result.text); + if (stream.reportedModels.some((reported) => mapGrokReportedModel(reported, declared) === undefined)) throw new Error("engine broker reported an undeclared model"); + rejected = false; + const snapshot = meter.snapshot(); + if (snapshot.limitReason !== "none") throw new Error("engine broker turn limit reached"); + const usage = decoded.usage === undefined ? streamOrMeterUsage(stream, snapshot) : usageOf(decoded.usage); + const accounting = { outcome: "completed", usage, model: declared, requests: requestCount(stream, snapshot), limitReason: "none" } as const; + const completed = { version: request.version, kind: "completed", requestId: request.requestId, turnId, text: decoded.text, workerPid: result.workerPid, workerUid: result.workerUid, workerStartTime: result.startTicks.toString(), ...accounting } as const; + await finishBrokerTurnWithUsage(deps.turns, request, completed, metering, { notionalUsd: decoded.usage?.notionalUsd ?? 0, complete: decoded.usage?.complete ?? false, requests: requestRows(stream, snapshot), ...(stream.sessionId === undefined ? {} : { session: stream.sessionId }) }); + return { text: completed.text, workerPid: completed.workerPid, workerUid: completed.workerUid, workerStartTime: completed.workerStartTime, ...accounting }; + } catch (error) { + const snapshot = meter.snapshot(); + const code: EngineBrokerTurnFailure["code"] = snapshot.limitReason !== "none" ? "limit_exceeded" : deps.credentialStale() ? "auth_stale" : controller.signal.aborted ? "cancelled" : "engine_failed"; + const diagnostic = error instanceof NativeBrokerTurnFailure ? error.diagnostic : nativeDiagnostic && !attested ? { ...nativeDiagnostic, status: "worker_failed" as const, stage: "attestation" as const, failureClass: error instanceof GrokWorkerAttestationFailure ? error.failureClass : "profile_invalid" as const, profileApplied: false } : undefined; + const stream = output === undefined ? undefined : decodeGrokStreamUsage(output); + const accounting = { outcome: "failed", usage: streamOrMeterUsage(stream, snapshot), model: declared, requests: requestCount(stream, snapshot), limitReason: snapshot.limitReason } as const; + const failed: EngineBrokerTerminalResponse = { version: request.version, kind: "failed", requestId: request.requestId, turnId, code, ...(diagnostic ? { diagnostic } : {}), ...accounting }; + const reason = snapshot.limitReason !== "none" ? limitReasonFor[snapshot.limitReason] : rejected ? "turn_rejected" : "unknown"; + await finishBrokerTurnWithUsage(deps.turns, request, failed, metering, { notionalUsd: 0, complete: false, reason, requests: requestRows(stream, snapshot), ...(stream?.sessionId === undefined ? {} : { session: stream.sessionId }) }); + throw new EngineBrokerTurnFailure(code, diagnostic, accounting); + } finally { + clearTimeout(timer); signal?.removeEventListener("abort", onAbort); + deps.proxy.revokeTurn(turnId); deps.proxy.revokeIsolationGuard(turnId); deps.proxy.capabilities.revoke(turnId); deps.mcp.revoke(turnId); + } +} + +function replay(response: EngineBrokerTerminalResponse): GrokEngineBrokerTurnResult { + const accounting = { outcome: response.outcome, usage: response.usage, model: response.model, requests: response.requests, limitReason: response.limitReason }; + if (response.kind === "completed") return { text: response.text, workerPid: response.workerPid, workerUid: response.workerUid, workerStartTime: response.workerStartTime, ...accounting, outcome: "completed" }; + const code = response.code === "turn_conflict" || response.code === "unavailable" ? "engine_failed" : response.code; + throw new EngineBrokerTurnFailure(code, response.diagnostic as NativeBrokerDiagnostic | undefined, accounting); +} + +/** The proxy saw every forwarded request; the stream is the fallback when no request crossed this proxy. */ +const requestCount = (stream: GrokStreamUsage | undefined, snapshot: GrokBrokerTurnMeterSnapshot): number => snapshot.requests > 0 ? snapshot.requests : stream?.requests.length ?? 0; + +/** Best partial usage: the worker's own per-request frames when any arrived, else what upstream reported to the proxy. */ +function streamOrMeterUsage(stream: GrokStreamUsage | undefined, snapshot: GrokBrokerTurnMeterSnapshot): EngineBrokerTurnUsage | null { + if (stream !== undefined && stream.requests.length > 0) { + const sum = (pick: (value: GrokStreamUsage["requests"][number]) => number) => stream.requests.reduce((total, value) => total + pick(value), 0); + return { input: sum((value) => value.input), cacheRead: sum((value) => value.cacheRead), cacheWrite: sum((value) => value.cacheWrite), output: sum((value) => value.output), total: sum((value) => value.total) }; + } + return snapshot.usage; +} + +/** Per-request rows: stream usage with proxy timing when both describe the same requests, else the proxy's own measured requests. */ +function requestRows(stream: GrokStreamUsage | undefined, snapshot: GrokBrokerTurnMeterSnapshot): BrokerTurnMeteringDetail["requests"] { + if (stream !== undefined && stream.requests.length > 0) { + const timed = snapshot.timings.length === stream.requests.length; + return stream.requests.map((value, index) => ({ ...value, ...(timed ? clock(snapshot.timings[index]!) : {}) })); + } + return snapshot.timings.flatMap((timing, index) => timing.usage === undefined ? [] : [{ index, ...usageOf(timing.usage), ...clock(timing) }]); +} +const clock = (timing: GrokBrokerTurnMeterSnapshot["timings"][number]) => ({ startedAt: timing.startedAt, ...(timing.endedAt === undefined ? {} : { endedAt: timing.endedAt }) }); diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index 7be6215..f66631d 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -1,130 +1,169 @@ import assert from "node:assert/strict"; -import { createHash, randomUUID } from "node:crypto"; +import { createHash } from "node:crypto"; import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { request as httpRequest } from "node:http"; import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import test from "node:test"; +import type { NativeBrokerTurn, NativeBrokerTurnResult } from "./engineBrokerNativeClient.js"; +import type { EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; import { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; -import { finishBrokerTurnWithUsage } from "./grokEngineBroker.js"; +import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; +import { EngineBrokerTurnFailure, runGrokEngineBrokerTurn, type GrokEngineBrokerTurnDependencies } from "./grokEngineBrokerTurn.js"; +import { TURN_REQUEST_LEDGER_VERSION } from "./turnRequestLedger.js"; import { TURN_USAGE_LEDGER_VERSION } from "./turnUsageLedger.js"; -const usage = { input: 8_746, output: 29, cacheRead: 5_760, cacheWrite: 12, total: 14_547, calls: 1, notionalUsd: 0.0035, complete: true }; - -const startRequest = (agentId: string, wakeId: string) => ({ - version: "noopolis.daimon.engine-broker.v1", kind: "start_turn", requestId: randomUUID(), - turnId: createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"), - agentId, wakeId, prompt: "prompt", mcpEndpoint: "http://127.0.0.1:43124/mcp" -} as const); +const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); +const leanBody = JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: lean }); +const upstreamUsage = { prompt_tokens: 2_696, completion_tokens: 79, total_tokens: 2_775, prompt_tokens_details: { cached_tokens: 128 } }; +const turnIdFor = (agentId: string, wakeId: string): string => createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"); + +const assistant = (id: string, usage: Record, content: unknown[], stop: string) => ({ type: "assistant", message: { id, type: "message", role: "assistant", model: "daimon-broker-grok", content, stop_reason: stop, usage }, parent_tool_use_id: null, session_id: "01a0ad21-a90f-7f71-8054-93fdb4334d6a" }); +const first = { input_tokens: 2_568, output_tokens: 79, cache_read_input_tokens: 128, cache_creation_input_tokens: 0 }; +const second = { input_tokens: 109, output_tokens: 13, cache_read_input_tokens: 2_688, cache_creation_input_tokens: 0 }; +const stream = (modelKey = "grok-4.6-build"): string => [ + { type: "system", subtype: "init", session_id: "01a0ad21-a90f-7f71-8054-93fdb4334d6a" }, + assistant("msg_0", first, [{ type: "tool_use", id: "call-0", name: "use_tool", input: {} }], "tool_use"), + assistant("msg_1", second, [{ type: "text", text: "TANGERINE-7" }], "end_turn"), + { type: "result", subtype: "success", is_error: false, num_turns: 2, result: "TANGERINE-7", stop_reason: "end_turn", total_cost_usd: 0.00248676, usage: { input_tokens: 2_677, output_tokens: 92, cache_read_input_tokens: 2_816, cache_creation_input_tokens: 0 }, modelUsage: { [modelKey]: {} }, session_id: "01a0ad21-a90f-7f71-8054-93fdb4334d6a" } +].map((frame) => JSON.stringify(frame)).join("\n"); + +function post(port: number, token: string): Promise { + return new Promise((resolve, reject) => { + const req = httpRequest({ host: "127.0.0.1", port, path: "/v1/chat/completions", method: "POST", agent: false, headers: { authorization: `Bearer ${token}`, "x-grok-client-version": "1.0.34", "content-type": "application/json" } }, (response) => { response.resume(); response.on("end", () => resolve(response.statusCode ?? 0)); }); + req.on("error", reject); req.end(leanBody); + }); +} -const completedFor = (request: ReturnType) => ({ - version: request.version, kind: "completed", requestId: request.requestId, turnId: request.turnId, - text: "ACK", workerPid: 4_242, workerUid: 2_200, workerStartTime: "99" -} as const); +type Worker = (post: () => Promise, signal: AbortSignal) => Promise; +const nativeResult = (text: string): NativeBrokerTurnResult => ({ text, workerPid: 4_242, workerUid: 2_200, startTicks: 99n, diagnostic: { status: "ok", stage: "output", failureClass: "none", profileApplied: false, exitCode: 0, termSignal: 0, workerPid: 4_242, workerUid: 2_200, startTicks: "99" } }); +const untilAborted = (signal: AbortSignal): Promise => new Promise((_resolve, reject) => { const fail = () => reject(new Error("engine broker turn failed")); if (signal.aborted) fail(); else signal.addEventListener("abort", fail, { once: true }); }); /** - * Reproduces the broker's turn control flow around the registry: replayed turns - * return before any work, and a fresh turn seals the record and then meters it. - * Everything but the engine call itself is the real production code. + * The real turn registry, proxy, meter and ledgers around a scripted worker that + * talks to the proxy exactly as the native worker does (capability bearer, + * pinned client version, lean body). Only the launcher and attestation are fakes. */ -const runTurn = async (turns: EngineBrokerTurnRegistry, ledger: string, agentId: string, wakeId: string): Promise<"start" | "replay"> => { - const request = startRequest(agentId, wakeId); - const begun = await turns.begin(request); - if (begun !== "start") return "replay"; - await finishBrokerTurnWithUsage(turns, request, completedFor(request), ledger, usage, agentId, wakeId); - return "start"; -}; - -const withStore = async (body: (turnStore: string, ledger: string, root: string) => Promise): Promise => { +const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId: string, worker: Worker, overrides?: Parameters[6], limits?: EngineBrokerServiceRegistration["limits"], turnStore?: string) => ReturnType; usageRows: () => Promise[]>; requestRows: () => Promise[]>; upstreamCalls: () => number }>) => Promise, usageLedgerPath?: string): Promise => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-usage-")); - try { await body(path.join(root, "turns"), path.join(root, "usage.jsonl"), root); } finally { await rm(root, { recursive: true, force: true }); } -}; - -const ledgerLines = async (file: string): Promise[]> => { - const text = await readFile(file, "utf8").catch(() => ""); - return text.split("\n").filter((line) => line.length > 0).map((line) => JSON.parse(line) as Record); + let calls = 0; + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(`data: ${JSON.stringify({ choices: [], usage: upstreamUsage })}\n\ndata: [DONE]\n\n`) }; }, undefined, 0); + const ledger = usageLedgerPath ?? path.join(root, "usage.jsonl"); + const rows = async (file: string) => (await readFile(file, "utf8").catch(() => "")).split("\n").filter((line) => line.length > 0).map((line) => JSON.parse(line) as Record); + try { + await body({ + root, + turn: (wakeId, worker, overrides, limits = { maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }, turnStore = path.join(root, "turns")) => { + const registration: EngineBrokerServiceRegistration = { agentId: "foreman", slot: 0, workerUid: 2_200, workspace: "/workspace", profilePath: "/workers/0/.grok/sandbox.toml", eventsPath: "/workers/0/.grok/sessions/sandbox-events.jsonl", profileSha256: "a".repeat(64), usageLedgerPath: ledger, limits, model: { model: "grok-4.6", reasoningEffort: "low" } }; + const deps: GrokEngineBrokerTurnDependencies = { + turns: new EngineBrokerTurnRegistry(turnStore), proxy, credentialStale: () => false, + mcp: { register: () => "mcp-capability-0123456789abcdef", revoke: () => undefined }, + prepareIsolation: async () => async () => undefined, + runNative: async (input: NativeBrokerTurn, signal: AbortSignal) => nativeResult(await worker(() => post(proxy.port, input.providerCapability), signal)) + }; + return runGrokEngineBrokerTurn(deps, registration, wakeId, "prompt", "http://127.0.0.1:43124/mcp", undefined, overrides); + }, + usageRows: () => rows(ledger), + requestRows: () => rows(path.join(path.dirname(ledger), "requests.jsonl")), + upstreamCalls: () => calls + }); + } finally { await proxy.close(); await rm(root, { recursive: true, force: true }); } }; -test("a completed broker turn writes exactly one metered line, and a replayed turn writes no second one", async () => { - await withStore(async (turnStore, ledger) => { - const turns = new EngineBrokerTurnRegistry(turnStore); - assert.equal(await runTurn(turns, ledger, "cogsworth", "wake-1"), "start"); - assert.deepEqual(await runTurn(turns, ledger, "cogsworth", "wake-1"), "replay"); +const twoRequests: Worker = async (send) => { assert.equal(await send(), 200); assert.equal(await send(), 200); return stream(); }; + +test("a completed turn seals its accounting, writes one usage row and per-request rows, and a replay never re-meters", async () => { + await withBroker(async ({ root, turn, usageRows, requestRows }) => { + const result = await turn("wake-1", twoRequests); + assert.deepEqual({ ...result, text: undefined }, { text: undefined, workerPid: 4_242, workerUid: 2_200, workerStartTime: "99", outcome: "completed", usage: { input: 2_677, cacheRead: 2_816, cacheWrite: 0, output: 92, total: 5_585 }, model: "grok-4.6", requests: 2, limitReason: "none" }); + // Mutation guard: metering on the replay path appends a second row here. + let replayedWorker = false; + assert.deepEqual(await turn("wake-1", async () => { replayedWorker = true; return stream(); }), result); + assert.deepEqual(await turn("wake-1", twoRequests, undefined, undefined, path.join(root, "turns")), result, "a fresh registry boot replays the sealed accounting"); + assert.equal(replayedWorker, false); + const usage = await usageRows(); + assert.equal(usage.length, 1); + assert.deepEqual({ ...usage[0], at: undefined }, { v: TURN_USAGE_LEDGER_VERSION, agent: "foreman", wake: "wake-1", engine: "grok", at: undefined, input: 2_677, output: 92, cache_read: 2_816, cache_write: 0, total: 5_585, calls: 2, notional_usd: 0.00248676, complete: true, outcome: "completed", turn: turnIdFor("foreman", "wake-1"), limit_reason: "none", model: "grok-4.6" }); + const requests = await requestRows(); + assert.deepEqual(requests.map((row) => [row.v, row.engine, row.request, row.requests, row.input, row.fresh_input, row.cached_input, row.total, row.turn]), [ + [TURN_REQUEST_LEDGER_VERSION, "grok", 0, 2, 2_696, 2_568, 128, 2_775, turnIdFor("foreman", "wake-1")], + [TURN_REQUEST_LEDGER_VERSION, "grok", 1, 2, 2_797, 109, 2_688, 2_810, turnIdFor("foreman", "wake-1")] + ]); + // Mutation guard: stamping every request with the wake end collapses these. + for (const row of requests) assert.match(String(row.started_at), /^\d{4}-\d{2}-\d{2}T/u); + assert.ok(String(requests[0]!.ended_at) <= String(requests[1]!.started_at), "request 1 ends before request 2 starts"); + }); +}); - // Mutation guard: removing the replay suppression makes the same wake - // append a second line and double-count the subscription. - const written = await ledgerLines(ledger); - assert.equal(written.length, 1); - assert.deepEqual(written[0], { - v: TURN_USAGE_LEDGER_VERSION, agent: "cogsworth", wake: "wake-1", engine: "grok", - at: written[0]?.at, input: 8_746, output: 29, cache_read: 5_760, cache_write: 12, - total: 14_547, calls: 1, notional_usd: 0.0035, complete: true, - // The broker only ever appends for a turn it finished, so its rows are - // completed by construction; the field still states it explicitly. - outcome: "completed" +test("a turn past maxRequests is refused before upstream, killed, sealed as limit_exceeded, and its partial usage is metered", async () => { + await withBroker(async ({ turn, usageRows, upstreamCalls }) => { + const worker: Worker = async (send, signal) => { for (;;) { if (await send() === 429) return untilAborted(signal); } }; + await assert.rejects(turn("wake-2", worker, undefined, { maxRequests: 3, maxTokens: 300_000, timeoutMs: 240_000 }), (error: unknown) => { + assert.ok(error instanceof EngineBrokerTurnFailure); + assert.equal(error.code, "limit_exceeded"); + assert.deepEqual(error.accounting, { outcome: "failed", usage: { input: 7_704, cacheRead: 384, cacheWrite: 0, output: 237, total: 8_325 }, model: "grok-4.6", requests: 3, limitReason: "requests" }); + return true; }); - - assert.equal(await runTurn(turns, ledger, "cogsworth", "wake-2"), "start"); - assert.equal((await ledgerLines(ledger)).length, 2); + assert.equal(upstreamCalls(), 3); + // Mutation guard: metering only completed turns leaves this ledger empty. + const [row, extra] = await usageRows(); + assert.equal(extra, undefined); + assert.deepEqual([row?.outcome, row?.reason, row?.limit_reason, row?.total, row?.calls, row?.complete], ["failed", "request_ceiling", "requests", 8_325, 3, false]); + await assert.rejects(turn("wake-2", twoRequests), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.accounting?.limitReason === "requests"); + assert.equal((await usageRows()).length, 1, "the replayed failure is not metered again"); }); }); -test("crash recovery replays the same completed turn without metering it again", async () => { - await withStore(async (turnStore, ledger) => { - assert.equal(await runTurn(new EngineBrokerTurnRegistry(turnStore), ledger, "foreman", "wake-9"), "start"); - // A fresh boot id is what the broker gets after a crash. - assert.equal(await runTurn(new EngineBrokerTurnRegistry(turnStore), ledger, "foreman", "wake-9"), "replay"); - assert.equal((await ledgerLines(ledger)).length, 1); +test("the token ceiling stops a turn one request past the ceiling at most", async () => { + await withBroker(async ({ turn, upstreamCalls }) => { + const worker: Worker = async (send, signal) => { for (;;) { if (await send() === 429) return untilAborted(signal); } }; + // 2,775 tokens per request against 5,000: requests 1 and 2 are admitted, 3 is refused. + await assert.rejects(turn("wake-3", worker, { maxTokens: 5_000 }), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.accounting?.limitReason === "tokens" && error.accounting.usage?.total === 5_550); + assert.equal(upstreamCalls(), 2); }); }); -test("an unwritable ledger leaves the turn recorded as completed, not failed", async () => { - // Mutation guard: deleting the advisory try/catch in recordTurnUsage makes - // finishBrokerTurnWithUsage reject. In the broker that rejection lands in the - // catch that calls finish(..., failed), which renames over this already - // completed record — turning a published turn into a failed one. - await withStore(async (turnStore, _ledger, root) => { - const turns = new EngineBrokerTurnRegistry(turnStore); - const unwritable = path.join(root, "not-provisioned", "usage.jsonl"); - const request = startRequest("brass", "wake-3"); - assert.equal(await turns.begin(request), "start"); - await assert.doesNotReject(finishBrokerTurnWithUsage(turns, request, completedFor(request), unwritable, usage, "brass", "wake-3")); - - const replayed = await new EngineBrokerTurnRegistry(turnStore).begin(request); - assert.notEqual(replayed, "start"); - assert.equal((replayed as { replay: { kind: string } }).replay.kind, "completed"); +test("the wall-clock limit aborts a worker that is mid-request", async () => { + await withBroker(async ({ turn, usageRows }) => { + const started = Date.now(); + const worker: Worker = async (send, signal) => { assert.equal(await send(), 200); return untilAborted(signal); }; + await assert.rejects(turn("wake-4", worker, { timeoutMs: 1_000 }), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.code === "limit_exceeded" && error.accounting?.limitReason === "timeout"); + assert.ok(Date.now() - started < 5_000); + assert.deepEqual((await usageRows()).map((row) => [row.reason, row.limit_reason, row.total]), [["wake_timeout", "timeout", 2_775]]); }); }); -test("a turn whose usage could not be decoded is sealed but writes no line", async () => { - await withStore(async (turnStore, ledger) => { - const turns = new EngineBrokerTurnRegistry(turnStore); - const request = startRequest("brass", "wake-4"); - assert.equal(await turns.begin(request), "start"); - await finishBrokerTurnWithUsage(turns, request, completedFor(request), ledger, undefined, "brass", "wake-4"); - assert.deepEqual(await ledgerLines(ledger), []); - assert.notEqual(await new EngineBrokerTurnRegistry(turnStore).begin(request), "start"); +test("a wake may only lower a declared limit: raising one is refused before any turn record or worker", async () => { + await withBroker(async ({ turn, usageRows, upstreamCalls }) => { + let ran = false; + // Mutation guard: clamping or accepting the raise runs the worker. + await assert.rejects(turn("wake-5", async () => { ran = true; return stream(); }, { maxTokens: 300_001 }), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.code === "invalid_request"); + assert.equal(ran, false); assert.equal(upstreamCalls(), 0); assert.deepEqual(await usageRows(), []); + assert.equal((await turn("wake-5", twoRequests, { maxTokens: 299_999, timeoutMs: 1_000 })).outcome, "completed", "a lowered limit is accepted and the turn was never recorded"); }); }); -test("usage is never written into the completed frame the strict wire parser re-validates", async () => { - await withStore(async (turnStore, ledger) => { - const turns = new EngineBrokerTurnRegistry(turnStore); - assert.equal(await runTurn(turns, ledger, "cogsworth", "wake-5"), "start"); - // The durable record is re-parsed on the next begin(); an extra field there - // makes it throw permanently and breaks crash-recovery replay for good. - const replayed = await new EngineBrokerTurnRegistry(turnStore).begin(startRequest("cogsworth", "wake-5")); - const response = (replayed as { replay: Record }).replay; - assert.deepEqual(Object.keys(response).sort(), ["kind", "requestId", "text", "turnId", "version", "workerPid", "workerStartTime", "workerUid"]); +test("a turn whose stream reports an undeclared model fails as rejected but is still metered", async () => { + await withBroker(async ({ turn, usageRows }) => { + await assert.rejects(turn("wake-6", async (send) => { await send(); await send(); return stream("grok-4.5-build"); }), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.code === "engine_failed"); + assert.deepEqual((await usageRows()).map((row) => [row.outcome, row.reason, row.model, row.total]), [["failed", "turn_rejected", "grok-4.6", 5_585]]); }); }); -test("the broker meters only on the success path, through the single sealing helper", async () => { - const source = await readFile(path.join(path.dirname(fileURLToPath(import.meta.url)), "grokEngineBroker.ts"), "utf8"); - const body = source.slice(source.indexOf("async turn(")); - assert.equal(body.includes("recordTurnUsage("), false, "the broker must meter only through finishBrokerTurnWithUsage"); - assert.equal((body.match(/finishBrokerTurnWithUsage\(/gu) ?? []).length, 1, "exactly one metering call, in the success branch"); - assert.equal(body.includes("turns.finish(request,completed)"), false, "the success branch must seal through the metering helper"); - assert.ok(body.indexOf("finishBrokerTurnWithUsage(") < body.indexOf("catch(error)"), "metering belongs to the success branch"); +test("an unwritable ledger leaves the turn recorded as completed, not failed", async () => { + await withBroker(async ({ root, turn }) => { + assert.equal((await turn("wake-7", twoRequests)).outcome, "completed"); + assert.equal((await turn("wake-7", twoRequests, undefined, undefined, path.join(root, "turns"))).outcome, "completed"); + }, path.join(os.tmpdir(), `daimon-missing-${process.pid}`, "not-provisioned", "usage.jsonl")); +}); + +test("the broker meters only through the single sealing helper, on both terminal branches", async () => { + const source = await readFile(path.join(path.dirname(fileURLToPath(import.meta.url)), "grokEngineBrokerTurn.ts"), "utf8"); + const body = source.slice(source.indexOf("export async function runGrokEngineBrokerTurn"), source.indexOf("function replay(")); + assert.equal(body.includes("recordTurnUsage("), false); + assert.equal(body.includes("turns.finish("), false, "every terminal record is sealed through the metering helper"); + assert.equal((body.match(/finishBrokerTurnWithUsage\(/gu) ?? []).length, 2); + assert.ok(body.indexOf("return replay(") < body.indexOf("finishBrokerTurnWithUsage("), "a replay returns before any metering"); }); From 65d0144be9e1513a03b71af9191841c95117d88e Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:13:24 +0200 Subject: [PATCH 032/124] feat: pass wake limits and declared model to the Grok broker from the dispatcher --- scripts/liveGrokBrokerSession.ts | 4 ++++ src/runtime/engineDispatcher.test.ts | 9 +++++++-- src/runtime/engineDispatcher.ts | 13 +++++++++++-- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/scripts/liveGrokBrokerSession.ts b/scripts/liveGrokBrokerSession.ts index 05cf30d..83c8772 100644 --- a/scripts/liveGrokBrokerSession.ts +++ b/scripts/liveGrokBrokerSession.ts @@ -8,7 +8,9 @@ import { readChild } from "../src/pi/cliChildOutput.ts"; import { terminateChild, trackCliChild } from "../src/pi/cliProcess.ts"; import { decodeGrokHeadlessTurn } from "../src/pi/grokHeadlessResult.ts"; import { readGrokBrokerCredential } from "../src/runtime/grokBrokerCredentialReader.ts"; +import { DEFAULT_GROK_BROKER_TURN_LIMITS } from "../src/runtime/engineBrokerTurnAccounting.ts"; import { startGrokBrokerProxy } from "../src/runtime/grokBrokerProxy.ts"; +import { GrokBrokerTurnMeter } from "../src/runtime/grokBrokerTurnMeter.ts"; import { DEFAULT_GROK_BROKER_MODEL_POLICY } from "../src/runtime/grokBrokerModelPolicy.ts"; import { GROK_BROKER_PROVIDER_CAPABILITY_ENV, renderGrokBrokerWorkerArgs, renderGrokBrokerWorkerConfigWith } from "../src/runtime/grokBrokerWorkerConfig.ts"; @@ -38,6 +40,8 @@ try { const capability = proxy.capabilities.issue("local-auth-probe", turnId); // This local transport probe deliberately does not attest a native worker. proxy.registerIsolationGuard(turnId, async () => undefined); + // The proxy forwards nothing unmetered; the probe runs under the default v1 limits. + proxy.registerTurn(turnId, { policy: DEFAULT_GROK_BROKER_MODEL_POLICY, meter: new GrokBrokerTurnMeter(DEFAULT_GROK_BROKER_TURN_LIMITS) }); // No MCP tools are needed for this exact-reply authentication probe. await writeFile(path.join(home, "config.toml"), renderGrokBrokerWorkerConfigWith(DEFAULT_GROK_BROKER_MODEL_POLICY, { proxyPort: proxy.port, mcpUrl: "http://127.0.0.1:43124/mcp" }).split("[mcp_servers.daimon]")[0]); const prompt = path.join(home, "prompt.txt"); diff --git a/src/runtime/engineDispatcher.test.ts b/src/runtime/engineDispatcher.test.ts index ef80d32..a5649b5 100644 --- a/src/runtime/engineDispatcher.test.ts +++ b/src/runtime/engineDispatcher.test.ts @@ -243,9 +243,14 @@ test("production Grok dispatcher routes every wake through the broker without ag process.env.PATH = `${root}${path.delimiter}${priorPath ?? ""}`; process.env.NOOPOLIS_RUN_ID = "dispatcher-grok-realm-test"; const broker: EngineBrokerTurnClient = { - async turn(agentId,wakeId,prompt,endpoint,signal) { turns += 1;assert.equal(agentId,config.id);assert.match(wakeId,/^(first|second)$/u);assert.match(prompt,/work/u);assert.match(endpoint,/^http:\/\/127\.0\.0\.1:\d+\/mcp$/u);assert.equal(signal?.aborted,false);return "brokered"; } + async turn(agentId,wakeId,prompt,endpoint,signal,options) { turns += 1;assert.equal(agentId,config.id);assert.match(wakeId,/^(first|second)$/u);assert.match(prompt,/work/u);assert.match(endpoint,/^http:\/\/127\.0\.0\.1:\d+\/mcp$/u);assert.equal(signal?.aborted,false); + // The engine-neutral wake bound reaches the broker as a lowering limit. + assert.deepEqual(options,{limits:{maxTokens:123_456}});return "brokered"; } }; - const handle = await startOrganizationRuntimeEngine(config, "DAIMON_UNUSED_CONTROL", undefined, undefined, broker); + const priorCeiling = process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING; process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING = "123456"; + let handle: Awaited>; + try { handle = await startOrganizationRuntimeEngine(config, "DAIMON_UNUSED_CONTROL", undefined, undefined, broker); } + finally { if (priorCeiling === undefined) delete process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING; else process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING = priorCeiling; } assert.equal((await handle.wake({ id: "first", kind: "manual", text: "work" })).text, "brokered"); assert.equal((await handle.wake({ id: "second", kind: "manual", text: "work" })).text, "brokered"); assert.equal(turns, 2); diff --git a/src/runtime/engineDispatcher.ts b/src/runtime/engineDispatcher.ts index 9414752..cbb5ad2 100644 --- a/src/runtime/engineDispatcher.ts +++ b/src/runtime/engineDispatcher.ts @@ -2,7 +2,7 @@ import { attentionTools, type AttentionRegistry } from "./attention.js"; import path from "node:path"; import type { AgentHandle } from "../core/types.js"; -import { AGY_MAX_TOOL_TURNS, createCliSessionFactory, resolveCodexWakeTimeoutMs, resolveCodexWakeTokenCeiling } from "../pi/cliSession.js"; +import { AGY_MAX_TOOL_TURNS, createCliSessionFactory, resolveCodexWakeTimeoutMs, resolveCodexWakeTokenCeiling, resolveEngineWakeLimitOverrides } from "../pi/cliSession.js"; import { GROK_DAIMON_SANDBOX_PROFILE, prepareAndVerifyGrokSandbox @@ -151,7 +151,10 @@ function adapterFor(agent: OrganizationRuntimeAgentConfig, controlTokenEnv: stri }) } : {}), ...(engine==="grok"&&grokBroker!==undefined?{}:{credentialSecretValues: () => readPortableEngineCredentialSecrets(agent.id, engine, engineHomePath)}), - ...(engine==="grok"&&grokBroker!==undefined?{grokBrokerTurn:(prompt:string,endpoint:string,signal:AbortSignal)=>grokBroker.turn(agent.id,wakeEnvironmentContext.current??"wake",prompt,endpoint,signal)}:{}), + // The broker seals usage and enforces its registration's limits; the + // wake may only lower them (DAIMON_ENGINE_WAKE_*), and a declared model + // must be the one the broker reports it ran. + ...(engine==="grok"&&grokBroker!==undefined?{grokBrokerTurn:grokBrokerTurnFor(agent,grokBroker,wakeEnvironmentContext)}:{}), ...(engine === "grok" && verifyGrokSandbox ? { grokSandboxProfile: GROK_DAIMON_SANDBOX_PROFILE, verifyGrokSandbox @@ -160,6 +163,12 @@ function adapterFor(agent: OrganizationRuntimeAgentConfig, controlTokenEnv: stri return cliHarness(agent, sessionFactory, [controlTokenEnv], productionTools, wakeEnvironmentContext); } +function grokBrokerTurnFor(agent: OrganizationRuntimeAgentConfig, grokBroker: EngineBrokerTurnClient, wakeEnvironmentContext: import("../pi/piAgentWakeSupport.js").PiWakeEnvironmentContextRef) { + const limits = resolveEngineWakeLimitOverrides(); + const options = { ...(limits === undefined ? {} : { limits }), ...(agent.engine.model === undefined ? {} : { model: agent.engine.model }) }; + return (prompt: string, endpoint: string, signal: AbortSignal) => grokBroker.turn(agent.id, wakeEnvironmentContext.current ?? "wake", prompt, endpoint, signal, options); +} + /** * CLI engines do not consume Pi's resource loader. Frame the same immutable * identity in JSON so arbitrary names/instructions cannot change its shape. From ae8dd17276bd70453beda90e87b09edf788633d7 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:15:16 +0200 Subject: [PATCH 033/124] feat: accept closed-list Grok model and reasoning effort and pin the broker accounting contract in the manifest --- src/contracts/organizationRuntimeContract.ts | 8 ++- src/contracts/runtimeContractManifest.ts | 15 ++++++ src/runtime/engineBrokerContract.test.ts | 17 +++++++ src/runtime/engineBrokerTurnAccounting.ts | 20 ++++---- src/runtime/engineDispatcher.test.ts | 5 ++ src/runtime/engineDispatcher.ts | 3 ++ src/runtime/organizationRuntime.test.ts | 51 ++++++++++++++++---- src/runtime/organizationRuntime.ts | 8 +-- src/runtime/organizationRuntimeParsing.ts | 32 +++++++++--- 9 files changed, 127 insertions(+), 32 deletions(-) create mode 100644 src/runtime/engineBrokerContract.test.ts diff --git a/src/contracts/organizationRuntimeContract.ts b/src/contracts/organizationRuntimeContract.ts index 8f2b9ca..ce87b07 100644 --- a/src/contracts/organizationRuntimeContract.ts +++ b/src/contracts/organizationRuntimeContract.ts @@ -1,3 +1,5 @@ +import { GROK_BROKER_MODELS, GROK_BROKER_REASONING_EFFORTS } from "./grokWorkerContract.js"; + /** The data-only organization-runtime constants shared by product code and artifacts. */ export const ORGANIZATION_RUNTIME_VERSION = "noopolis.daimon.organization-runtime.v1" as const; export const ORGANIZATION_RUNTIME_V2_VERSION = "noopolis.daimon.organization-runtime.v2" as const; @@ -74,7 +76,11 @@ export const ORGANIZATION_RUNTIME_CONFIG_SCHEMA = { codexSandbox: { type: "object", additionalProperties: false, required: ["mode", "networkAccess", "webSearch"], properties: { mode: { const: "workspace-write" }, networkAccess: { const: false }, webSearch: { const: "disabled" } } } - } }, + }, allOf: [ + // grok: a declared model is the closed broker pair, both or neither; never a Codex sandbox. + { if: { properties: { kind: { const: "grok" } } }, then: { properties: { model: { enum: GROK_BROKER_MODELS }, reasoningEffort: { enum: GROK_BROKER_REASONING_EFFORTS }, codexSandbox: false }, dependentRequired: { model: ["reasoningEffort"], reasoningEffort: ["model"] } } }, + { if: { properties: { kind: { const: "agy" } } }, then: { properties: { model: false, reasoningEffort: false, codexSandbox: false } } } + ] }, ...PRODUCTION_TOOL_PROPERTIES } } } diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 20ae6be..ceef199 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -68,6 +68,21 @@ export const GROK_ENGINE_BROKER = { } }, bounds: { promptBytes: 65_536, capabilityBytes: 4_096, capabilityBundleBytes: 8_196, outputBytes: 65_536 }, + // Accounting and limits (P2). The broker is the single sealed usage writer. + controlProtocolVersion: "noopolis.daimon.engine-broker.v2", + turnRecordVersions: ["noopolis.daimon.engine-broker-turn.v1", "noopolis.daimon.engine-broker-turn.v2"], + serviceConfigVersions: ["noopolis.daimon.engine-broker-service.v1", "noopolis.daimon.engine-broker-service.v2"], + turnLimits: { + keys: ["maxRequests", "maxTokens", "timeoutMs"], + v1Defaults: { maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }, + bounds: { maxRequests: [1, GROK_WORKER_MAX_TURNS], maxTokens: [1, 10_000_000], timeoutMs: [1_000, 3_600_000] }, + limitReasons: ["tokens", "requests", "timeout", "none"], + wakeMayOnlyLower: true, + tokenCeilingOvershoot: "at-most-one-request" + }, + wakeLimitEnvironment: { timeoutMs: "DAIMON_ENGINE_WAKE_TIMEOUT_MS", maxTokens: "DAIMON_ENGINE_WAKE_TOKEN_CEILING" }, + projectionVersion: "noopolis.daimon.grok-broker-projection.v1", + slotPreflightVersion: "noopolis.daimon.grok-slot-preflight.v1", artifacts: { sourceSha256: "36f60689f0a8af0e3108f5f53d78ed52b7d4b6f934c75b6184606dfa82bc741e", x64Sha256: "36dc76b134eb59cf5a6720b6f94228eb279108e20ea3343fa6efd9ffcb60a4d3", diff --git a/src/runtime/engineBrokerContract.test.ts b/src/runtime/engineBrokerContract.test.ts new file mode 100644 index 0000000..17ae209 --- /dev/null +++ b/src/runtime/engineBrokerContract.test.ts @@ -0,0 +1,17 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import { ENGINE_BROKER_VERSION } from "./engineBrokerProtocol.js"; +import { ENGINE_BROKER_SERVICE_V1, ENGINE_BROKER_SERVICE_V2 } from "./engineBrokerServiceConfig.js"; +import { ENGINE_BROKER_TURN_RECORD_V1, ENGINE_BROKER_TURN_RECORD_V2 } from "./engineBrokerTurnRegistry.js"; +import { DEFAULT_CODEX_WAKE_TIMEOUT_MS, DEFAULT_CODEX_WAKE_TOKEN_CEILING, DAIMON_ENGINE_WAKE_TIMEOUT_MS_ENV, DAIMON_ENGINE_WAKE_TOKEN_CEILING_ENV } from "../pi/engineWakeLimits.js"; + +test("the manifest pins the broker accounting contract the runtime actually speaks", () => { + assert.equal(GROK_ENGINE_BROKER.controlProtocolVersion, ENGINE_BROKER_VERSION); + assert.deepEqual(GROK_ENGINE_BROKER.turnRecordVersions, [ENGINE_BROKER_TURN_RECORD_V1, ENGINE_BROKER_TURN_RECORD_V2]); + assert.deepEqual(GROK_ENGINE_BROKER.serviceConfigVersions, [ENGINE_BROKER_SERVICE_V1, ENGINE_BROKER_SERVICE_V2]); + assert.deepEqual(GROK_ENGINE_BROKER.wakeLimitEnvironment, { timeoutMs: DAIMON_ENGINE_WAKE_TIMEOUT_MS_ENV, maxTokens: DAIMON_ENGINE_WAKE_TOKEN_CEILING_ENV }); + assert.deepEqual([GROK_ENGINE_BROKER.turnLimits.v1Defaults.timeoutMs, GROK_ENGINE_BROKER.turnLimits.v1Defaults.maxTokens], [DEFAULT_CODEX_WAKE_TIMEOUT_MS, DEFAULT_CODEX_WAKE_TOKEN_CEILING]); + assert.ok(GROK_ENGINE_BROKER.turnLimits.bounds.maxRequests[1] <= GROK_ENGINE_BROKER.worker.maxTurns, "the broker request ceiling fires before the launcher --max-turns backstop"); +}); diff --git a/src/runtime/engineBrokerTurnAccounting.ts b/src/runtime/engineBrokerTurnAccounting.ts index 951c4f4..7f7f346 100644 --- a/src/runtime/engineBrokerTurnAccounting.ts +++ b/src/runtime/engineBrokerTurnAccounting.ts @@ -1,4 +1,5 @@ -import { GROK_BROKER_MODELS, GROK_WORKER_MAX_TURNS } from "../contracts/grokWorkerContract.js"; +import { GROK_BROKER_MODELS } from "../contracts/grokWorkerContract.js"; +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; import type { GrokBrokerModel } from "./grokBrokerModelPolicy.js"; /** @@ -15,7 +16,7 @@ import type { GrokBrokerModel } from "./grokBrokerModelPolicy.js"; * inside `output` and never added to `total`. */ export type EngineBrokerTurnUsage = Readonly<{ input: number; cacheRead: number; cacheWrite: number; output: number; total: number; reasoning?: number }>; -export const ENGINE_BROKER_LIMIT_REASONS = ["tokens", "requests", "timeout", "none"] as const; +export const ENGINE_BROKER_LIMIT_REASONS = GROK_ENGINE_BROKER.turnLimits.limitReasons; export type EngineBrokerLimitReason = (typeof ENGINE_BROKER_LIMIT_REASONS)[number]; export type EngineBrokerTurnLimits = Readonly<{ maxRequests: number; maxTokens: number; timeoutMs: number }>; export type EngineBrokerTurnLimitOverrides = Readonly>; @@ -28,18 +29,15 @@ export type EngineBrokerTurnAccounting = Readonly<{ }>; /** - * Bounds every declared limit must sit inside. `maxRequests` stays at or below - * the launcher's compiled `--max-turns` backstop, so the broker ceiling is the - * one that fires first. + * Bounds every declared limit must sit inside, and the v1 defaults, both from + * the runtime contract manifest. `maxRequests` stays at or below the + * launcher's compiled `--max-turns` backstop, so the broker ceiling is the one + * that fires first. */ -export const ENGINE_BROKER_LIMIT_BOUNDS = Object.freeze({ - maxRequests: [1, GROK_WORKER_MAX_TURNS], - maxTokens: [1, 10_000_000], - timeoutMs: [1_000, 3_600_000] -} as const); +export const ENGINE_BROKER_LIMIT_BOUNDS = GROK_ENGINE_BROKER.turnLimits.bounds; /** What a v1 `service.json` registration gets; equal to the Codex per-wake defaults. */ -export const DEFAULT_GROK_BROKER_TURN_LIMITS: EngineBrokerTurnLimits = Object.freeze({ maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }); +export const DEFAULT_GROK_BROKER_TURN_LIMITS: EngineBrokerTurnLimits = Object.freeze({ ...GROK_ENGINE_BROKER.turnLimits.v1Defaults }); const LIMIT_KEYS = ["maxRequests", "maxTokens", "timeoutMs"] as const; type JsonRecord = Record; diff --git a/src/runtime/engineDispatcher.test.ts b/src/runtime/engineDispatcher.test.ts index a5649b5..488a658 100644 --- a/src/runtime/engineDispatcher.test.ts +++ b/src/runtime/engineDispatcher.test.ts @@ -401,3 +401,8 @@ async function seedAuth(root: string, kind: "codex" | "grok" | "agy"): Promise { + const config = { ...rootConfig("/tmp/daimon-unused-direct-grok", "grok"), engine: { kind: "grok", model: "grok-4.6", reasoningEffort: "low" } } as OrganizationRuntimeAgentConfig; + await assert.rejects(startOrganizationRuntimeEngine(config, "DAIMON_UNUSED_CONTROL"), /requires the engine broker/u); +}); diff --git a/src/runtime/engineDispatcher.ts b/src/runtime/engineDispatcher.ts index cbb5ad2..4666aea 100644 --- a/src/runtime/engineDispatcher.ts +++ b/src/runtime/engineDispatcher.ts @@ -32,6 +32,9 @@ export async function startOrganizationRuntimeEngine( sharedProtectedPaths: readonly string[] = [], attention?: AttentionRegistry ): Promise { + // A declared Grok model is enforced by the broker proxy and worker config; + // the direct path has neither, so it refuses rather than silently ignoring it. + if (agent.engine.kind === "grok" && agent.engine.model !== undefined && grokBroker === undefined) throw new Error(`Agent ${agent.id} declares a Grok model, which requires the engine broker`); await paths?.verify(); const canonicalAgent = paths === undefined ? agent : { ...agent, workspacePath: paths.workspacePath, runtimeHomePath: paths.runtimeHomePath }; const readiness = canonicalAgent.engine.kind === "grok" && grokBroker !== undefined diff --git a/src/runtime/organizationRuntime.test.ts b/src/runtime/organizationRuntime.test.ts index f5f1c7f..3636225 100644 --- a/src/runtime/organizationRuntime.test.ts +++ b/src/runtime/organizationRuntime.test.ts @@ -271,6 +271,20 @@ test("the engine JSON Schema and the parser agree on model/reasoningEffort", () } }); +test("the engine JSON Schema and the parser agree on grok and agy model declarations", async () => { + const { Ajv2020 } = await import("ajv/dist/2020.js") as unknown as { Ajv2020: new (options: Record) => { compile(schema: unknown): (value: unknown) => boolean } }; + const validate = new Ajv2020({ strict: false }).compile(ORGANIZATION_RUNTIME_CONFIG_SCHEMA); + for (const engine of [ + { kind: "grok" }, { kind: "grok", model: "grok-4.6", reasoningEffort: "low" }, { kind: "grok", model: "grok-4.6" }, + { kind: "grok", model: "grok-4.6-build", reasoningEffort: "low" }, { kind: "grok", model: "grok-4.5", reasoningEffort: "xhigh" }, + { kind: "agy" }, { kind: "agy", model: "x" }, { kind: "codex", model: "gpt-5-codex", reasoningEffort: "xhigh" } + ]) { + const config = valid(); + config.agents[0]!.engine = engine as never; + assert.equal(validate(config), validateOrganizationRuntimeConfig(config), JSON.stringify(engine)); + } +}); + test("accepts only the narrow optional Codex workspace policy", () => { const config = valid(); config.agents[0]!.engine = { kind: "codex", codexSandbox: { mode: "workspace-write", networkAccess: false, webSearch: "disabled" } } as never; @@ -287,22 +301,41 @@ test("accepts only the narrow optional Codex workspace policy", () => { } }); -test("rejects model and reasoningEffort on a non-codex engine", () => { +test("rejects model and reasoningEffort on agy, and codexSandbox on every non-codex engine", () => { + const withModel = valid(); + withModel.agents[0]!.engine = { kind: "agy", model: "gpt-5-codex" } as never; + assert.throws(() => parseOrganizationRuntimeConfig(withModel), /codex-only/); + const withEffort = valid(); + withEffort.agents[0]!.engine = { kind: "agy", reasoningEffort: "high" } as never; + assert.throws(() => parseOrganizationRuntimeConfig(withEffort), /codex-only/); for (const kind of ["grok", "agy"] as const) { - const withModel = valid(); - withModel.agents[0]!.engine = { kind, model: "gpt-5-codex" } as never; - assert.throws(() => parseOrganizationRuntimeConfig(withModel), /codex-only/); - - const withEffort = valid(); - withEffort.agents[0]!.engine = { kind, reasoningEffort: "high" } as never; - assert.throws(() => parseOrganizationRuntimeConfig(withEffort), /codex-only/); - const withPolicy = valid(); withPolicy.agents[0]!.engine = { kind, codexSandbox: { mode: "workspace-write", networkAccess: false, webSearch: "disabled" } } as never; assert.throws(() => parseOrganizationRuntimeConfig(withPolicy), /codex-only/); } }); +test("grok declares model and reasoning effort together from the closed broker lists, never half-inherited", () => { + const declared = valid(); + declared.agents[0]!.engine = { kind: "grok", model: "grok-4.6", reasoningEffort: "low" } as never; + assert.deepEqual(parseOrganizationRuntimeConfig(declared).agents[0]!.engine, { kind: "grok", model: "grok-4.6", reasoningEffort: "low" }); + const bare = valid(); + bare.agents[0]!.engine = { kind: "grok" } as never; + assert.deepEqual(parseOrganizationRuntimeConfig(bare).agents[0]!.engine, { kind: "grok" }); + for (const engine of [ + { kind: "grok", model: "grok-4.6" }, + { kind: "grok", reasoningEffort: "low" }, + { kind: "grok", model: "grok-4.6-build", reasoningEffort: "low" }, + { kind: "grok", model: "gpt-5-codex", reasoningEffort: "low" }, + { kind: "grok", model: "grok-4.6", reasoningEffort: "xhigh" }, + { kind: "grok", model: "grok-4.6", reasoningEffort: "low", provider: "xai" } + ]) { + const invalid = valid(); + invalid.agents[0]!.engine = engine as never; + assert.throws(() => parseOrganizationRuntimeConfig(invalid), TypeError, JSON.stringify(engine)); + } +}); + const withMemory = (agent: Record, memory: unknown): Record => ({ ...agent, memory }); test("parses a declared memory bank and round-trips its fields", () => { diff --git a/src/runtime/organizationRuntime.ts b/src/runtime/organizationRuntime.ts index 76684fe..f201306 100644 --- a/src/runtime/organizationRuntime.ts +++ b/src/runtime/organizationRuntime.ts @@ -35,10 +35,10 @@ export { export type OrganizationRuntimeEngineKind = "codex" | "grok" | "agy"; /** - * `model`/`reasoningEffort` are codex-only: grok and agy own their own model - * selection (their subscription auth and model selection are Daimon-owned), - * and `organizationRuntimeParsing.ts` rejects either field on a non-codex - * engine at parse time rather than silently ignoring it. The type stays flat + * `model`/`reasoningEffort` are accepted for codex (open model name) and for + * grok (closed broker lists, declared together); agy owns its own model + * selection and `organizationRuntimeParsing.ts` rejects either field there at + * parse time rather than silently ignoring it. The type stays flat * — not a `kind`-discriminated union — because every parsed value already * satisfies the invariant; callers that need it narrow on `kind === "codex"`. */ diff --git a/src/runtime/organizationRuntimeParsing.ts b/src/runtime/organizationRuntimeParsing.ts index a6f1d39..cc65b39 100644 --- a/src/runtime/organizationRuntimeParsing.ts +++ b/src/runtime/organizationRuntimeParsing.ts @@ -1,3 +1,4 @@ +import { GROK_BROKER_MODELS, GROK_BROKER_REASONING_EFFORTS } from "../contracts/grokWorkerContract.js"; import { parseAttention } from "./attention.js"; import { ORGANIZATION_RUNTIME_CODEX_REASONING_EFFORTS, @@ -187,18 +188,19 @@ function cronValues(field: string, [minimum, maximum]: readonly [number, number] function engine(value: unknown, label: string): OrganizationRuntimeEngineIntent { const input = object(value, label); const kind = string(input.kind, `${label}.kind`); if (!ENGINE_KINDS.has(kind)) throw new TypeError(`${label}.kind is not a supported engine`); - // `model`/`reasoningEffort` are codex-only: grok and agy own their own model - // selection, so either field on a non-codex engine is rejected explicitly - // here (a clear, named error) rather than falling through to the generic - // "must contain exactly" rejection every other unexpected key gets below. - const allowed = kind === "codex" ? ["kind", "model", "reasoningEffort", "codexSandbox"] : ["kind"]; + // `codexSandbox` is codex-only; `model`/`reasoningEffort` are accepted for + // codex (open model name, Codex effort list) and for grok (closed broker + // lists, declared together or not at all). agy owns its own model selection. + const allowed = kind === "codex" ? ["kind", "model", "reasoningEffort", "codexSandbox"] : kind === "grok" ? ["kind", "model", "reasoningEffort"] : ["kind"]; const extras = Object.keys(input).filter((key) => !allowed.includes(key)); if (extras.length > 0) { - if (kind !== "codex" && (extras.includes("model") || extras.includes("reasoningEffort") || extras.includes("codexSandbox"))) { - throw new TypeError(`${label}.model, ${label}.reasoningEffort, and ${label}.codexSandbox are codex-only; their subscription auth, model selection, and sandbox policy are Daimon-owned`); + if (kind === "agy" && (extras.includes("model") || extras.includes("reasoningEffort") || extras.includes("codexSandbox"))) { + throw new TypeError(`${label}.model, ${label}.reasoningEffort, and ${label}.codexSandbox are codex-only for agy; its subscription auth, model selection, and sandbox policy are Daimon-owned`); } + if (kind === "grok" && extras.includes("codexSandbox")) throw new TypeError(`${label}.codexSandbox is codex-only; the Grok worker sandbox is Daimon-owned`); throw new TypeError(`${label} must contain exactly ${allowed.join(", ")}`); } + if (kind === "grok") return grokEngine(input, label); return { kind: kind as OrganizationRuntimeEngineKind, ...(input.model === undefined ? {} : { model: nonEmpty(input.model, `${label}.model`) }), @@ -207,6 +209,22 @@ function engine(value: unknown, label: string): OrganizationRuntimeEngineIntent }; } +/** + * A Grok model is declared explicitly or not at all: both members from the + * closed broker lists, never one of them with the other inherited. Omitting + * both keeps a config parseable by pre-declaration producers; the brokered + * paths that need a model (`grokBrokerProjection.ts`, `service.json` v2) + * require it there instead of defaulting it here. + */ +function grokEngine(input: RecordValue, label: string): OrganizationRuntimeEngineIntent { + if ((input.model === undefined) !== (input.reasoningEffort === undefined)) throw new TypeError(`${label}.model and ${label}.reasoningEffort must be declared together for grok`); + if (input.model === undefined) return { kind: "grok" }; + const model = string(input.model, `${label}.model`), effort = string(input.reasoningEffort, `${label}.reasoningEffort`); + if (!(GROK_BROKER_MODELS as readonly string[]).includes(model)) throw new TypeError(`${label}.model must be one of ${GROK_BROKER_MODELS.join(", ")}`); + if (!(GROK_BROKER_REASONING_EFFORTS as readonly string[]).includes(effort)) throw new TypeError(`${label}.reasoningEffort must be one of ${GROK_BROKER_REASONING_EFFORTS.join(", ")}`); + return { kind: "grok", model, reasoningEffort: effort }; +} + function codexSandbox(value: unknown, label: string): OrganizationRuntimeEngineIntent["codexSandbox"] { const input = object(value, label); exact(input, ["mode", "networkAccess", "webSearch"], label); From 48e9b8b6178610bce48289ea6277cccf1a3a3522 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:17:49 +0200 Subject: [PATCH 034/124] feat: add the public Grok broker projection --- src/runtime/grokBrokerProjection.test.ts | 52 +++++++++ src/runtime/grokBrokerProjection.ts | 130 +++++++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 src/runtime/grokBrokerProjection.test.ts create mode 100644 src/runtime/grokBrokerProjection.ts diff --git a/src/runtime/grokBrokerProjection.test.ts b/src/runtime/grokBrokerProjection.test.ts new file mode 100644 index 0000000..aa8689a --- /dev/null +++ b/src/runtime/grokBrokerProjection.test.ts @@ -0,0 +1,52 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { GROK_ENGINE_BROKER, GROK_SUBSCRIPTION_REALM } from "../contracts/runtimeContractManifest.js"; +import { parseEngineBrokerServiceConfig } from "./engineBrokerServiceConfig.js"; +import { grokBrokerProjectionSha256, grokBrokerServiceRegistrationFor, resolveOrganizationGrokBrokerProjection, verifyGrokBrokerRegistrationMatchesProjection } from "./grokBrokerProjection.js"; +import { grokBrokerWorkerConfigSha256 } from "./grokBrokerWorkerConfig.js"; +import { grokWorkerSandboxProfileSha256 } from "./grokWorkerSandboxProfile.js"; + +const agent = (id: string, engine: Record) => ({ id, name: id, instructions: "Unused", workspacePath: `/var/lib/spawnfile/instance/workspace/agents/${id}`, runtimeHomePath: `/var/lib/spawnfile/instance/homes/${id}`, schedule: { kind: "disabled" }, engine }); +const config = { version: "noopolis.daimon.organization-runtime.v2", host: { bindHost: "127.0.0.1", port: 19700, controlTokenEnv: "UNIT_CONTROL_TOKEN" }, + agents: [agent("foreman", { kind: "grok", model: "grok-4.6", reasoningEffort: "low" }), agent("peer", { kind: "codex" })] }; +const options = { slot: 0, workerUid: 2_200, workerHomePath: "/var/lib/daimon-workers/2200", architecture: "arm64", usageLedgerPath: "/run/slots/0/usage/usage.jsonl", + limits: { maxRequests: 24, maxTokens: 400_000, timeoutMs: 480_000 }, acceptanceStorePath: "/run/paideia/control", denyPaths: ["/run/paideia", "/run/training/inputs"] } as const; + +test("the projection is Daimon's own renderers and collectors, fully declared and deterministic", () => { + const projection = resolveOrganizationGrokBrokerProjection(config, "foreman", options); + const denyPaths = [GROK_SUBSCRIPTION_REALM.bootstrapMountPath, GROK_SUBSCRIPTION_REALM.durableMountPath, "/run/paideia/control", "/var/lib/spawnfile/instance/homes/peer", "/var/lib/spawnfile/instance/workspace/agents/peer", "/run/paideia", "/run/training/inputs"].sort(); + assert.deepEqual(projection, { + version: "noopolis.daimon.grok-broker-projection.v1", agentId: "foreman", + workspacePath: "/var/lib/spawnfile/instance/workspace/agents/foreman", runtimeHomePath: "/var/lib/spawnfile/instance/homes/foreman", + workerUid: 2_200, slot: 0, profilePath: "/var/lib/daimon-workers/2200/.grok/sandbox.toml", profileSha256: grokWorkerSandboxProfileSha256(denyPaths), denyPaths, + workerConfigSha256: grokBrokerWorkerConfigSha256({ model: "grok-4.6", reasoningEffort: "low" }), systemPromptSha256: GROK_ENGINE_BROKER.worker.systemPromptSha256, + grokCliVersion: "1.0.34", grokExecutableSha256: GROK_ENGINE_BROKER.grokCliArtifacts.arm64.sha256, nativeAbiVersion: GROK_ENGINE_BROKER.nativeAbiVersion, + model: "grok-4.6", reasoningEffort: "low", limits: options.limits, usageLedgerPath: options.usageLedgerPath, + attestation: { platform: "linux/landlock", enforced: true, restrictNetwork: true, profileName: "daimon-strict", eventsPath: "/var/lib/daimon-workers/2200/.grok/sessions/sandbox-events.jsonl" } + }); + assert.equal(grokBrokerProjectionSha256(resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, denyPaths: [...options.denyPaths].reverse() })), grokBrokerProjectionSha256(projection)); + assert.match(grokBrokerProjectionSha256(projection), /^[a-f0-9]{64}$/u); +}); + +test("the projection refuses undeclared models, non-Grok agents, and a profile digest it did not render", () => { + assert.throws(() => resolveOrganizationGrokBrokerProjection({ ...config, agents: [agent("foreman", { kind: "grok" })] }, "foreman", options), /declared model/u); + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "peer", options), /known Grok agent/u); + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "missing", options), /known Grok agent/u); + // Mutation guard: skipping the digest comparison accepts a weaker profile's digest. + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, profileSha256: grokWorkerSandboxProfileSha256([]) }), /profile digest mismatch/u); + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, denyPaths: ["relative"] }), /deny path/u); + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, limits: { ...options.limits, maxRequests: 49 } }), /invalid/u); + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, usageLedgerPath: "/run/slots/0/usage/requests.jsonl" }), /invalid engine broker service config/u); + assert.equal(resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, profileSha256: resolveOrganizationGrokBrokerProjection(config, "foreman", options).profileSha256 }).agentId, "foreman"); +}); + +test("a provisioned registration must describe its projection exactly", () => { + const projection = resolveOrganizationGrokBrokerProjection(config, "foreman", options); + const parse = (registration: Record) => parseEngineBrokerServiceConfig({ version: "noopolis.daimon.engine-broker-service.v2", credentialHome: "/c", turnStore: "/t", registrations: [registration] }).registrations[0]!; + const registration = grokBrokerServiceRegistrationFor(projection); + verifyGrokBrokerRegistrationMatchesProjection(parse(registration), projection); + for (const drift of [{ profileSha256: grokWorkerSandboxProfileSha256([]) }, { model: { id: "grok-4.5", reasoningEffort: "low" } }, { limits: { ...options.limits, maxTokens: 400_001 } }, { usageLedgerPath: "/run/slots/1/usage/usage.jsonl" }]) { + assert.throws(() => verifyGrokBrokerRegistrationMatchesProjection(parse({ ...registration, ...drift }), projection), /does not match/u, JSON.stringify(drift)); + } +}); diff --git a/src/runtime/grokBrokerProjection.ts b/src/runtime/grokBrokerProjection.ts new file mode 100644 index 0000000..ac1e5f6 --- /dev/null +++ b/src/runtime/grokBrokerProjection.ts @@ -0,0 +1,130 @@ +import { createHash } from "node:crypto"; +import path from "node:path"; + +import { canonicalJson } from "../contracts/canonicalJson.js"; +import { DAIMON_GROK_SYSTEM_PROMPT } from "../contracts/grokWorkerContract.js"; +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import { grokSandboxProtectedPaths } from "./engineDispatcher.js"; +import { parseEngineBrokerServiceConfig, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +import { parseEngineBrokerTurnLimits, type EngineBrokerTurnLimits } from "./engineBrokerTurnAccounting.js"; +import { grokBrokerWorkerConfigSha256 } from "./grokBrokerWorkerConfig.js"; +import type { GrokBrokerModel, GrokBrokerReasoningEffort } from "./grokBrokerModelPolicy.js"; +import { GROK_WORKER_SANDBOX_PROFILE, grokWorkerEventsPathFor, renderGrokWorkerSandboxProfile, grokWorkerSandboxProfileSha256 } from "./grokWorkerSandboxProfile.js"; +import { parseOrganizationRuntimeConfig } from "./organizationRuntime.js"; + +export const GROK_BROKER_PROJECTION_VERSION = GROK_ENGINE_BROKER.projectionVersion; + +/** + * Everything a consumer (Spawnfile provisioning, Paideia's native adapter, the + * root slot supervisor) needs to know about one brokered Grok agent's slot, + * computed by Daimon from the same renderers and collectors the broker attests + * against. It never reads credentials, runs a worker, or touches the realm. + */ +export type OrganizationGrokBrokerProjection = Readonly<{ + version: typeof GROK_BROKER_PROJECTION_VERSION; + agentId: string; + workspacePath: string; + runtimeHomePath: string; + workerUid: number; + slot: number; + profilePath: string; + profileSha256: string; + denyPaths: readonly string[]; + workerConfigSha256: string; + systemPromptSha256: string; + grokCliVersion: string; + grokExecutableSha256: string; + nativeAbiVersion: number; + model: GrokBrokerModel; + reasoningEffort: GrokBrokerReasoningEffort; + limits: EngineBrokerTurnLimits; + usageLedgerPath: string; + attestation: Readonly<{ platform: "linux/landlock"; enforced: true; restrictNetwork: true; profileName: typeof GROK_WORKER_SANDBOX_PROFILE; eventsPath: string }>; +}>; + +export type OrganizationGrokBrokerProjectionOptions = Readonly<{ + /** Deployment-assigned slot identity. */ + slot: number; + workerUid: number; + /** The worker's home; its `GROK_HOME` is `/.grok`. */ + workerHomePath: string; + architecture: "arm64" | "x64"; + usageLedgerPath: string; + limits: EngineBrokerTurnLimits; + /** The wake-acceptance store, always denied like the Codex projection's. */ + acceptanceStorePath: string; + /** Evaluator and host-bind paths the deployment must keep from the worker (R4). */ + denyPaths?: readonly string[]; + /** When the caller already holds a rendered profile digest, it must equal Daimon's. */ + profileSha256?: string; +}>; + +/** + * Resolve the public Grok broker projection for one agent. + * + * Deterministic and I/O-free on purpose: its digest + * ({@link grokBrokerProjectionSha256}) is what the slot preflight receipt + * binds, so the supervisor that writes the receipt and the evaluator that reads + * it must compute byte-equal projections from the same inputs. + * + * The agent must be a Grok agent that *declares* its model and reasoning + * effort; nothing is defaulted. The deny list is Daimon's own protected set for + * this agent (realm, bootstrap, peers, acceptance store) plus the caller's + * evaluator paths, sorted and deduplicated exactly as the profile renderer + * does. A supplied `profileSha256` that differs is refused. + */ +export function resolveOrganizationGrokBrokerProjection(config: unknown, agentId: string, options: OrganizationGrokBrokerProjectionOptions): OrganizationGrokBrokerProjection { + const parsed = parseOrganizationRuntimeConfig(config); + const agent = parsed.agents.find((entry) => entry.id === agentId); + if (agent === undefined || agent.engine.kind !== "grok") throw new Error("Grok broker projection requires a known Grok agent"); + if (agent.engine.model === undefined || agent.engine.reasoningEffort === undefined) throw new Error("Grok broker projection requires a declared model and reasoning effort"); + for (const [label, value] of [["workerHomePath", options.workerHomePath], ["acceptanceStorePath", options.acceptanceStorePath]] as const) { + if (!path.posix.isAbsolute(value) || path.posix.normalize(value) !== value || value === "/" || value.endsWith("/")) throw new Error(`Grok broker projection requires a canonical absolute ${label}`); + } + const model = agent.engine.model as GrokBrokerModel, reasoningEffort = agent.engine.reasoningEffort as GrokBrokerReasoningEffort; + const denyPaths = [...new Set([...grokSandboxProtectedPaths(agent.id, parsed.agents, [options.acceptanceStorePath]), ...(options.denyPaths ?? [])])].sort(); + renderGrokWorkerSandboxProfile(denyPaths); + const profileSha256 = grokWorkerSandboxProfileSha256(denyPaths); + if (options.profileSha256 !== undefined && options.profileSha256 !== profileSha256) throw new Error("Grok broker projection profile digest mismatch"); + const workerConfigSha256 = grokBrokerWorkerConfigSha256({ model, reasoningEffort }); + if (workerConfigSha256 !== GROK_ENGINE_BROKER.worker.configSha256[model][reasoningEffort]) throw new Error("Grok broker projection worker config drifted from the manifest"); + if (createHash("sha256").update(DAIMON_GROK_SYSTEM_PROMPT).digest("hex") !== GROK_ENGINE_BROKER.worker.systemPromptSha256) throw new Error("Grok broker projection system prompt drifted from the manifest"); + const artifact = GROK_ENGINE_BROKER.grokCliArtifacts[options.architecture]; + if (artifact === undefined) throw new Error("Grok broker projection requires a pinned architecture"); + const profilePath = path.posix.join(options.workerHomePath, ".grok", "sandbox.toml"); + const projection: OrganizationGrokBrokerProjection = { + version: GROK_BROKER_PROJECTION_VERSION, agentId, workspacePath: agent.workspacePath, runtimeHomePath: agent.runtimeHomePath, + workerUid: options.workerUid, slot: options.slot, profilePath, profileSha256, denyPaths, workerConfigSha256, + systemPromptSha256: GROK_ENGINE_BROKER.worker.systemPromptSha256, grokCliVersion: GROK_ENGINE_BROKER.grokCliVersion, grokExecutableSha256: artifact.sha256, + nativeAbiVersion: GROK_ENGINE_BROKER.nativeAbiVersion, model, reasoningEffort, limits: parseEngineBrokerTurnLimits(options.limits), usageLedgerPath: options.usageLedgerPath, + attestation: { platform: "linux/landlock", enforced: true, restrictNetwork: true, profileName: GROK_WORKER_SANDBOX_PROFILE, eventsPath: grokWorkerEventsPathFor(profilePath) } + }; + // The registration this projection implies must itself be a valid v2 service.json entry. + grokBrokerServiceRegistrationFor(projection); + return projection; +} + +/** sha256 over the projection's canonical JSON; what a slot preflight receipt binds. */ +export const grokBrokerProjectionSha256 = (projection: OrganizationGrokBrokerProjection): string => + createHash("sha256").update(canonicalJson(projection)).digest("hex"); + +/** The `service.json` v2 registration a deployment provisions for this projection, validated by the broker's own parser. */ +export function grokBrokerServiceRegistrationFor(projection: OrganizationGrokBrokerProjection): Readonly> { + const registration = { + agentId: projection.agentId, slot: projection.slot, workerUid: projection.workerUid, workspace: projection.workspacePath, + profilePath: projection.profilePath, eventsPath: projection.attestation.eventsPath, profileSha256: projection.profileSha256, + usageLedgerPath: projection.usageLedgerPath, limits: projection.limits, model: { id: projection.model, reasoningEffort: projection.reasoningEffort } + }; + parseEngineBrokerServiceConfig({ version: "noopolis.daimon.engine-broker-service.v2", credentialHome: GROK_ENGINE_BROKER.credentialHomePath, turnStore: GROK_ENGINE_BROKER.turnStorePath, registrations: [registration] }); + return registration; +} + +/** + * Refuses a provisioned registration that does not describe this projection: + * any differing member — a weaker profile digest, another model, raised + * limits, a different ledger — is a mismatch, never a merge. + */ +export function verifyGrokBrokerRegistrationMatchesProjection(registration: EngineBrokerServiceRegistration, projection: OrganizationGrokBrokerProjection): void { + const expected = parseEngineBrokerServiceConfig({ version: "noopolis.daimon.engine-broker-service.v2", credentialHome: GROK_ENGINE_BROKER.credentialHomePath, turnStore: GROK_ENGINE_BROKER.turnStorePath, registrations: [grokBrokerServiceRegistrationFor(projection)] }).registrations[0]!; + if (canonicalJson(expected) !== canonicalJson(registration)) throw new Error("Grok broker registration does not match its projection"); +} From b5be62ca96f7afcf792338901611e5a0d229d206 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:17:49 +0200 Subject: [PATCH 035/124] feat: define the Grok slot preflight receipt schema with fixtures and export the broker contract --- package-lock.json | 3 +- package.json | 3 +- .../grok-slot-preflight/projection-input.json | 20 +++++ .../receipt.missing-canary.json | 32 ++++++++ .../receipt.projection-mismatch.json | 37 ++++++++++ .../receipt.readable-canary.json | 37 ++++++++++ .../receipt.unknown-member.json | 38 ++++++++++ .../grok-slot-preflight/receipt.valid.v1.json | 37 ++++++++++ src/runtime/grokSlotPreflightReceipt.test.ts | 44 +++++++++++ src/runtime/grokSlotPreflightReceipt.ts | 74 +++++++++++++++++++ src/runtime/index.ts | 10 +++ 11 files changed, 333 insertions(+), 2 deletions(-) create mode 100644 src/runtime/fixtures/grok-slot-preflight/projection-input.json create mode 100644 src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json create mode 100644 src/runtime/fixtures/grok-slot-preflight/receipt.projection-mismatch.json create mode 100644 src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json create mode 100644 src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json create mode 100644 src/runtime/fixtures/grok-slot-preflight/receipt.valid.v1.json create mode 100644 src/runtime/grokSlotPreflightReceipt.test.ts create mode 100644 src/runtime/grokSlotPreflightReceipt.ts diff --git a/package-lock.json b/package-lock.json index d046ccf..ffeb234 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,8 @@ "@earendil-works/pi-coding-agent": "^0.79.10", "@modelcontextprotocol/sdk": "^1.29.0", "@noopolis/mneme": "^0.1.1", - "ajv": "^8.17.1" + "ajv": "^8.17.1", + "zod": "^4.4.3" }, "bin": { "daimon-runtime": "dist/runtime/cli.js" diff --git a/package.json b/package.json index dc42793..1f22e90 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,8 @@ "@earendil-works/pi-coding-agent": "^0.79.10", "@modelcontextprotocol/sdk": "^1.29.0", "@noopolis/mneme": "^0.1.1", - "ajv": "^8.17.1" + "ajv": "^8.17.1", + "zod": "^4.4.3" }, "devDependencies": { "@types/node": "^24.12.4", diff --git a/src/runtime/fixtures/grok-slot-preflight/projection-input.json b/src/runtime/fixtures/grok-slot-preflight/projection-input.json new file mode 100644 index 0000000..b75a102 --- /dev/null +++ b/src/runtime/fixtures/grok-slot-preflight/projection-input.json @@ -0,0 +1,20 @@ +{ + "config": { + "version": "noopolis.daimon.organization-runtime.v2", + "host": { "bindHost": "127.0.0.1", "port": 19700, "controlTokenEnv": "DAIMON_CONTROL_TOKEN" }, + "agents": [ + { "id": "foreman", "name": "Foreman", "instructions": "Fixture agent.", "workspacePath": "/var/lib/spawnfile/instance/workspace/agents/foreman", "runtimeHomePath": "/var/lib/spawnfile/instance/homes/foreman", "schedule": { "kind": "disabled" }, "engine": { "kind": "grok", "model": "grok-4.6", "reasoningEffort": "low" } } + ] + }, + "agentId": "foreman", + "options": { + "slot": 0, + "workerUid": 2200, + "workerHomePath": "/var/lib/daimon-workers/2200", + "architecture": "arm64", + "usageLedgerPath": "/run/daimon-slots/0/usage/usage.jsonl", + "limits": { "maxRequests": 24, "maxTokens": 400000, "timeoutMs": 480000 }, + "acceptanceStorePath": "/run/paideia/control", + "denyPaths": ["/run/paideia", "/run/training/inputs"] + } +} diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json new file mode 100644 index 0000000..6671c48 --- /dev/null +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json @@ -0,0 +1,32 @@ +{ + "version": "noopolis.daimon.grok-slot-preflight.v1", + "slot": 0, + "worker_uid": 2200, + "projection_sha256": "b41a6952dfe3319f014866e7468df69ca2e77cfd28d781ac39cfdf0d3d587bd8", + "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", + "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", + "canaries": [ + { + "path": "/run/paideia/control", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/training/inputs", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-bootstrap-auth", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-subscription-realm", + "method": "sandboxed-read", + "result": "denied" + } + ], + "created_at": "2026-09-17T12:00:00.000Z" +} diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.projection-mismatch.json b/src/runtime/fixtures/grok-slot-preflight/receipt.projection-mismatch.json new file mode 100644 index 0000000..cdcb916 --- /dev/null +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.projection-mismatch.json @@ -0,0 +1,37 @@ +{ + "version": "noopolis.daimon.grok-slot-preflight.v1", + "slot": 0, + "worker_uid": 2200, + "projection_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", + "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", + "canaries": [ + { + "path": "/run/paideia", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/paideia/control", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/training/inputs", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-bootstrap-auth", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-subscription-realm", + "method": "sandboxed-read", + "result": "denied" + } + ], + "created_at": "2026-09-17T12:00:00.000Z" +} diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json new file mode 100644 index 0000000..34d05ff --- /dev/null +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json @@ -0,0 +1,37 @@ +{ + "version": "noopolis.daimon.grok-slot-preflight.v1", + "slot": 0, + "worker_uid": 2200, + "projection_sha256": "b41a6952dfe3319f014866e7468df69ca2e77cfd28d781ac39cfdf0d3d587bd8", + "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", + "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", + "canaries": [ + { + "path": "/run/paideia", + "method": "sandboxed-read", + "result": "readable" + }, + { + "path": "/run/paideia/control", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/training/inputs", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-bootstrap-auth", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-subscription-realm", + "method": "sandboxed-read", + "result": "denied" + } + ], + "created_at": "2026-09-17T12:00:00.000Z" +} diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json b/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json new file mode 100644 index 0000000..c69658a --- /dev/null +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json @@ -0,0 +1,38 @@ +{ + "version": "noopolis.daimon.grok-slot-preflight.v1", + "slot": 0, + "worker_uid": 2200, + "projection_sha256": "b41a6952dfe3319f014866e7468df69ca2e77cfd28d781ac39cfdf0d3d587bd8", + "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", + "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", + "canaries": [ + { + "path": "/run/paideia", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/paideia/control", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/training/inputs", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-bootstrap-auth", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-subscription-realm", + "method": "sandboxed-read", + "result": "denied" + } + ], + "created_at": "2026-09-17T12:00:00.000Z", + "operator": "root" +} diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v1.json b/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v1.json new file mode 100644 index 0000000..46954bc --- /dev/null +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v1.json @@ -0,0 +1,37 @@ +{ + "version": "noopolis.daimon.grok-slot-preflight.v1", + "slot": 0, + "worker_uid": 2200, + "projection_sha256": "b41a6952dfe3319f014866e7468df69ca2e77cfd28d781ac39cfdf0d3d587bd8", + "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", + "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", + "canaries": [ + { + "path": "/run/paideia", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/paideia/control", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/training/inputs", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-bootstrap-auth", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-subscription-realm", + "method": "sandboxed-read", + "result": "denied" + } + ], + "created_at": "2026-09-17T12:00:00.000Z" +} diff --git a/src/runtime/grokSlotPreflightReceipt.test.ts b/src/runtime/grokSlotPreflightReceipt.test.ts new file mode 100644 index 0000000..77d80c1 --- /dev/null +++ b/src/runtime/grokSlotPreflightReceipt.test.ts @@ -0,0 +1,44 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { resolveOrganizationGrokBrokerProjection } from "./grokBrokerProjection.js"; +import { parseGrokSlotPreflightReceipt, verifyGrokSlotPreflightReceipt } from "./grokSlotPreflightReceipt.js"; + +const fixture = async (name: string): Promise> => JSON.parse(await readFile(new URL(`./fixtures/grok-slot-preflight/${name}`, import.meta.url), "utf8")) as Record; +const projection = async () => { const input = await fixture("projection-input.json") as { config: unknown; agentId: string; options: Parameters[2] }; return resolveOrganizationGrokBrokerProjection(input.config, input.agentId, input.options); }; + +test("the committed valid receipt fixture proves the committed projection input", async () => { + const receipt = verifyGrokSlotPreflightReceipt(await fixture("receipt.valid.v1.json"), await projection()); + assert.equal(receipt.canaries.length, (await projection()).denyPaths.length); + assert.ok(receipt.canaries.every((canary) => canary.method === "sandboxed-read" && canary.result === "denied")); +}); + +test("the schema refuses a readable canary, an unknown member, duplicates and malformed digests or times", async () => { + const valid = await fixture("receipt.valid.v1.json"); + await assert.rejects(async () => parseGrokSlotPreflightReceipt(await fixture("receipt.readable-canary.json")), /invalid Grok slot preflight receipt/u); + await assert.rejects(async () => parseGrokSlotPreflightReceipt(await fixture("receipt.unknown-member.json")), /invalid Grok slot preflight receipt/u); + const canaries = valid.canaries as Record[]; + for (const bad of [ + { ...valid, canaries: [...canaries, canaries[0]] }, + { ...valid, canaries: [{ ...canaries[0], method: "stat" }] }, + { ...valid, canaries: [{ ...canaries[0], path: "/run/../etc" }] }, + { ...valid, canaries: [{ ...canaries[0], extra: true }] }, + { ...valid, canaries: [] }, + { ...valid, projection_sha256: "A".repeat(64) }, + { ...valid, worker_uid: 2_000 }, + { ...valid, created_at: "2026-09-17T12:00:00Z" }, + { ...valid, version: "noopolis.daimon.grok-slot-preflight.v2" } + ]) assert.throws(() => parseGrokSlotPreflightReceipt(bad), /invalid Grok slot preflight receipt/u); +}); + +test("a receipt for a different projection, slot, profile or deny set is refused", async () => { + const projected = await projection(); + const valid = await fixture("receipt.valid.v1.json"); + // Mutation guard: dropping the digest comparison accepts this fixture. + await assert.rejects(async () => verifyGrokSlotPreflightReceipt(await fixture("receipt.projection-mismatch.json"), projected), /projection_sha256/u); + await assert.rejects(async () => verifyGrokSlotPreflightReceipt(await fixture("receipt.missing-canary.json"), projected), /canaries/u); + assert.throws(() => verifyGrokSlotPreflightReceipt(valid, { ...projected, limits: { ...projected.limits, maxTokens: 1 } }), /projection_sha256/u); + assert.throws(() => verifyGrokSlotPreflightReceipt({ ...valid, sandbox_profile_sha256: "1".repeat(64) }, projected), /sandbox_profile_sha256/u); + assert.throws(() => verifyGrokSlotPreflightReceipt({ ...valid, slot: 1 }, projected), /slot/u); +}); diff --git a/src/runtime/grokSlotPreflightReceipt.ts b/src/runtime/grokSlotPreflightReceipt.ts new file mode 100644 index 0000000..3bed48d --- /dev/null +++ b/src/runtime/grokSlotPreflightReceipt.ts @@ -0,0 +1,74 @@ +import path from "node:path"; +import { z } from "zod"; + +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import { grokBrokerProjectionSha256, type OrganizationGrokBrokerProjection } from "./grokBrokerProjection.js"; + +export const GROK_SLOT_PREFLIGHT_VERSION = GROK_ENGINE_BROKER.slotPreflightVersion; + +const sha256 = z.string().regex(/^[a-f0-9]{64}$/u); +const canonicalAbsolute = z.string().max(4_096).refine((value) => path.posix.isAbsolute(value) && path.posix.normalize(value) === value && value !== "/" && !value.endsWith("/") && !value.includes("\0"), "canonical absolute path"); + +/** + * One denied-path canary: the root supervisor ran a real sandboxed read of + * `path` as the slot's worker uid (a stub-model turn under the attested + * profile) and the read was denied. Only denials are representable — a + * supervisor that observed a readable path writes no receipt at all. + */ +export const grokSlotPreflightCanarySchema = z.strictObject({ + path: canonicalAbsolute, + method: z.literal("sandboxed-read"), + result: z.literal("denied") +}); + +/** + * `noopolis.daimon.grok-slot-preflight.v1`: what the root slot supervisor (P5) + * writes after provisioning or recycling one broker slot, and what an + * evaluator (Paideia, P4) must hold before it runs a Grok subject turn in that + * slot. It binds the slot to one exact projection by digest, so any change to + * the model, limits, deny list, profile, worker config, or pinned executable + * invalidates it. + */ +export const grokSlotPreflightReceiptSchema = z.strictObject({ + version: z.literal(GROK_SLOT_PREFLIGHT_VERSION), + slot: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER), + worker_uid: z.number().int().min(GROK_ENGINE_BROKER.identities.firstWorkerUid).max(4_294_967_294), + projection_sha256: sha256, + /** The bubblewrap/Landlock `daimon-strict` profile bytes' digest (the projection's `profileSha256`). */ + sandbox_profile_sha256: sha256, + /** The container seccomp profile the worker ran under. */ + seccomp_profile_sha256: sha256, + grok_executable_sha256: sha256, + canaries: z.array(grokSlotPreflightCanarySchema).min(1).max(256), + created_at: z.string().regex(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/u).refine((value) => !Number.isNaN(Date.parse(value)) && new Date(value).toISOString() === value, "exact RFC3339 timestamp") +}).superRefine((receipt, context) => { + const paths = receipt.canaries.map((canary) => canary.path); + if (new Set(paths).size !== paths.length) context.addIssue({ code: "custom", path: ["canaries"], message: "duplicate canary path" }); +}); + +export type GrokSlotPreflightReceipt = z.infer; + +/** Strict parse: unknown members, off-contract values, or duplicate canaries throw. */ +export function parseGrokSlotPreflightReceipt(value: unknown): GrokSlotPreflightReceipt { + const result = grokSlotPreflightReceiptSchema.safeParse(value); + if (!result.success) throw new TypeError(`invalid Grok slot preflight receipt: ${result.error.issues.map((issue) => `${issue.path.join(".") || "receipt"}: ${issue.message}`).join("; ")}`); + return result.data; +} + +/** + * Parse a receipt and require that it proves *this* projection's slot: same + * digest, slot, worker uid, profile and executable, and a denied canary for + * exactly every projected deny path (no more, no fewer). + */ +export function verifyGrokSlotPreflightReceipt(value: unknown, projection: OrganizationGrokBrokerProjection): GrokSlotPreflightReceipt { + const receipt = parseGrokSlotPreflightReceipt(value); + const mismatch = (member: string): never => { throw new Error(`Grok slot preflight receipt does not match the projection: ${member}`); }; + if (receipt.projection_sha256 !== grokBrokerProjectionSha256(projection)) mismatch("projection_sha256"); + if (receipt.slot !== projection.slot) mismatch("slot"); + if (receipt.worker_uid !== projection.workerUid) mismatch("worker_uid"); + if (receipt.sandbox_profile_sha256 !== projection.profileSha256) mismatch("sandbox_profile_sha256"); + if (receipt.grok_executable_sha256 !== projection.grokExecutableSha256) mismatch("grok_executable_sha256"); + const denied = receipt.canaries.map((canary) => canary.path).sort(); + if (denied.length !== projection.denyPaths.length || denied.some((entry, index) => entry !== projection.denyPaths[index])) mismatch("canaries"); + return receipt; +} diff --git a/src/runtime/index.ts b/src/runtime/index.ts index b72ee90..83d00a4 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -5,6 +5,16 @@ export { createOrganizationRuntimeHost } from "./organizationRuntimeHost.js"; export { createOrganizationRuntimeControlHost } from "./organizationRuntimeControl.js"; export { CODEX_SANDBOX_PROJECTION_VERSION, resolveOrganizationCodexSandboxProjection, type OrganizationCodexSandboxProjection } from "./codexSandboxProjection.js"; +export { GROK_BROKER_PROJECTION_VERSION, grokBrokerProjectionSha256, grokBrokerServiceRegistrationFor, resolveOrganizationGrokBrokerProjection, + verifyGrokBrokerRegistrationMatchesProjection, type OrganizationGrokBrokerProjection, type OrganizationGrokBrokerProjectionOptions } from "./grokBrokerProjection.js"; +export { GROK_SLOT_PREFLIGHT_VERSION, grokSlotPreflightCanarySchema, grokSlotPreflightReceiptSchema, parseGrokSlotPreflightReceipt, + verifyGrokSlotPreflightReceipt, type GrokSlotPreflightReceipt } from "./grokSlotPreflightReceipt.js"; +export { grokBrokerWorkerConfigSha256, renderGrokBrokerWorkerConfig } from "./grokBrokerWorkerConfig.js"; +export { grokWorkerSandboxProfileSha256, renderGrokWorkerSandboxProfile } from "./grokWorkerSandboxProfile.js"; +export { engineBrokerRequestLedgerPathFor, parseEngineBrokerServiceConfig, type EngineBrokerServiceConfig, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +export { DEFAULT_GROK_BROKER_TURN_LIMITS, ENGINE_BROKER_LIMIT_REASONS, type EngineBrokerLimitReason, type EngineBrokerTurnAccounting, + type EngineBrokerTurnLimits, type EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; +export { dedupeTurnUsageRows } from "./turnUsageLedger.js"; export { WakeTransitionLockBlockedError } from "./wakeAcceptanceStore.js"; export { OFFLINE_RECONCILIATION_BLOCKED_CODE, From 55852f3d24018b6d1cade6ddd1f5b3d1f8470fd6 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:18:19 +0200 Subject: [PATCH 036/124] docs: describe Grok broker limits, sealed accounting, projection and slot receipts --- docs/engines.md | 15 ++++++++++++ src/runtime/AGENTS.md | 56 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/docs/engines.md b/docs/engines.md index ba2b288..b2517f6 100644 --- a/docs/engines.md +++ b/docs/engines.md @@ -88,6 +88,21 @@ skills, workflows, plan mode, subagents, memory or web search, and a declared model and reasoning effort from a closed list (default `grok-4.6` at `low`). The broker proxy refuses any request outside that shape before it spends. +Each broker registration (`service.json` v2) declares its model and effort, +its usage ledger, and turn limits `{maxRequests, maxTokens, timeoutMs}`. A wake +may only lower them (`DAIMON_ENGINE_WAKE_TIMEOUT_MS`, +`DAIMON_ENGINE_WAKE_TOKEN_CEILING`; the `DAIMON_CODEX_WAKE_*` names are +aliases). The proxy refuses request `maxRequests + 1` and any request after the +deadline with HTTP 429 before upstream, and stops admitting requests once the +upstream-reported running total (cached input included) reaches `maxTokens`, so +a turn overshoots its token ceiling by at most one request. A tripped limit +kills the worker. The broker seals every terminal turn with its usage, request +count, declared model and limit reason, and writes one usage row (keyed by +`turn`) plus per-request rows for completed and failed turns alike; a replayed +turn is never metered twice. `resolveOrganizationGrokBrokerProjection` exposes +a slot's full declared shape, and `noopolis.daimon.grok-slot-preflight.v1` +receipts bind a slot's denied-path canaries to that projection's digest. + AGY uses OS-native secure storage through one private D-Bus and Secret Service realm. Enroll it once with: diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 4ddaa1b..ca8eb55 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -28,6 +28,51 @@ tool), and any body whose `model`/`reasoning_effort` differ from the declared `grokBrokerModelPolicy.ts` policy (closed lists; default `grok-4.6`/`low`). The model override header follows that declaration. +The proxy is the per-turn limit gate too. Every broker turn registers a +`grokBrokerTurnMeter.ts` meter with its registration's model policy, and the +proxy forwards nothing for a turn without one. After a body is proven a lean +worker request and before any upstream call, the meter refuses request +`maxRequests + 1`, any request past `timeoutMs`, and any request once the +upstream-reported running total (prompt tokens *including* cached, plus +completion) has reached `maxTokens` — HTTP 429, and the tripped limit aborts +the worker through the ordinary cancel/kill path. The token ceiling is checked +between requests, so a turn overshoots it by at most the last admitted +request; if an upstream body carries no `usage`, only `maxRequests` and +`timeoutMs` bound that turn mid-flight. A broker timer also trips `timeout` +for a worker that is mid-request. Limits come from `service.json` v2 +(`engineBrokerServiceConfig.ts`; v1 gets `GROK_ENGINE_BROKER.turnLimits.v1Defaults`) +and a wake may only lower them: a raise is refused as `invalid_request`, never +clamped. + +The broker stays the single sealed usage writer. `grokEngineBrokerTurn.ts` +seals every terminal turn — completed, failed, limit, cancelled — through +`finishBrokerTurnWithUsage` (`grokEngineBrokerMetering.ts`): the turn registry +record v2 stores the control-protocol v2 terminal response *with* its +numeric-only accounting (`usage`, `outcome`, declared `model`, `requests`, +closed `limitReason`), and only then are ledger rows appended. A replay +returns the sealed accounting and never meters again; v1 records still replay +(upgraded with `usage: null`). Completed usage is the terminal `result.usage`; +a failed turn's partial usage is its per-request stream frames +(`../pi/grokStreamUsage.ts`) when output arrived, else the upstream usage the +proxy saw. Usage rows carry `turn` (the idempotency key readers dedupe on — +`wakeFuse.ts` does), `limit_reason` and `model`; per-request rows go to +`requests.jsonl` beside the registration's `usageLedgerPath` with proxy-measured +`started_at`/`ended_at`. A provider-reported model key must map to the declared +model (`grok-4.6-build` → `grok-4.6`), otherwise the turn fails as rejected and +is still metered. Control protocol v2 is refused-v1 on the wire because both +ends ship in this package. + +`grokBrokerProjection.ts` is the public, I/O-free projection of one brokered +Grok agent's slot (`noopolis.daimon.grok-broker-projection.v1`): Daimon's own +deny collectors plus the caller's evaluator paths, profile/config/prompt +digests, pinned executable, model, limits and ledger. A Grok agent must declare +`model` and `reasoningEffort` for it; nothing is defaulted, and a supplied +profile digest that differs is refused. `grokSlotPreflightReceipt.ts` is the +zod schema a root slot supervisor's receipt must satisfy +(`noopolis.daimon.grok-slot-preflight.v1`, fixtures under +`fixtures/grok-slot-preflight/`); `verifyGrokSlotPreflightReceipt` binds it to +the projection digest and requires a denied canary for exactly every deny path. + `grokBrokerWorkerConfig.ts` is the only source of worker `config.toml` bytes; the manifest pins the sha256 of every model/effort combination and the broker refuses a turn whose worker config does not hash to the declared one. Three @@ -91,8 +136,10 @@ the only place its per-wake tool-call bound is decided. `maxToolTurns` only mediates daimon-MCP tool calls; Codex's own shell (`exec_command`) is never routed through it, so Codex gets its own bounds instead — `DEFAULT_CODEX_WAKE_TIMEOUT_MS` (wall clock) and -`DEFAULT_CODEX_WAKE_TOKEN_CEILING`, both in `../pi/cliSession.ts`, overridable -via `DAIMON_CODEX_WAKE_TIMEOUT_MS`/`DAIMON_CODEX_WAKE_TOKEN_CEILING`. The token +`DEFAULT_CODEX_WAKE_TOKEN_CEILING`, both in `../pi/engineWakeLimits.ts`, overridable +via the engine-neutral `DAIMON_ENGINE_WAKE_TIMEOUT_MS`/`DAIMON_ENGINE_WAKE_TOKEN_CEILING` +(the `DAIMON_CODEX_*` names are aliases; conflicting values are refused), which +the dispatcher also passes to the Grok broker as lowering limits. The token ceiling can only be checked when Codex reports it: its `--json` stream carries usage exactly once, on the turn's own `turn.completed`, so crossing it kills the child immediately and fails the wake instead of letting an over-budget @@ -133,7 +180,10 @@ request count) look like the first without proving it. `../pi/cliChildOutput.ts` carries the thread id off Codex's own `thread.started` frame, and `../pi/codexRolloutUsage.ts` reads that thread's rollout under `$CODEX_HOME/sessions/**` for the per-request `token_usage_record` frames the -`--json` stream never emits. Rows go to `requests.jsonl` beside `usage.jsonl` +`--json` stream never emits. Each Codex row carries its own `started_at`/`ended_at` +from the rollout frame timestamps (end = the usage frame; start = the first +non-usage frame after the previous request's usage frame, else that request's +end), absent rather than substituted when a frame has no valid timestamp. Rows go to `requests.jsonl` beside `usage.jsonl` (`DAIMON_TURN_REQUESTS_LEDGER_PATH` relocates it) under the same invariants: a wake whose rollout is absent, unreadable, or undecodable writes *nothing*, because a fabricated zero is byte-identical to a measured one; and every failure From 50a932557797e780dd24d4bf945b00de168f1903 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:19:39 +0200 Subject: [PATCH 037/124] test: prove Grok per-request rows carry measured proxy intervals, not the append time --- src/runtime/grokEngineBrokerUsage.test.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index f66631d..20ab62d 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -49,7 +49,7 @@ const untilAborted = (signal: AbortSignal): Promise => new Promise((_reso const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId: string, worker: Worker, overrides?: Parameters[6], limits?: EngineBrokerServiceRegistration["limits"], turnStore?: string) => ReturnType; usageRows: () => Promise[]>; requestRows: () => Promise[]>; upstreamCalls: () => number }>) => Promise, usageLedgerPath?: string): Promise => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-usage-")); let calls = 0; - const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(`data: ${JSON.stringify({ choices: [], usage: upstreamUsage })}\n\ndata: [DONE]\n\n`) }; }, undefined, 0); + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; await new Promise((resolve) => setTimeout(resolve, 15)); return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(`data: ${JSON.stringify({ choices: [], usage: upstreamUsage })}\n\ndata: [DONE]\n\n`) }; }, undefined, 0); const ledger = usageLedgerPath ?? path.join(root, "usage.jsonl"); const rows = async (file: string) => (await readFile(file, "utf8").catch(() => "")).split("\n").filter((line) => line.length > 0).map((line) => JSON.parse(line) as Record); try { @@ -92,8 +92,10 @@ test("a completed turn seals its accounting, writes one usage row and per-reques [TURN_REQUEST_LEDGER_VERSION, "grok", 1, 2, 2_797, 109, 2_688, 2_810, turnIdFor("foreman", "wake-1")] ]); // Mutation guard: stamping every request with the wake end collapses these. - for (const row of requests) assert.match(String(row.started_at), /^\d{4}-\d{2}-\d{2}T/u); - assert.ok(String(requests[0]!.ended_at) <= String(requests[1]!.started_at), "request 1 ends before request 2 starts"); + // The upstream stub takes 15 ms per request, so each request has a measurable interval. + const [a, b] = requests.map((row) => [Date.parse(String(row.started_at)), Date.parse(String(row.ended_at))] as const); + assert.ok(a![0] < a![1] && a![1] <= b![0] && b![0] < b![1], JSON.stringify(requests.map((row) => [row.started_at, row.ended_at]))); + assert.ok(b![1] <= Date.parse(String(requests[1]!.at)), "every request ended before the rows were appended"); }); }); From 39deffaf3c8ea10c0a4cd11cadea95a4899cb3f1 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:23:41 +0200 Subject: [PATCH 038/124] fix: count a killed turn's in-flight request in every Grok per-request row --- src/runtime/grokEngineBrokerMetering.ts | 2 +- src/runtime/grokEngineBrokerUsage.test.ts | 19 +++++++++++-------- src/runtime/turnRequestLedger.ts | 9 +++++++-- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/runtime/grokEngineBrokerMetering.ts b/src/runtime/grokEngineBrokerMetering.ts index f254554..bd4e482 100644 --- a/src/runtime/grokEngineBrokerMetering.ts +++ b/src/runtime/grokEngineBrokerMetering.ts @@ -39,5 +39,5 @@ export async function finishBrokerTurnWithUsage(turns: EngineBrokerTurnRegistry, outcome: terminal.kind === "completed" ? { status: "completed" } : { status: "failed", reason: detail.reason ?? "unknown" }, turn: terminal.turnId, limitReason: terminal.limitReason, model: terminal.model }); - await recordGrokTurnRequests(metering.requestLedgerPath, { agent: metering.agentId, wake: metering.wakeId, turn: terminal.turnId, model: terminal.model, requests: detail.requests, ...(detail.session === undefined ? {} : { session: detail.session }) }); + await recordGrokTurnRequests(metering.requestLedgerPath, { agent: metering.agentId, wake: metering.wakeId, turn: terminal.turnId, model: terminal.model, requests: detail.requests, requestCount: terminal.requests, ...(detail.session === undefined ? {} : { session: detail.session }) }); } diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index 20ab62d..408850b 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -46,10 +46,10 @@ const untilAborted = (signal: AbortSignal): Promise => new Promise((_reso * talks to the proxy exactly as the native worker does (capability bearer, * pinned client version, lean body). Only the launcher and attestation are fakes. */ -const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId: string, worker: Worker, overrides?: Parameters[6], limits?: EngineBrokerServiceRegistration["limits"], turnStore?: string) => ReturnType; usageRows: () => Promise[]>; requestRows: () => Promise[]>; upstreamCalls: () => number }>) => Promise, usageLedgerPath?: string): Promise => { +const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId: string, worker: Worker, overrides?: Parameters[6], limits?: EngineBrokerServiceRegistration["limits"], turnStore?: string) => ReturnType; usageRows: () => Promise[]>; requestRows: () => Promise[]>; upstreamCalls: () => number }>) => Promise, usageLedgerPath?: string, upstreamDelayMs: (call: number) => number = () => 15): Promise => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-usage-")); let calls = 0; - const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; await new Promise((resolve) => setTimeout(resolve, 15)); return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(`data: ${JSON.stringify({ choices: [], usage: upstreamUsage })}\n\ndata: [DONE]\n\n`) }; }, undefined, 0); + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; await new Promise((resolve) => setTimeout(resolve, upstreamDelayMs(calls))); return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(`data: ${JSON.stringify({ choices: [], usage: upstreamUsage })}\n\ndata: [DONE]\n\n`) }; }, undefined, 0); const ledger = usageLedgerPath ?? path.join(root, "usage.jsonl"); const rows = async (file: string) => (await readFile(file, "utf8").catch(() => "")).split("\n").filter((line) => line.length > 0).map((line) => JSON.parse(line) as Record); try { @@ -128,13 +128,16 @@ test("the token ceiling stops a turn one request past the ceiling at most", asyn }); test("the wall-clock limit aborts a worker that is mid-request", async () => { - await withBroker(async ({ turn, usageRows }) => { + await withBroker(async ({ turn, usageRows, requestRows }) => { const started = Date.now(); - const worker: Worker = async (send, signal) => { assert.equal(await send(), 200); return untilAborted(signal); }; - await assert.rejects(turn("wake-4", worker, { timeoutMs: 1_000 }), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.code === "limit_exceeded" && error.accounting?.limitReason === "timeout"); - assert.ok(Date.now() - started < 5_000); - assert.deepEqual((await usageRows()).map((row) => [row.reason, row.limit_reason, row.total]), [["wake_timeout", "timeout", 2_775]]); - }); + // Request 2 is still upstream (1.5 s) when the 1 s wall clock fires. + const worker: Worker = async (send, signal) => { assert.equal(await send(), 200); void send().catch(() => undefined); return untilAborted(signal); }; + await assert.rejects(turn("wake-4", worker, { timeoutMs: 1_000 }), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.code === "limit_exceeded" && error.accounting?.limitReason === "timeout" && error.accounting.requests === 2); + assert.ok(Date.now() - started < 1_400, "the turn ends at the deadline, not when the in-flight request returns"); + assert.deepEqual((await usageRows()).map((row) => [row.reason, row.limit_reason, row.total, row.calls]), [["wake_timeout", "timeout", 2_775, 2]]); + // One measured row, but both admitted requests count: the killed one was sent upstream. + assert.deepEqual((await requestRows()).map((row) => [row.request, row.requests]), [[0, 2]]); + }, undefined, (call) => call === 2 ? 1_500 : 15); }); test("a wake may only lower a declared limit: raising one is refused before any turn record or worker", async () => { diff --git a/src/runtime/turnRequestLedger.ts b/src/runtime/turnRequestLedger.ts index a13bfdf..ef8dfca 100644 --- a/src/runtime/turnRequestLedger.ts +++ b/src/runtime/turnRequestLedger.ts @@ -106,7 +106,12 @@ const requestClockFields = (request: Readonly<{ startedAt?: string; endedAt?: st /** One Grok broker model request: usage from the worker stream, timing from the proxy. */ export type GrokTurnRequest = Readonly<{ index: number; input: number; cacheRead: number; cacheWrite: number; output: number; total: number; startedAt?: string; endedAt?: string }>; -export type GrokTurnRequestEntry = Readonly<{ agent: string; wake: string; turn: string; session?: string; model: GrokBrokerModel; requests: readonly GrokTurnRequest[]; at?: string }>; +/** + * `requestCount` is the turn's admitted request count when it exceeds the rows: + * a killed turn's in-flight request was sent upstream but never reported usage, + * so it has no row yet still counts in every row's `requests`. + */ +export type GrokTurnRequestEntry = Readonly<{ agent: string; wake: string; turn: string; session?: string; model: GrokBrokerModel; requests: readonly GrokTurnRequest[]; requestCount?: number; at?: string }>; /** * Grok rows share the Codex row's field meaning: `input` is the whole prompt @@ -128,7 +133,7 @@ export const renderGrokTurnRequestLines = (entry: GrokTurnRequestEntry): string ...(entry.session === undefined ? {} : { thread: bounded(entry.session) }), model: entry.model, request: request.index, - requests: entry.requests.length, + requests: Math.max(entry.requests.length, entry.requestCount ?? 0), input: request.input + request.cacheRead, cached_input: request.cacheRead, fresh_input: request.input, From cb1971f3318d67931bbfe5b804222f7b20290796 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:25:43 +0200 Subject: [PATCH 039/124] test: keep Grok engine declaration tests in their own file under the line limit --- src/runtime/engineDispatcher.test.ts | 13 +---- src/runtime/organizationRuntime.test.ts | 35 ----------- .../organizationRuntimeGrokEngine.test.ts | 58 +++++++++++++++++++ 3 files changed, 60 insertions(+), 46 deletions(-) create mode 100644 src/runtime/organizationRuntimeGrokEngine.test.ts diff --git a/src/runtime/engineDispatcher.test.ts b/src/runtime/engineDispatcher.test.ts index 488a658..0dae9a8 100644 --- a/src/runtime/engineDispatcher.test.ts +++ b/src/runtime/engineDispatcher.test.ts @@ -244,13 +244,9 @@ test("production Grok dispatcher routes every wake through the broker without ag process.env.NOOPOLIS_RUN_ID = "dispatcher-grok-realm-test"; const broker: EngineBrokerTurnClient = { async turn(agentId,wakeId,prompt,endpoint,signal,options) { turns += 1;assert.equal(agentId,config.id);assert.match(wakeId,/^(first|second)$/u);assert.match(prompt,/work/u);assert.match(endpoint,/^http:\/\/127\.0\.0\.1:\d+\/mcp$/u);assert.equal(signal?.aborted,false); - // The engine-neutral wake bound reaches the broker as a lowering limit. - assert.deepEqual(options,{limits:{maxTokens:123_456}});return "brokered"; } + assert.deepEqual(options,{limits:{maxTokens:123_456}});return "brokered"; } // the engine-neutral wake bound reaches the broker as a lowering limit }; - const priorCeiling = process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING; process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING = "123456"; - let handle: Awaited>; - try { handle = await startOrganizationRuntimeEngine(config, "DAIMON_UNUSED_CONTROL", undefined, undefined, broker); } - finally { if (priorCeiling === undefined) delete process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING; else process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING = priorCeiling; } + const priorCeiling = process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING; process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING = "123456"; const handle = await startOrganizationRuntimeEngine(config, "DAIMON_UNUSED_CONTROL", undefined, undefined, broker).finally(() => { if (priorCeiling === undefined) delete process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING; else process.env.DAIMON_ENGINE_WAKE_TOKEN_CEILING = priorCeiling; }); assert.equal((await handle.wake({ id: "first", kind: "manual", text: "work" })).text, "brokered"); assert.equal((await handle.wake({ id: "second", kind: "manual", text: "work" })).text, "brokered"); assert.equal(turns, 2); @@ -401,8 +397,3 @@ async function seedAuth(root: string, kind: "codex" | "grok" | "agy"): Promise { - const config = { ...rootConfig("/tmp/daimon-unused-direct-grok", "grok"), engine: { kind: "grok", model: "grok-4.6", reasoningEffort: "low" } } as OrganizationRuntimeAgentConfig; - await assert.rejects(startOrganizationRuntimeEngine(config, "DAIMON_UNUSED_CONTROL"), /requires the engine broker/u); -}); diff --git a/src/runtime/organizationRuntime.test.ts b/src/runtime/organizationRuntime.test.ts index 3636225..60f98f8 100644 --- a/src/runtime/organizationRuntime.test.ts +++ b/src/runtime/organizationRuntime.test.ts @@ -271,20 +271,6 @@ test("the engine JSON Schema and the parser agree on model/reasoningEffort", () } }); -test("the engine JSON Schema and the parser agree on grok and agy model declarations", async () => { - const { Ajv2020 } = await import("ajv/dist/2020.js") as unknown as { Ajv2020: new (options: Record) => { compile(schema: unknown): (value: unknown) => boolean } }; - const validate = new Ajv2020({ strict: false }).compile(ORGANIZATION_RUNTIME_CONFIG_SCHEMA); - for (const engine of [ - { kind: "grok" }, { kind: "grok", model: "grok-4.6", reasoningEffort: "low" }, { kind: "grok", model: "grok-4.6" }, - { kind: "grok", model: "grok-4.6-build", reasoningEffort: "low" }, { kind: "grok", model: "grok-4.5", reasoningEffort: "xhigh" }, - { kind: "agy" }, { kind: "agy", model: "x" }, { kind: "codex", model: "gpt-5-codex", reasoningEffort: "xhigh" } - ]) { - const config = valid(); - config.agents[0]!.engine = engine as never; - assert.equal(validate(config), validateOrganizationRuntimeConfig(config), JSON.stringify(engine)); - } -}); - test("accepts only the narrow optional Codex workspace policy", () => { const config = valid(); config.agents[0]!.engine = { kind: "codex", codexSandbox: { mode: "workspace-write", networkAccess: false, webSearch: "disabled" } } as never; @@ -315,27 +301,6 @@ test("rejects model and reasoningEffort on agy, and codexSandbox on every non-co } }); -test("grok declares model and reasoning effort together from the closed broker lists, never half-inherited", () => { - const declared = valid(); - declared.agents[0]!.engine = { kind: "grok", model: "grok-4.6", reasoningEffort: "low" } as never; - assert.deepEqual(parseOrganizationRuntimeConfig(declared).agents[0]!.engine, { kind: "grok", model: "grok-4.6", reasoningEffort: "low" }); - const bare = valid(); - bare.agents[0]!.engine = { kind: "grok" } as never; - assert.deepEqual(parseOrganizationRuntimeConfig(bare).agents[0]!.engine, { kind: "grok" }); - for (const engine of [ - { kind: "grok", model: "grok-4.6" }, - { kind: "grok", reasoningEffort: "low" }, - { kind: "grok", model: "grok-4.6-build", reasoningEffort: "low" }, - { kind: "grok", model: "gpt-5-codex", reasoningEffort: "low" }, - { kind: "grok", model: "grok-4.6", reasoningEffort: "xhigh" }, - { kind: "grok", model: "grok-4.6", reasoningEffort: "low", provider: "xai" } - ]) { - const invalid = valid(); - invalid.agents[0]!.engine = engine as never; - assert.throws(() => parseOrganizationRuntimeConfig(invalid), TypeError, JSON.stringify(engine)); - } -}); - const withMemory = (agent: Record, memory: unknown): Record => ({ ...agent, memory }); test("parses a declared memory bank and round-trips its fields", () => { diff --git a/src/runtime/organizationRuntimeGrokEngine.test.ts b/src/runtime/organizationRuntimeGrokEngine.test.ts new file mode 100644 index 0000000..93ea7ab --- /dev/null +++ b/src/runtime/organizationRuntimeGrokEngine.test.ts @@ -0,0 +1,58 @@ +import { strict as assert } from "node:assert"; +import test from "node:test"; + +import { startOrganizationRuntimeEngine } from "./engineDispatcher.js"; +import { + ORGANIZATION_RUNTIME_CONFIG_SCHEMA, + ORGANIZATION_RUNTIME_VERSION, + validateOrganizationRuntimeConfig, + parseOrganizationRuntimeConfig, + type OrganizationRuntimeAgentConfig, + type OrganizationRuntimeEngineIntent +} from "./organizationRuntime.js"; + +const valid = () => ({ + version: ORGANIZATION_RUNTIME_VERSION, + host: { bindHost: "127.0.0.1", port: 4318, controlTokenEnv: "DAIMON_CONTROL_TOKEN" }, + agents: [{ id: "editor", name: "Editor", instructions: "Write a concise report.", workspacePath: "/runtime/workspaces/editor", runtimeHomePath: "/runtime/homes/editor", engine: { kind: "codex" } as OrganizationRuntimeEngineIntent }] +}); + +test("grok declares model and reasoning effort together from the closed broker lists, never half-inherited", () => { + const declared = valid(); + declared.agents[0]!.engine = { kind: "grok", model: "grok-4.6", reasoningEffort: "low" } as never; + assert.deepEqual(parseOrganizationRuntimeConfig(declared).agents[0]!.engine, { kind: "grok", model: "grok-4.6", reasoningEffort: "low" }); + const bare = valid(); + bare.agents[0]!.engine = { kind: "grok" } as never; + assert.deepEqual(parseOrganizationRuntimeConfig(bare).agents[0]!.engine, { kind: "grok" }); + for (const engine of [ + { kind: "grok", model: "grok-4.6" }, + { kind: "grok", reasoningEffort: "low" }, + { kind: "grok", model: "grok-4.6-build", reasoningEffort: "low" }, + { kind: "grok", model: "gpt-5-codex", reasoningEffort: "low" }, + { kind: "grok", model: "grok-4.6", reasoningEffort: "xhigh" }, + { kind: "grok", model: "grok-4.6", reasoningEffort: "low", provider: "xai" } + ]) { + const invalid = valid(); + invalid.agents[0]!.engine = engine as never; + assert.throws(() => parseOrganizationRuntimeConfig(invalid), TypeError, JSON.stringify(engine)); + } +}); + +test("the engine JSON Schema and the parser agree on grok and agy model declarations", async () => { + const { Ajv2020 } = await import("ajv/dist/2020.js") as unknown as { Ajv2020: new (options: Record) => { compile(schema: unknown): (value: unknown) => boolean } }; + const validate = new Ajv2020({ strict: false }).compile(ORGANIZATION_RUNTIME_CONFIG_SCHEMA); + for (const engine of [ + { kind: "grok" }, { kind: "grok", model: "grok-4.6", reasoningEffort: "low" }, { kind: "grok", model: "grok-4.6" }, + { kind: "grok", model: "grok-4.6-build", reasoningEffort: "low" }, { kind: "grok", model: "grok-4.5", reasoningEffort: "xhigh" }, + { kind: "agy" }, { kind: "agy", model: "x" }, { kind: "codex", model: "gpt-5-codex", reasoningEffort: "xhigh" } + ]) { + const config = valid(); + config.agents[0]!.engine = engine as never; + assert.equal(validate(config), validateOrganizationRuntimeConfig(config), JSON.stringify(engine)); + } +}); + +test("a declared Grok model is refused on the direct path that cannot enforce it", async () => { + const config = { ...valid().agents[0]!, engine: { kind: "grok", model: "grok-4.6", reasoningEffort: "low" } } as OrganizationRuntimeAgentConfig; + await assert.rejects(startOrganizationRuntimeEngine(config, "DAIMON_UNUSED_CONTROL"), /requires the engine broker/u); +}); From a77a6d4be0b26f1ed1f8075af982656281b0bbed Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:36:52 +0200 Subject: [PATCH 040/124] fix: allow one in-flight upstream request per Grok turn and abort it when a limit trips --- src/runtime/grokBrokerProxy.ts | 9 ++-- src/runtime/grokBrokerTurnMeter.test.ts | 54 +++++++++++++++++++++++ src/runtime/grokBrokerTurnMeter.ts | 25 +++++++++-- src/runtime/grokEngineBrokerTurn.ts | 2 +- src/runtime/grokEngineBrokerUsage.test.ts | 12 ++--- 5 files changed, 89 insertions(+), 13 deletions(-) diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index 4f53103..57e9091 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -10,7 +10,7 @@ import { parseGrokUpstreamUsage, type GrokBrokerTurnMeter } from "./grokBrokerTu export type GrokBrokerProxyTurn = Readonly<{ policy: GrokBrokerModelPolicy; meter: GrokBrokerTurnMeter }>; export type GrokBrokerCredentialAuthority = Readonly<{ accessToken(forceRefresh: boolean): Promise; refreshAfterRejection?(rejectedTokenDigest:string):Promise; markRejected(rejectedTokenDigest?:string): Promise }>; -export type GrokBrokerUpstream = (request: ReturnType) => Promise>; body: Uint8Array }>>; +export type GrokBrokerUpstream = (request: ReturnType, signal?: AbortSignal) => Promise>; body: Uint8Array }>>; /** * `policy` is the fallback declared model/effort (closed list); a registered @@ -36,12 +36,13 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori // (a refused session-title body never counts) and before any upstream call. const admission=turn.meter.admit(); if("refused" in admission){response.writeHead(429,{"content-type":"application/json","cache-control":"no-store"});response.end(JSON.stringify({error:"turn limit reached",limit:admission.refused}));return;} + if("busy" in admission){response.writeHead(429,{"content-type":"application/json","cache-control":"no-store"});response.end('{"error":"turn request in flight"}');return;} settle=(usage)=>{turn.meter.settle(admission.index,usage);settle=undefined;}; - let result = await upstream(prepared); - if (result.status === 401) { token = authority.refreshAfterRejection?await authority.refreshAfterRejection(rejectedDigest):await authority.accessToken(true);const refreshedDigest=createHash("sha256").update(token).digest("hex"); prepared = { ...prepared, headers: { ...prepared.headers, authorization: `Bearer ${token}` } }; token = ""; result = await upstream(prepared);if(result.status===401)await authority.markRejected(refreshedDigest); } + let result = await upstream(prepared,admission.signal); + if (result.status === 401) { token = authority.refreshAfterRejection?await authority.refreshAfterRejection(rejectedDigest):await authority.accessToken(true);const refreshedDigest=createHash("sha256").update(token).digest("hex"); prepared = { ...prepared, headers: { ...prepared.headers, authorization: `Bearer ${token}` } }; token = ""; result = await upstream(prepared,admission.signal);if(result.status===401)await authority.markRejected(refreshedDigest); } settle?.(parseGrokUpstreamUsage(result.body,result.headers["content-type"])); response.writeHead(result.status, { "content-type": result.headers["content-type"] ?? "application/json", "cache-control": "no-store" }); response.end(result.body); } catch { settle?.(undefined); response.writeHead(503, { "content-type": "application/json", "cache-control": "no-store" }); response.end('{"error":"broker unavailable"}'); } } async function readBody(request: IncomingMessage): Promise { const chunks: Buffer[] = []; let bytes = 0; for await (const chunk of request) { const value = Buffer.from(chunk); bytes += value.length; if (bytes > 2 * 1024 * 1024) throw new Error("too large"); chunks.push(value); } return Buffer.concat(chunks); } -const defaultUpstream: GrokBrokerUpstream = async (request) => { const result = await fetch(request.url, { method: "POST", headers: request.headers, body: Buffer.from(request.body) }); return { status: result.status, headers: { "content-type": result.headers.get("content-type") ?? "application/json" }, body: new Uint8Array(await result.arrayBuffer()) }; }; +const defaultUpstream: GrokBrokerUpstream = async (request, signal) => { const result = await fetch(request.url, { method: "POST", headers: request.headers, body: Buffer.from(request.body), ...(signal === undefined ? {} : { signal }) }); return { status: result.status, headers: { "content-type": result.headers.get("content-type") ?? "application/json" }, body: new Uint8Array(await result.arrayBuffer()) }; }; diff --git a/src/runtime/grokBrokerTurnMeter.test.ts b/src/runtime/grokBrokerTurnMeter.test.ts index 04c0a6a..cd55082 100644 --- a/src/runtime/grokBrokerTurnMeter.test.ts +++ b/src/runtime/grokBrokerTurnMeter.test.ts @@ -101,3 +101,57 @@ test("upstream usage parsing takes the last usage block and never zero-fills", ( assert.equal(parseGrokUpstreamUsage(sse({ prompt_tokens: 4, completion_tokens: 1, prompt_tokens_details: { cached_tokens: 9 } }), "text/event-stream"), undefined); assert.equal(parseGrokUpstreamUsage(Buffer.from("not json"), "application/json"), undefined); }); + +test("at most one upstream request is in flight per turn: an overlapping request is refused, uncounted", async () => { + // Mutation guard: without the in-flight gate both overlapping requests pass on + // the same pre-settle token total and the one-request overshoot bound is gone. + const meter = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 100, timeoutMs: 60_000 }); + const first = meter.admit(); + assert.ok("index" in first); + assert.deepEqual(meter.admit(), { busy: true }); + assert.equal(meter.snapshot().requests, 1); + meter.settle(first.index, { input: 60, cacheRead: 0, cacheWrite: 0, output: 60, total: 120 }); + assert.deepEqual(meter.admit(), { refused: "tokens" }, "once settled, the next request sees the reported total"); + + let release!: () => void; let calls = 0; + const gate = new Promise((resolve) => { release = resolve; }); + const live = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 1_000_000, timeoutMs: 60_000 }); + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; await gate; return { status: 200, headers: { "content-type": "text/event-stream" }, body: sse({ prompt_tokens: 1, completion_tokens: 1 }) }; }, undefined, 0); + try { + const token = proxy.capabilities.issue("agent", "turn"); + proxy.registerIsolationGuard("turn", async () => undefined); + proxy.registerTurn("turn", { policy: { model: "grok-4.6", reasoningEffort: "low" }, meter: live }); + const pending = post(proxy.port, token); + while (calls === 0) await new Promise((resolve) => setTimeout(resolve, 5)); + const overlapping = await post(proxy.port, token); + assert.deepEqual([overlapping.status, JSON.parse(overlapping.text)], [429, { error: "turn request in flight" }]); + assert.equal(calls, 1); + release(); + assert.equal((await pending).status, 200); + assert.equal((await post(proxy.port, token)).status, 200); + assert.deepEqual([calls, live.snapshot().requests, live.snapshot().limitReason], [2, 2, "none"]); + } finally { release(); await proxy.close(); } +}); + +test("tripping a limit aborts the in-flight upstream call instead of letting it run", async () => { + let observed: AbortSignal | undefined; + const meter = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 1_000_000, timeoutMs: 60_000 }); + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async (_request, signal) => { + observed = signal; + await new Promise((_resolve, reject) => signal?.addEventListener("abort", () => reject(new Error("aborted")), { once: true })); + throw new Error("unreachable"); + }, undefined, 0); + try { + const token = proxy.capabilities.issue("agent", "turn"); + proxy.registerIsolationGuard("turn", async () => undefined); + proxy.registerTurn("turn", { policy: { model: "grok-4.6", reasoningEffort: "low" }, meter }); + const pending = post(proxy.port, token); + while (observed === undefined) await new Promise((resolve) => setTimeout(resolve, 5)); + assert.equal(observed.aborted, false); + // Mutation guard: a trip that leaves the upstream signal alone hangs this request. + meter.trip("timeout"); + assert.equal(observed.aborted, true); + assert.equal((await pending).status, 503); + assert.deepEqual(meter.admit(), { refused: "timeout" }); + } finally { await proxy.close(); } +}); diff --git a/src/runtime/grokBrokerTurnMeter.ts b/src/runtime/grokBrokerTurnMeter.ts index 203a9be..5136734 100644 --- a/src/runtime/grokBrokerTurnMeter.ts +++ b/src/runtime/grokBrokerTurnMeter.ts @@ -21,32 +21,47 @@ export type GrokBrokerTurnMeterSnapshot = Readonly<{ requests: number; tokens: n * * The first limit that fires is sticky: every later request is refused with * the same reason, and `onLimit` runs once. + * + * The token bound is only a bound if no request can be admitted on a total + * that an in-flight request has not yet reported into. So a turn has at most + * ONE upstream request in flight: a second request arriving before the first + * settled is refused (`busy`, HTTP 429) without being counted or tripping a + * limit. Grok's headless loop is sequential — every live capture (P1 + * live-round1, P2 live) shows each request ending before the next starts — so + * this refuses only a worker that is not behaving like Grok. Tripping a limit + * (including the broker's timer) aborts that in-flight upstream call through + * its own `AbortSignal` rather than letting it run to completion. */ export class GrokBrokerTurnMeter { private readonly startedAt: number; private readonly timings: { startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage }[] = []; private tokens = 0; private reason: EngineBrokerLimitReason = "none"; + private inFlight: { index: number; controller: AbortController } | undefined; constructor(readonly limits: EngineBrokerTurnLimits, private readonly onLimit: (reason: Exclude) => void = () => undefined, private readonly now: () => number = Date.now) { this.startedAt = now(); } - /** Returns the request index when admitted, or the limit that refused it. */ - admit(): Readonly<{ index: number } | { refused: Exclude }> { + /** Returns the request index and its upstream abort signal when admitted, the limit that refused it, or `busy` while another request is in flight. */ + admit(): Readonly<{ index: number; signal: AbortSignal } | { refused: Exclude } | { busy: true }> { if (this.reason === "none") { if (this.now() - this.startedAt >= this.limits.timeoutMs) this.trip("timeout"); else if (this.timings.length >= this.limits.maxRequests) this.trip("requests"); else if (this.tokens >= this.limits.maxTokens) this.trip("tokens"); } if (this.reason !== "none") return { refused: this.reason }; + if (this.inFlight !== undefined) return { busy: true }; this.timings.push({ startedAt: new Date(this.now()).toISOString() }); - return { index: this.timings.length - 1 }; + const controller = new AbortController(); + this.inFlight = { index: this.timings.length - 1, controller }; + return { index: this.inFlight.index, signal: controller.signal }; } /** Records one admitted request's end and its upstream-reported usage, when the body carried any. */ settle(index: number, usage: EngineBrokerTurnUsage | undefined): void { const timing = this.timings[index]; if (timing === undefined || timing.endedAt !== undefined) return; + if (this.inFlight?.index === index) this.inFlight = undefined; timing.endedAt = new Date(this.now()).toISOString(); if (usage === undefined) return; timing.usage = usage; @@ -57,9 +72,13 @@ export class GrokBrokerTurnMeter { trip(reason: Exclude): void { if (this.reason !== "none") return; this.reason = reason; + this.abortInFlight(); this.onLimit(reason); } + /** Aborts the in-flight upstream call, if any (limit trip, or the broker ending the turn). */ + abortInFlight(): void { this.inFlight?.controller.abort(); } + snapshot(): GrokBrokerTurnMeterSnapshot { const measured = this.timings.flatMap((timing) => timing.usage === undefined ? [] : [timing.usage]); return { requests: this.timings.length, tokens: this.tokens, limitReason: this.reason, usage: sumEngineBrokerTurnUsage(measured), timings: this.timings.map((timing) => Object.freeze({ ...timing })) }; diff --git a/src/runtime/grokEngineBrokerTurn.ts b/src/runtime/grokEngineBrokerTurn.ts index 105ff02..22656a8 100644 --- a/src/runtime/grokEngineBrokerTurn.ts +++ b/src/runtime/grokEngineBrokerTurn.ts @@ -89,7 +89,7 @@ export async function runGrokEngineBrokerTurn(deps: GrokEngineBrokerTurnDependen await finishBrokerTurnWithUsage(deps.turns, request, failed, metering, { notionalUsd: 0, complete: false, reason, requests: requestRows(stream, snapshot), ...(stream?.sessionId === undefined ? {} : { session: stream.sessionId }) }); throw new EngineBrokerTurnFailure(code, diagnostic, accounting); } finally { - clearTimeout(timer); signal?.removeEventListener("abort", onAbort); + clearTimeout(timer); signal?.removeEventListener("abort", onAbort); meter.abortInFlight(); deps.proxy.revokeTurn(turnId); deps.proxy.revokeIsolationGuard(turnId); deps.proxy.capabilities.revoke(turnId); deps.mcp.revoke(turnId); } } diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index 408850b..6cca960 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -46,10 +46,10 @@ const untilAborted = (signal: AbortSignal): Promise => new Promise((_reso * talks to the proxy exactly as the native worker does (capability bearer, * pinned client version, lean body). Only the launcher and attestation are fakes. */ -const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId: string, worker: Worker, overrides?: Parameters[6], limits?: EngineBrokerServiceRegistration["limits"], turnStore?: string) => ReturnType; usageRows: () => Promise[]>; requestRows: () => Promise[]>; upstreamCalls: () => number }>) => Promise, usageLedgerPath?: string, upstreamDelayMs: (call: number) => number = () => 15): Promise => { +const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId: string, worker: Worker, overrides?: Parameters[6], limits?: EngineBrokerServiceRegistration["limits"], turnStore?: string) => ReturnType; usageRows: () => Promise[]>; requestRows: () => Promise[]>; upstreamCalls: () => number; upstreamAborts: () => number }>) => Promise, usageLedgerPath?: string, upstreamDelayMs: (call: number) => number = () => 15): Promise => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-usage-")); - let calls = 0; - const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; await new Promise((resolve) => setTimeout(resolve, upstreamDelayMs(calls))); return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(`data: ${JSON.stringify({ choices: [], usage: upstreamUsage })}\n\ndata: [DONE]\n\n`) }; }, undefined, 0); + let calls = 0, aborted = 0; + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async (_request, signal) => { calls++; await new Promise((resolve, reject) => { const timer = setTimeout(resolve, upstreamDelayMs(calls)); signal?.addEventListener("abort", () => { clearTimeout(timer); aborted++; reject(new Error("aborted")); }, { once: true }); }); return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(`data: ${JSON.stringify({ choices: [], usage: upstreamUsage })}\n\ndata: [DONE]\n\n`) }; }, undefined, 0); const ledger = usageLedgerPath ?? path.join(root, "usage.jsonl"); const rows = async (file: string) => (await readFile(file, "utf8").catch(() => "")).split("\n").filter((line) => line.length > 0).map((line) => JSON.parse(line) as Record); try { @@ -67,7 +67,8 @@ const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId }, usageRows: () => rows(ledger), requestRows: () => rows(path.join(path.dirname(ledger), "requests.jsonl")), - upstreamCalls: () => calls + upstreamCalls: () => calls, + upstreamAborts: () => aborted }); } finally { await proxy.close(); await rm(root, { recursive: true, force: true }); } }; @@ -128,12 +129,13 @@ test("the token ceiling stops a turn one request past the ceiling at most", asyn }); test("the wall-clock limit aborts a worker that is mid-request", async () => { - await withBroker(async ({ turn, usageRows, requestRows }) => { + await withBroker(async ({ turn, usageRows, requestRows, upstreamAborts }) => { const started = Date.now(); // Request 2 is still upstream (1.5 s) when the 1 s wall clock fires. const worker: Worker = async (send, signal) => { assert.equal(await send(), 200); void send().catch(() => undefined); return untilAborted(signal); }; await assert.rejects(turn("wake-4", worker, { timeoutMs: 1_000 }), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.code === "limit_exceeded" && error.accounting?.limitReason === "timeout" && error.accounting.requests === 2); assert.ok(Date.now() - started < 1_400, "the turn ends at the deadline, not when the in-flight request returns"); + assert.equal(upstreamAborts(), 1, "the stuck upstream call is aborted, not left running"); assert.deepEqual((await usageRows()).map((row) => [row.reason, row.limit_reason, row.total, row.calls]), [["wake_timeout", "timeout", 2_775, 2]]); // One measured row, but both admitted requests count: the killed one was sent upstream. assert.deepEqual((await requestRows()).map((row) => [row.request, row.requests]), [[0, 2]]); From 8932725781f7ba2b20592523357f1f5ceef74d83 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:39:24 +0200 Subject: [PATCH 041/124] fix: bound per-request Grok usage and charge an estimate when a response reports none --- src/contracts/runtimeContractManifest.ts | 8 ++++- src/pi/grokStreamUsage.test.ts | 7 +++++ src/pi/grokStreamUsage.ts | 7 ++++- src/runtime/grokBrokerProxy.ts | 2 +- src/runtime/grokBrokerTurnMeter.test.ts | 29 +++++++++++++++-- src/runtime/grokBrokerTurnMeter.ts | 38 +++++++++++++++++------ src/runtime/grokEngineBrokerMetering.ts | 4 +-- src/runtime/grokEngineBrokerTurn.ts | 8 ++--- src/runtime/grokEngineBrokerUsage.test.ts | 6 ++-- src/runtime/turnRequestLedger.ts | 8 ++++- src/runtime/turnUsageLedger.ts | 7 +++-- 11 files changed, 96 insertions(+), 28 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index ceef199..8aebfb1 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -78,7 +78,13 @@ export const GROK_ENGINE_BROKER = { bounds: { maxRequests: [1, GROK_WORKER_MAX_TURNS], maxTokens: [1, 10_000_000], timeoutMs: [1_000, 3_600_000] }, limitReasons: ["tokens", "requests", "timeout", "none"], wakeMayOnlyLower: true, - tokenCeilingOvershoot: "at-most-one-request" + tokenCeilingOvershoot: "at-most-one-request", + maxInFlightRequests: 1, + // A per-request usage block above this is implausible (beyond the model + // context window) and treated as invalid rather than added to any total. + requestUsageMaxTokens: 500_000, + // A request whose response carries no valid usage is charged this estimate. + missingUsageEstimate: { inputBytesPerToken: 2, outputTokens: 4_096 } }, wakeLimitEnvironment: { timeoutMs: "DAIMON_ENGINE_WAKE_TIMEOUT_MS", maxTokens: "DAIMON_ENGINE_WAKE_TOKEN_CEILING" }, projectionVersion: "noopolis.daimon.grok-broker-projection.v1", diff --git a/src/pi/grokStreamUsage.test.ts b/src/pi/grokStreamUsage.test.ts index 37942fb..8bf0f32 100644 --- a/src/pi/grokStreamUsage.test.ts +++ b/src/pi/grokStreamUsage.test.ts @@ -38,3 +38,10 @@ test("frames repeating one message id are one request, and a torn line is skippe test("the captured fixture carries no capturing machine's environment", async () => { assert.doesNotMatch(await fixture(), /\/Users\/|\/private\/|scratchpad|\/home\//u); }); + +test("a per-request stream block beyond the context-window bound is invalid, not counted", async () => { + const lines = (await fixture()).split("\n"); + const index = lines.findIndex((line) => line.includes('"msg_1"')); + lines[index] = lines[index]!.replace('"input_tokens":109', '"input_tokens":900000000'); + assert.deepEqual(decodeGrokStreamUsage(lines.join("\n")).requests, []); +}); diff --git a/src/pi/grokStreamUsage.ts b/src/pi/grokStreamUsage.ts index 79b4aef..bcb39e1 100644 --- a/src/pi/grokStreamUsage.ts +++ b/src/pi/grokStreamUsage.ts @@ -1,3 +1,5 @@ +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; + /** * Per-request token accounting read off a Grok `streaming-messages-json` * stream. @@ -27,7 +29,10 @@ const decodeUsage = (usage: unknown): Omit | undefine if (!isRecord(usage)) return undefined; const input = tokenCount(usage.input_tokens), output = tokenCount(usage.output_tokens), cacheRead = tokenCount(usage.cache_read_input_tokens), cacheWrite = tokenCount(usage.cache_creation_input_tokens); if (input === undefined || output === undefined || cacheRead === undefined || cacheWrite === undefined) return undefined; - return { input, cacheRead, cacheWrite, output, total: input + cacheRead + cacheWrite + output }; + const total = input + cacheRead + cacheWrite + output; + // Beyond the model context window one request cannot have spent it: invalid, like a malformed block. + if (total > GROK_ENGINE_BROKER.turnLimits.requestUsageMaxTokens) return undefined; + return { input, cacheRead, cacheWrite, output, total }; }; export const decodeGrokStreamUsage = (output: string): GrokStreamUsage => { diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index 57e9091..187bf7a 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -37,7 +37,7 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori const admission=turn.meter.admit(); if("refused" in admission){response.writeHead(429,{"content-type":"application/json","cache-control":"no-store"});response.end(JSON.stringify({error:"turn limit reached",limit:admission.refused}));return;} if("busy" in admission){response.writeHead(429,{"content-type":"application/json","cache-control":"no-store"});response.end('{"error":"turn request in flight"}');return;} - settle=(usage)=>{turn.meter.settle(admission.index,usage);settle=undefined;}; + settle=(usage)=>{turn.meter.settle(admission.index,usage,body.byteLength);settle=undefined;}; let result = await upstream(prepared,admission.signal); if (result.status === 401) { token = authority.refreshAfterRejection?await authority.refreshAfterRejection(rejectedDigest):await authority.accessToken(true);const refreshedDigest=createHash("sha256").update(token).digest("hex"); prepared = { ...prepared, headers: { ...prepared.headers, authorization: `Bearer ${token}` } }; token = ""; result = await upstream(prepared,admission.signal);if(result.status===401)await authority.markRejected(refreshedDigest); } settle?.(parseGrokUpstreamUsage(result.body,result.headers["content-type"])); diff --git a/src/runtime/grokBrokerTurnMeter.test.ts b/src/runtime/grokBrokerTurnMeter.test.ts index cd55082..f64911c 100644 --- a/src/runtime/grokBrokerTurnMeter.test.ts +++ b/src/runtime/grokBrokerTurnMeter.test.ts @@ -78,8 +78,11 @@ test("a request after the elapsed deadline is refused, and every admitted reques }); const snapshot = meter.snapshot(); assert.equal(snapshot.limitReason, "timeout"); - assert.equal(snapshot.usage, null, "a body without usage contributes no invented zero"); - assert.deepEqual(snapshot.timings, [{ startedAt: new Date(1_000_000).toISOString(), endedAt: new Date(1_000_000).toISOString() }]); + // A body without usage is never a zero: it is charged the conservative estimate (402-byte body). + const estimate = { input: 201, cacheRead: 0, cacheWrite: 0, output: 4_096, total: 4_297 }; + assert.deepEqual(snapshot.usage, estimate); + assert.equal(snapshot.estimatedRequests, 1); + assert.deepEqual(snapshot.timings, [{ startedAt: new Date(1_000_000).toISOString(), endedAt: new Date(1_000_000).toISOString(), usage: estimate, estimated: true }]); }); test("a turn without a registered meter is never forwarded", async () => { @@ -110,7 +113,7 @@ test("at most one upstream request is in flight per turn: an overlapping request assert.ok("index" in first); assert.deepEqual(meter.admit(), { busy: true }); assert.equal(meter.snapshot().requests, 1); - meter.settle(first.index, { input: 60, cacheRead: 0, cacheWrite: 0, output: 60, total: 120 }); + meter.settle(first.index, { input: 60, cacheRead: 0, cacheWrite: 0, output: 60, total: 120 }, 0); assert.deepEqual(meter.admit(), { refused: "tokens" }, "once settled, the next request sees the reported total"); let release!: () => void; let calls = 0; @@ -155,3 +158,23 @@ test("tripping a limit aborts the in-flight upstream call instead of letting it assert.deepEqual(meter.admit(), { refused: "timeout" }); } finally { await proxy.close(); } }); + +test("an implausible per-request usage block is never added, and missing usage still trips the token ceiling", async () => { + // Mutation guard: without the plausibility bound this adds 400 billion tokens to the total. + assert.equal(parseGrokUpstreamUsage(sse({ prompt_tokens: 400_000_000_000, completion_tokens: 1 }), "text/event-stream"), undefined); + assert.equal(parseGrokUpstreamUsage(sse({ prompt_tokens: 499_990, completion_tokens: 11 }), "text/event-stream"), undefined); + assert.equal(parseGrokUpstreamUsage(sse({ prompt_tokens: 499_990, completion_tokens: 10 }), "text/event-stream")?.total, 500_000); + const huge = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 1_000_000, timeoutMs: 60_000 }); + await withProxy({ prompt_tokens: 400_000_000_000, completion_tokens: 1 }, huge, async (send) => { assert.equal((await send()).status, 200); }); + assert.deepEqual([huge.snapshot().tokens, huge.snapshot().estimatedRequests], [4_297, 1]); + + // Mutation guard: settling a usage-less response as zero lets this turn run to maxRequests. + const blind = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 10_000, timeoutMs: 60_000 }); + await withProxy(undefined, blind, async (send, calls) => { + const statuses = []; + for (let index = 0; index < 5; index++) statuses.push((await send()).status); + assert.deepEqual(statuses, [200, 200, 200, 429, 429]); + assert.equal(calls(), 3); + }); + assert.deepEqual([blind.snapshot().limitReason, blind.snapshot().tokens, blind.snapshot().estimatedRequests], ["tokens", 12_891, 3]); +}); diff --git a/src/runtime/grokBrokerTurnMeter.ts b/src/runtime/grokBrokerTurnMeter.ts index 5136734..d5b03f4 100644 --- a/src/runtime/grokBrokerTurnMeter.ts +++ b/src/runtime/grokBrokerTurnMeter.ts @@ -1,7 +1,9 @@ +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; import { sumEngineBrokerTurnUsage, type EngineBrokerLimitReason, type EngineBrokerTurnLimits, type EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; -export type GrokBrokerRequestTiming = Readonly<{ startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage }>; -export type GrokBrokerTurnMeterSnapshot = Readonly<{ requests: number; tokens: number; limitReason: EngineBrokerLimitReason; usage: EngineBrokerTurnUsage | null; timings: readonly GrokBrokerRequestTiming[] }>; +/** `estimated` marks a request whose response carried no valid usage and was charged {@link estimateGrokRequestUsage}. */ +export type GrokBrokerRequestTiming = Readonly<{ startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage; estimated?: true }>; +export type GrokBrokerTurnMeterSnapshot = Readonly<{ requests: number; tokens: number; limitReason: EngineBrokerLimitReason; usage: EngineBrokerTurnUsage | null; estimatedRequests: number; timings: readonly GrokBrokerRequestTiming[] }>; /** * The proxy's per-turn spend gate. @@ -34,7 +36,7 @@ export type GrokBrokerTurnMeterSnapshot = Readonly<{ requests: number; tokens: n */ export class GrokBrokerTurnMeter { private readonly startedAt: number; - private readonly timings: { startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage }[] = []; + private readonly timings: { startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage; estimated?: true }[] = []; private tokens = 0; private reason: EngineBrokerLimitReason = "none"; private inFlight: { index: number; controller: AbortController } | undefined; @@ -57,15 +59,20 @@ export class GrokBrokerTurnMeter { return { index: this.inFlight.index, signal: controller.signal }; } - /** Records one admitted request's end and its upstream-reported usage, when the body carried any. */ - settle(index: number, usage: EngineBrokerTurnUsage | undefined): void { + /** + * Records one admitted request's end and its usage. A response without valid + * usage (absent, malformed, implausible, or a failed/aborted call) is charged + * a conservative estimate from the request body size, so a missing `usage` + * can never silently disable the token ceiling. + */ + settle(index: number, usage: EngineBrokerTurnUsage | undefined, requestBytes: number): void { const timing = this.timings[index]; if (timing === undefined || timing.endedAt !== undefined) return; if (this.inFlight?.index === index) this.inFlight = undefined; timing.endedAt = new Date(this.now()).toISOString(); - if (usage === undefined) return; - timing.usage = usage; - this.tokens += usage.total; + if (usage === undefined) { timing.usage = estimateGrokRequestUsage(requestBytes); timing.estimated = true; } + else timing.usage = usage; + this.tokens += timing.usage.total; } /** Trips a limit from outside the request path (the broker's wall-clock timer). */ @@ -81,10 +88,18 @@ export class GrokBrokerTurnMeter { snapshot(): GrokBrokerTurnMeterSnapshot { const measured = this.timings.flatMap((timing) => timing.usage === undefined ? [] : [timing.usage]); - return { requests: this.timings.length, tokens: this.tokens, limitReason: this.reason, usage: sumEngineBrokerTurnUsage(measured), timings: this.timings.map((timing) => Object.freeze({ ...timing })) }; + return { requests: this.timings.length, tokens: this.tokens, limitReason: this.reason, usage: sumEngineBrokerTurnUsage(measured), estimatedRequests: this.timings.filter((timing) => timing.estimated === true).length, timings: this.timings.map((timing) => Object.freeze({ ...timing })) }; } } +const { requestUsageMaxTokens, missingUsageEstimate } = GROK_ENGINE_BROKER.turnLimits; + +/** The charge for a request without valid usage: `ceil(bodyBytes / 2)` input plus a fixed output allowance. */ +export const estimateGrokRequestUsage = (requestBytes: number): EngineBrokerTurnUsage => { + const input = Math.ceil(Math.max(0, requestBytes) / missingUsageEstimate.inputBytesPerToken), output = missingUsageEstimate.outputTokens; + return { input, cacheRead: 0, cacheWrite: 0, output, total: input + output }; +}; + type JsonRecord = Record; const isRecord = (value: unknown): value is JsonRecord => value !== null && typeof value === "object" && !Array.isArray(value); const count = (value: unknown): number | undefined => typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : undefined; @@ -98,7 +113,9 @@ const count = (value: unknown): number | undefined => typeof value === "number" * include cached tokens; they are split into disjoint buckets here, and any * reasoning tokens reported outside `completion_tokens` (visible as * `total_tokens` above prompt + completion) are folded into `output` so the - * total invariant holds. A malformed block is ignored, never zero-filled. + * total invariant holds. A malformed block, or one whose total exceeds + * `GROK_ENGINE_BROKER.turnLimits.requestUsageMaxTokens`, is invalid: never + * zero-filled and never added — the meter charges an estimate instead. */ export function parseGrokUpstreamUsage(body: Uint8Array, contentType: string | undefined): EngineBrokerTurnUsage | undefined { const text = Buffer.from(body.buffer, body.byteOffset, body.byteLength).toString("utf8"); @@ -133,5 +150,6 @@ function decodeOpenAiUsage(usage: JsonRecord): EngineBrokerTurnUsage | undefined const completionDetails = isRecord(usage.completion_tokens_details) ? usage.completion_tokens_details : {}; const reasoning = count(completionDetails.reasoning_tokens); const output = total - prompt; + if (total > requestUsageMaxTokens) return undefined; return { input: prompt - cached, cacheRead: cached, cacheWrite: 0, output, total, ...(reasoning === undefined || reasoning > output ? {} : { reasoning }) }; } diff --git a/src/runtime/grokEngineBrokerMetering.ts b/src/runtime/grokEngineBrokerMetering.ts index bd4e482..12c25a4 100644 --- a/src/runtime/grokEngineBrokerMetering.ts +++ b/src/runtime/grokEngineBrokerMetering.ts @@ -9,7 +9,7 @@ export type BrokerTurnMetering = Readonly<{ agentId: string; wakeId: string; }>; -export type BrokerTurnMeteringDetail = Readonly<{ notionalUsd: number; complete: boolean; reason?: TurnUsageFailureReason; requests: readonly GrokTurnRequest[]; session?: string }>; +export type BrokerTurnMeteringDetail = Readonly<{ notionalUsd: number; complete: boolean; reason?: TurnUsageFailureReason; requests: readonly GrokTurnRequest[]; session?: string; estimatedRequests: number }>; /** * Seal a terminal turn, then meter it. The broker is the single writer. @@ -37,7 +37,7 @@ export async function finishBrokerTurnWithUsage(turns: EngineBrokerTurnRegistry, agent: metering.agentId, wake: metering.wakeId, engine: "grok", usage: { input: usage.input, output: usage.output, cacheRead: usage.cacheRead, cacheWrite: usage.cacheWrite, total: usage.total, calls: terminal.requests, notionalUsd: detail.notionalUsd, complete: detail.complete }, outcome: terminal.kind === "completed" ? { status: "completed" } : { status: "failed", reason: detail.reason ?? "unknown" }, - turn: terminal.turnId, limitReason: terminal.limitReason, model: terminal.model + turn: terminal.turnId, limitReason: terminal.limitReason, model: terminal.model, estimatedRequests: detail.estimatedRequests }); await recordGrokTurnRequests(metering.requestLedgerPath, { agent: metering.agentId, wake: metering.wakeId, turn: terminal.turnId, model: terminal.model, requests: detail.requests, requestCount: terminal.requests, ...(detail.session === undefined ? {} : { session: detail.session }) }); } diff --git a/src/runtime/grokEngineBrokerTurn.ts b/src/runtime/grokEngineBrokerTurn.ts index 22656a8..650c94b 100644 --- a/src/runtime/grokEngineBrokerTurn.ts +++ b/src/runtime/grokEngineBrokerTurn.ts @@ -76,7 +76,7 @@ export async function runGrokEngineBrokerTurn(deps: GrokEngineBrokerTurnDependen const usage = decoded.usage === undefined ? streamOrMeterUsage(stream, snapshot) : usageOf(decoded.usage); const accounting = { outcome: "completed", usage, model: declared, requests: requestCount(stream, snapshot), limitReason: "none" } as const; const completed = { version: request.version, kind: "completed", requestId: request.requestId, turnId, text: decoded.text, workerPid: result.workerPid, workerUid: result.workerUid, workerStartTime: result.startTicks.toString(), ...accounting } as const; - await finishBrokerTurnWithUsage(deps.turns, request, completed, metering, { notionalUsd: decoded.usage?.notionalUsd ?? 0, complete: decoded.usage?.complete ?? false, requests: requestRows(stream, snapshot), ...(stream.sessionId === undefined ? {} : { session: stream.sessionId }) }); + await finishBrokerTurnWithUsage(deps.turns, request, completed, metering, { notionalUsd: decoded.usage?.notionalUsd ?? 0, complete: decoded.usage?.complete ?? false, estimatedRequests: snapshot.estimatedRequests, requests: requestRows(stream, snapshot), ...(stream.sessionId === undefined ? {} : { session: stream.sessionId }) }); return { text: completed.text, workerPid: completed.workerPid, workerUid: completed.workerUid, workerStartTime: completed.workerStartTime, ...accounting }; } catch (error) { const snapshot = meter.snapshot(); @@ -86,7 +86,7 @@ export async function runGrokEngineBrokerTurn(deps: GrokEngineBrokerTurnDependen const accounting = { outcome: "failed", usage: streamOrMeterUsage(stream, snapshot), model: declared, requests: requestCount(stream, snapshot), limitReason: snapshot.limitReason } as const; const failed: EngineBrokerTerminalResponse = { version: request.version, kind: "failed", requestId: request.requestId, turnId, code, ...(diagnostic ? { diagnostic } : {}), ...accounting }; const reason = snapshot.limitReason !== "none" ? limitReasonFor[snapshot.limitReason] : rejected ? "turn_rejected" : "unknown"; - await finishBrokerTurnWithUsage(deps.turns, request, failed, metering, { notionalUsd: 0, complete: false, reason, requests: requestRows(stream, snapshot), ...(stream?.sessionId === undefined ? {} : { session: stream.sessionId }) }); + await finishBrokerTurnWithUsage(deps.turns, request, failed, metering, { notionalUsd: 0, complete: false, reason, estimatedRequests: snapshot.estimatedRequests, requests: requestRows(stream, snapshot), ...(stream?.sessionId === undefined ? {} : { session: stream.sessionId }) }); throw new EngineBrokerTurnFailure(code, diagnostic, accounting); } finally { clearTimeout(timer); signal?.removeEventListener("abort", onAbort); meter.abortInFlight(); @@ -117,8 +117,8 @@ function streamOrMeterUsage(stream: GrokStreamUsage | undefined, snapshot: GrokB function requestRows(stream: GrokStreamUsage | undefined, snapshot: GrokBrokerTurnMeterSnapshot): BrokerTurnMeteringDetail["requests"] { if (stream !== undefined && stream.requests.length > 0) { const timed = snapshot.timings.length === stream.requests.length; - return stream.requests.map((value, index) => ({ ...value, ...(timed ? clock(snapshot.timings[index]!) : {}) })); + return stream.requests.map((value, index) => ({ ...value, usageSource: "stream" as const, ...(timed ? clock(snapshot.timings[index]!) : {}) })); } - return snapshot.timings.flatMap((timing, index) => timing.usage === undefined ? [] : [{ index, ...usageOf(timing.usage), ...clock(timing) }]); + return snapshot.timings.flatMap((timing, index) => timing.usage === undefined ? [] : [{ index, ...usageOf(timing.usage), usageSource: timing.estimated === true ? "estimated" as const : "upstream" as const, ...clock(timing) }]); } const clock = (timing: GrokBrokerTurnMeterSnapshot["timings"][number]) => ({ startedAt: timing.startedAt, ...(timing.endedAt === undefined ? {} : { endedAt: timing.endedAt }) }); diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index 6cca960..9dec696 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -136,9 +136,9 @@ test("the wall-clock limit aborts a worker that is mid-request", async () => { await assert.rejects(turn("wake-4", worker, { timeoutMs: 1_000 }), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.code === "limit_exceeded" && error.accounting?.limitReason === "timeout" && error.accounting.requests === 2); assert.ok(Date.now() - started < 1_400, "the turn ends at the deadline, not when the in-flight request returns"); assert.equal(upstreamAborts(), 1, "the stuck upstream call is aborted, not left running"); - assert.deepEqual((await usageRows()).map((row) => [row.reason, row.limit_reason, row.total, row.calls]), [["wake_timeout", "timeout", 2_775, 2]]); - // One measured row, but both admitted requests count: the killed one was sent upstream. - assert.deepEqual((await requestRows()).map((row) => [row.request, row.requests]), [[0, 2]]); + // The aborted request reported nothing, so it is charged the estimate and says so. + assert.deepEqual((await usageRows()).map((row) => [row.reason, row.limit_reason, row.total, row.calls, row.estimated_requests]), [["wake_timeout", "timeout", 2_775 + 4_297, 2, 1]]); + assert.deepEqual((await requestRows()).map((row) => [row.request, row.requests, row.usage_source, row.total]), [[0, 2, "upstream", 2_775], [1, 2, "estimated", 4_297]]); }, undefined, (call) => call === 2 ? 1_500 : 15); }); diff --git a/src/runtime/turnRequestLedger.ts b/src/runtime/turnRequestLedger.ts index ef8dfca..6cd2844 100644 --- a/src/runtime/turnRequestLedger.ts +++ b/src/runtime/turnRequestLedger.ts @@ -105,7 +105,12 @@ const requestClockFields = (request: Readonly<{ startedAt?: string; endedAt?: st }); /** One Grok broker model request: usage from the worker stream, timing from the proxy. */ -export type GrokTurnRequest = Readonly<{ index: number; input: number; cacheRead: number; cacheWrite: number; output: number; total: number; startedAt?: string; endedAt?: string }>; +/** + * `usageSource`: `stream` (the worker's own per-request frame), `upstream` + * (the provider response the proxy saw), or `estimated` (no valid usage; the + * proxy's conservative charge, see `grokBrokerTurnMeter.ts`). + */ +export type GrokTurnRequest = Readonly<{ index: number; input: number; cacheRead: number; cacheWrite: number; output: number; total: number; startedAt?: string; endedAt?: string; usageSource?: "stream" | "upstream" | "estimated" }>; /** * `requestCount` is the turn's admitted request count when it exceeds the rows: * a killed turn's in-flight request was sent upstream but never reported usage, @@ -140,6 +145,7 @@ export const renderGrokTurnRequestLines = (entry: GrokTurnRequestEntry): string cache_write: request.cacheWrite, output: request.output, total: request.total, + ...(request.usageSource === undefined ? {} : { usage_source: request.usageSource }), ...requestClockFields(request) })}\n`).join(""); }; diff --git a/src/runtime/turnUsageLedger.ts b/src/runtime/turnUsageLedger.ts index 44e7621..6256caf 100644 --- a/src/runtime/turnUsageLedger.ts +++ b/src/runtime/turnUsageLedger.ts @@ -125,6 +125,8 @@ export type TurnUsageEntry = Readonly<{ turn?: string; limitReason?: EngineBrokerLimitReason; model?: GrokBrokerModel; + /** Broker rows only: how many of the turn's requests were charged an estimate because their response carried no valid usage. */ + estimatedRequests?: number; }>; /** @@ -185,10 +187,11 @@ export const renderTurnUsageLine = (entry: TurnUsageEntry): string => `${JSON.st ...brokerFields(entry) })}\n`; -const brokerFields = (entry: TurnUsageEntry): Record => ({ +const brokerFields = (entry: TurnUsageEntry): Record => ({ ...(entry.turn !== undefined && /^[a-f0-9]{64}$/u.test(entry.turn) ? { turn: entry.turn } : {}), ...(entry.limitReason !== undefined && ENGINE_BROKER_LIMIT_REASONS.includes(entry.limitReason) ? { limit_reason: entry.limitReason } : {}), - ...(entry.model !== undefined && (GROK_BROKER_MODELS as readonly string[]).includes(entry.model) ? { model: entry.model } : {}) + ...(entry.model !== undefined && (GROK_BROKER_MODELS as readonly string[]).includes(entry.model) ? { model: entry.model } : {}), + ...(entry.estimatedRequests !== undefined && Number.isSafeInteger(entry.estimatedRequests) && entry.estimatedRequests > 0 ? { estimated_requests: entry.estimatedRequests } : {}) }); /** From 9e49e924505687cbb27d2bd8a91869214d355de7 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:42:11 +0200 Subject: [PATCH 042/124] fix: seal Grok turn ledger bytes in the turn record and complete an interrupted append on replay --- src/runtime/engineBrokerTurnRegistry.test.ts | 26 ++++++- src/runtime/engineBrokerTurnRegistry.ts | 22 +++--- src/runtime/grokEngineBrokerLedger.ts | 82 ++++++++++++++++++++ src/runtime/grokEngineBrokerMetering.ts | 45 ++++++----- src/runtime/grokEngineBrokerTurn.ts | 13 ++-- src/runtime/grokEngineBrokerUsage.test.ts | 19 ++++- src/runtime/turnRequestLedger.ts | 10 +++ 7 files changed, 177 insertions(+), 40 deletions(-) create mode 100644 src/runtime/grokEngineBrokerLedger.ts diff --git a/src/runtime/engineBrokerTurnRegistry.test.ts b/src/runtime/engineBrokerTurnRegistry.test.ts index 3de4e20..249cbd6 100644 --- a/src/runtime/engineBrokerTurnRegistry.test.ts +++ b/src/runtime/engineBrokerTurnRegistry.test.ts @@ -11,14 +11,14 @@ test("turn registry replays terminal results across restart and rejects conflict try { const first = new EngineBrokerTurnRegistry(root,"boot-a"); assert.equal(await first.begin(start(),"grok-4.6"), "start"); await assert.rejects(first.begin(start(),"grok-4.6"), /already active/); - const response = { version: "noopolis.daimon.engine-broker.v2", kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 11, workerUid: 2200, workerStartTime: "123", outcome: "completed", usage: { input: 3, cacheRead: 2, cacheWrite: 0, output: 1, total: 6 }, model: "grok-4.6", requests: 1, limitReason: "none" } as const; + const response = { version: "noopolis.daimon.engine-broker.v2", kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 11, workerUid: 2200, workerStartTime: "123", outcome: "completed", usage: null, model: "grok-4.6", requests: 1, limitReason: "none" } as const; await first.finish(start(), response); - assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-b").begin(start(),"grok-4.6"), { replay: response }); + assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-b").begin(start(),"grok-4.6"), { replay: response, ledger: { usage: null, requests: "" } }); await assert.rejects(first.begin(start("different"),"grok-4.6"), /conflict/); } finally { await rm(root, { recursive: true, force: true }); } }); test("turn registry fails an orphaned active turn once after broker restart",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{assert.equal(await new EngineBrokerTurnRegistry(root,"boot-a").begin(start(),"grok-4.6"),"start");const replay=await new EngineBrokerTurnRegistry(root,"boot-b").begin(start(),"grok-4.6");assert.equal(typeof replay,"object");if(typeof replay==="object")assert.equal(replay.replay.kind,"failed");assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-c").begin(start(),"grok-4.6"),replay);}finally{await rm(root,{recursive:true,force:true});}}); -test("turn registry durably replays a sanitized pre-attestation failure",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{const registry=new EngineBrokerTurnRegistry(root,"boot-a");assert.equal(await registry.begin(start(),"grok-4.6"),"start");const response={version:"noopolis.daimon.engine-broker.v2",kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",diagnostic:{status:"worker_failed",stage:"attestation",failureClass:"profile_missing",profileApplied:false,exitCode:0,termSignal:0,workerPid:42,workerUid:2200,startTicks:"123"},outcome:"failed",usage:null,model:"grok-4.6",requests:0,limitReason:"none"} as const;await registry.finish(start(),response);assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-b").begin(start(),"grok-4.6"),{replay:response});}finally{await rm(root,{recursive:true,force:true});}}); +test("turn registry durably replays a sanitized pre-attestation failure",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{const registry=new EngineBrokerTurnRegistry(root,"boot-a");assert.equal(await registry.begin(start(),"grok-4.6"),"start");const response={version:"noopolis.daimon.engine-broker.v2",kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",diagnostic:{status:"worker_failed",stage:"attestation",failureClass:"profile_missing",profileApplied:false,exitCode:0,termSignal:0,workerPid:42,workerUid:2200,startTicks:"123"},outcome:"failed",usage:null,model:"grok-4.6",requests:0,limitReason:"none"} as const;await registry.finish(start(),response);assert.deepEqual(await new EngineBrokerTurnRegistry(root,"boot-b").begin(start(),"grok-4.6"),{replay:response,ledger:{usage:null,requests:""}});}finally{await rm(root,{recursive:true,force:true});}}); test("turn registry rejects a persisted diagnostic with undeclared secret-bearing fields",async()=>{const root=await mkdtemp(path.join(os.tmpdir(),"daimon-broker-turns-"));try{const registry=new EngineBrokerTurnRegistry(root,"boot-a");assert.equal(await registry.begin(start(),"grok-4.6"),"start");const [name]=await readdir(root);const file=path.join(root,name!);const record=JSON.parse(await readFile(file,"utf8")) as Record;record.state="terminal";record.response={version:"noopolis.daimon.engine-broker.v2",kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",rawOutput:"secret",outcome:"failed",usage:null,model:"grok-4.6",requests:0,limitReason:"none"};await writeFile(file,JSON.stringify(record));await assert.rejects(new EngineBrokerTurnRegistry(root,"boot-b").begin(start(),"grok-4.6"),/registry unavailable/u);}finally{await rm(root,{recursive:true,force:true});}}); const withRoot = async (run: (root: string) => Promise): Promise => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-turns-")); try { await run(root); } finally { await rm(root, { recursive: true, force: true }); } }; @@ -31,7 +31,7 @@ test("a v1 record sealed before the upgrade still replays, upgraded with no usag const file = await recordFile(root); const record = JSON.parse(await readFile(file, "utf8")) as Record; const v1 = { version: "noopolis.daimon.engine-broker.v1", kind: "completed", requestId: "request-1", turnId: "turn-1", text: "done", workerPid: 11, workerUid: 2200, workerStartTime: "123" }; await writeFile(file, JSON.stringify({ version: "noopolis.daimon.engine-broker-turn.v1", digest: record.digest, state: "terminal", bootId: "boot-a", response: v1 })); - assert.deepEqual(await new EngineBrokerTurnRegistry(root, "boot-b").begin(start(), "grok-4.5"), { replay: { ...v1, version: "noopolis.daimon.engine-broker.v2", outcome: "completed", usage: null, model: "grok-4.5", requests: 0, limitReason: "none" } }); + assert.deepEqual(await new EngineBrokerTurnRegistry(root, "boot-b").begin(start(), "grok-4.5"), { replay: { ...v1, version: "noopolis.daimon.engine-broker.v2", outcome: "completed", usage: null, model: "grok-4.5", requests: 0, limitReason: "none" }, ledger: { usage: null, requests: "" } }); }); }); @@ -44,6 +44,7 @@ test("the v2 record parser is strict: an unknown member or a v1 frame inside a v const file = await recordFile(root); const record = JSON.parse(await readFile(file, "utf8")) as Record; assert.equal(record.version, "noopolis.daimon.engine-broker-turn.v2"); // Mutation guard: dropping the exact-member check accepts this record. + assert.deepEqual(record.ledger, { usage: null, requests: "" }); await writeFile(file, JSON.stringify({ ...record, usageRow: "extra" })); await assert.rejects(new EngineBrokerTurnRegistry(root, "boot-b").begin(start(), "grok-4.6"), /registry unavailable/u); const { outcome: _o, usage: _u, model: _m, requests: _r, limitReason: _l, ...legacy } = response; @@ -51,3 +52,20 @@ test("the v2 record parser is strict: an unknown member or a v1 frame inside a v await assert.rejects(new EngineBrokerTurnRegistry(root, "boot-b").begin(start(), "grok-4.6"), /registry unavailable/u); }); }); + +test("sealed ledger bytes must be this turn's own rows and agree with the sealed usage", async () => { + await withRoot(async (root) => { + const registry = new EngineBrokerTurnRegistry(root, "boot-a"); + assert.equal(await registry.begin(start(), "grok-4.6"), "start"); + const turnId = "turn-1"; + const usage = { input: 3, cacheRead: 2, cacheWrite: 0, output: 1, total: 6 }; + const response = { version: "noopolis.daimon.engine-broker.v2", kind: "completed", requestId: "request-1", turnId, text: "done", workerPid: 11, workerUid: 2200, workerStartTime: "123", outcome: "completed", usage, model: "grok-4.6", requests: 1, limitReason: "none" } as const; + const line = (turn: string) => `${JSON.stringify({ v: "noopolis.daimon.turn-usage.v1", turn, total: 6 })}\n`; + await registry.finish(start(), response, { usage: line(turnId), requests: "" }); + assert.deepEqual(await new EngineBrokerTurnRegistry(root, "boot-b").begin(start(), "grok-4.6"), { replay: response, ledger: { usage: line(turnId), requests: "" } }); + for (const ledger of [{ usage: line("other-turn"), requests: "" }, { usage: null, requests: "" }, { usage: line(turnId), requests: "not json\n" }, { usage: line(turnId), requests: "", extra: 1 }]) { + await registry.finish(start(), response, ledger as never); + await assert.rejects(new EngineBrokerTurnRegistry(root, "boot-c").begin(start(), "grok-4.6"), /registry unavailable/u, JSON.stringify(ledger)); + } + }); +}); diff --git a/src/runtime/engineBrokerTurnRegistry.ts b/src/runtime/engineBrokerTurnRegistry.ts index 1ddd01c..d59bab7 100644 --- a/src/runtime/engineBrokerTurnRegistry.ts +++ b/src/runtime/engineBrokerTurnRegistry.ts @@ -4,6 +4,7 @@ import { mkdir, open, readFile, rename, unlink } from "node:fs/promises"; import path from "node:path"; import { parseEngineBrokerResponse, parseEngineBrokerV1TerminalResponse, type EngineBrokerRequest, type EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; import type { GrokBrokerModel } from "./grokBrokerModelPolicy.js"; +import { EMPTY_BROKER_TURN_LEDGER, parseBrokerTurnLedgerLines, type BrokerTurnLedgerLines } from "./grokEngineBrokerLedger.js"; type Start = Extract; type Terminal = EngineBrokerTerminalResponse; @@ -13,7 +14,7 @@ export const ENGINE_BROKER_TURN_RECORD_V2 = "noopolis.daimon.engine-broker-turn. // record written before the upgrade still identifies the same turn. const digest = (request: Start): string => createHash("sha256").update(JSON.stringify([request.turnId, request.agentId, request.wakeId, request.prompt,request.mcpEndpoint])).digest("hex"); const safe = (turnId: string): string => `${createHash("sha256").update(turnId).digest("hex")}.json`; -type Observed = Readonly<{ version: typeof ENGINE_BROKER_TURN_RECORD_V1 | typeof ENGINE_BROKER_TURN_RECORD_V2; digest: string; state: "active" | "terminal"; bootId: string; response?: Terminal }>; +type Observed = Readonly<{ version: typeof ENGINE_BROKER_TURN_RECORD_V1 | typeof ENGINE_BROKER_TURN_RECORD_V2; digest: string; state: "active" | "terminal"; bootId: string; response?: Terminal; ledger?: BrokerTurnLedgerLines }>; /** * Durable per-turn state. Record v2 stores the terminal response *with* its @@ -24,28 +25,29 @@ type Observed = Readonly<{ version: typeof ENGINE_BROKER_TURN_RECORD_V1 | typeof export class EngineBrokerTurnRegistry { constructor(private readonly root: string,private readonly bootId:string=randomUUID()) {} /** `model` is the registration's declared model, used only to upgrade a v1 record on replay. */ - async begin(request: Start, model: GrokBrokerModel): Promise<"start" | { replay: Terminal }> { + async begin(request: Start, model: GrokBrokerModel): Promise<"start" | { replay: Terminal; ledger: BrokerTurnLedgerLines }> { await mkdir(this.root, { recursive: true, mode: 0o700 }); const file = path.join(this.root, safe(request.turnId)); const expected = digest(request); try { const handle = await open(file, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); try { await handle.writeFile(`${JSON.stringify({ version: ENGINE_BROKER_TURN_RECORD_V2, digest: expected, state: "active",bootId:this.bootId })}\n`); await handle.sync(); } finally { await handle.close(); } await syncDirectory(this.root); return "start"; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw new Error("broker turn registry unavailable"); } const observed = parseEngineBrokerTurnRecord(await readFile(file, "utf8"), model); if (observed.digest !== expected) throw new Error("broker turn conflict"); - if (observed.state === "terminal" && observed.response !== undefined) return { replay: observed.response }; - if(observed.state==="active"&&observed.bootId!==this.bootId){const response={version:request.version,kind:"failed",requestId:request.requestId,turnId:request.turnId,code:"engine_failed",outcome:"failed",usage:null,model,requests:0,limitReason:"none"} as const;await this.finish(request,response);return {replay:response};} + if (observed.state === "terminal" && observed.response !== undefined) return { replay: observed.response, ledger: observed.ledger ?? EMPTY_BROKER_TURN_LEDGER }; + if(observed.state==="active"&&observed.bootId!==this.bootId){const response={version:request.version,kind:"failed",requestId:request.requestId,turnId:request.turnId,code:"engine_failed",outcome:"failed",usage:null,model,requests:0,limitReason:"none"} as const;await this.finish(request,response);return {replay:response,ledger:EMPTY_BROKER_TURN_LEDGER};} throw new Error("broker turn already active"); } - async finish(request: Start, response: Terminal): Promise { + /** `ledger` is the exact ledger bytes this turn owes, sealed with it so a replay can finish an interrupted append. */ + async finish(request: Start, response: Terminal, ledger: BrokerTurnLedgerLines = EMPTY_BROKER_TURN_LEDGER): Promise { const file = path.join(this.root, safe(request.turnId)); const temporary = `${file}.${randomUUID()}.tmp`; const handle = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); - try { await handle.writeFile(`${JSON.stringify({ version: ENGINE_BROKER_TURN_RECORD_V2, digest: digest(request), state: "terminal",bootId:this.bootId, response })}\n`); await handle.sync(); } finally { await handle.close(); } + try { await handle.writeFile(`${JSON.stringify({ version: ENGINE_BROKER_TURN_RECORD_V2, digest: digest(request), state: "terminal",bootId:this.bootId, response, ledger })}\n`); await handle.sync(); } finally { await handle.close(); } try { await rename(temporary, file); await syncDirectory(this.root); } finally { await unlink(temporary).catch(() => undefined); } } } /** * Strict record parser. v2 accepts exactly `{version,digest,state,bootId}` - * plus `response` when terminal, and the response must be a v2 terminal frame. + * plus `response` and its sealed `ledger` bytes when terminal, and the response must be a v2 terminal frame. * v1 records keep their historical looser shape and are upgraded on read: no * usage (`null`), zero requests, `limitReason: "none"`, the declared model. */ @@ -65,13 +67,15 @@ export function parseEngineBrokerTurnRecord(text: string, model: GrokBrokerModel return { version: ENGINE_BROKER_TURN_RECORD_V1, ...base, response }; } if (input.version !== ENGINE_BROKER_TURN_RECORD_V2) throw new Error("broker turn conflict"); - const fields = input.state === "terminal" ? ["version", "digest", "state", "bootId", "response"] : ["version", "digest", "state", "bootId"]; + const fields = input.state === "terminal" ? ["version", "digest", "state", "bootId", "response", "ledger"] : ["version", "digest", "state", "bootId"]; if (Object.keys(input).length !== fields.length || fields.some((field) => !Object.hasOwn(input, field))) throw new Error("broker turn registry unavailable"); if (input.state === "active") return { version: ENGINE_BROKER_TURN_RECORD_V2, ...base }; let response; try { response = parseEngineBrokerResponse(input.response); } catch { throw new Error("broker turn registry unavailable"); } if (response.kind !== "completed" && response.kind !== "failed") throw new Error("broker turn registry unavailable"); - return { version: ENGINE_BROKER_TURN_RECORD_V2, ...base, response }; + const ledger = parseBrokerTurnLedgerLines(input.ledger, response.turnId); + if ((response.usage === null) !== (ledger.usage === null)) throw new Error("broker turn registry unavailable"); + return { version: ENGINE_BROKER_TURN_RECORD_V2, ...base, response, ledger }; } async function syncDirectory(directory: string): Promise { const handle = await open(directory, constants.O_RDONLY); try { await handle.sync(); } finally { await handle.close(); } } diff --git a/src/runtime/grokEngineBrokerLedger.ts b/src/runtime/grokEngineBrokerLedger.ts new file mode 100644 index 0000000..ebf6073 --- /dev/null +++ b/src/runtime/grokEngineBrokerLedger.ts @@ -0,0 +1,82 @@ +import { readFile } from "node:fs/promises"; + +import { recordLedgerLines, renderGrokTurnRequestLines, TURN_REQUEST_LEDGER_VERSION, type GrokTurnRequest } from "./turnRequestLedger.js"; +import { renderTurnUsageLine, TURN_USAGE_LEDGER_VERSION, type TurnUsageFailureReason } from "./turnUsageLedger.js"; +import type { EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; + +/** + * The exact ledger bytes a terminal broker turn owes, sealed into its turn + * record *before* they are appended. + * + * The record is published first and the ledger appended second, so a crash in + * between used to leave a sealed turn whose spend never reached the ledger — + * and a replay never metered. Now a replay re-checks: if the ledger holds no + * row for this `turn`, it appends these same bytes (same `at`, same numbers). + * That is completing the original metering, not re-metering: a replay after a + * normal append finds the row and writes nothing, and readers dedupe on `turn` + * should two replays race. + */ +export type BrokerTurnLedgerLines = Readonly<{ usage: string | null; requests: string }>; +export const EMPTY_BROKER_TURN_LEDGER: BrokerTurnLedgerLines = Object.freeze({ usage: null, requests: "" }); + +export type BrokerTurnLedgerDetail = Readonly<{ agentId: string; wakeId: string; notionalUsd: number; complete: boolean; reason?: TurnUsageFailureReason; requests: readonly GrokTurnRequest[]; session?: string; estimatedRequests: number }>; + +export function renderBrokerTurnLedger(terminal: EngineBrokerTerminalResponse, detail: BrokerTurnLedgerDetail): BrokerTurnLedgerLines { + if (terminal.usage === null) return EMPTY_BROKER_TURN_LEDGER; + const { usage } = terminal, at = new Date().toISOString(); + return { + usage: renderTurnUsageLine({ + agent: detail.agentId, wake: detail.wakeId, engine: "grok", at, + usage: { input: usage.input, output: usage.output, cacheRead: usage.cacheRead, cacheWrite: usage.cacheWrite, total: usage.total, calls: terminal.requests, notionalUsd: detail.notionalUsd, complete: detail.complete }, + outcome: terminal.kind === "completed" ? { status: "completed" } : { status: "failed", reason: detail.reason ?? "unknown" }, + turn: terminal.turnId, limitReason: terminal.limitReason, model: terminal.model, estimatedRequests: detail.estimatedRequests + }), + requests: renderGrokTurnRequestLines({ agent: detail.agentId, wake: detail.wakeId, turn: terminal.turnId, model: terminal.model, requests: detail.requests, requestCount: terminal.requests, at, ...(detail.session === undefined ? {} : { session: detail.session }) }) + }; +} + +const MAX_USAGE_LINE_BYTES = 4_096, MAX_REQUEST_LINES_BYTES = 262_144; +const rows = (text: string): Record[] => text.split("\n").filter((line) => line.length > 0).map((line) => { const value: unknown = JSON.parse(line); if (value === null || typeof value !== "object" || Array.isArray(value)) throw new Error(); return value as Record; }); + +/** Strict check of stored ledger bytes: exactly the turn's own rows, newline-terminated, bounded. */ +export function parseBrokerTurnLedgerLines(value: unknown, turnId: string): BrokerTurnLedgerLines { + const invalid = () => new Error("broker turn registry unavailable"); + if (value === null || typeof value !== "object" || Array.isArray(value) || Object.keys(value).length !== 2 || !Object.hasOwn(value, "usage") || !Object.hasOwn(value, "requests")) throw invalid(); + const { usage, requests } = value as { usage: unknown; requests: unknown }; + try { + if (usage !== null) { + if (typeof usage !== "string" || !usage.endsWith("\n") || Buffer.byteLength(usage) > MAX_USAGE_LINE_BYTES) throw invalid(); + const parsed = rows(usage); + if (parsed.length !== 1 || parsed[0]!.v !== TURN_USAGE_LEDGER_VERSION || parsed[0]!.turn !== turnId) throw invalid(); + } + if (typeof requests !== "string" || (requests.length > 0 && (usage === null || !requests.endsWith("\n"))) || Buffer.byteLength(requests) > MAX_REQUEST_LINES_BYTES) throw invalid(); + if (rows(requests).some((row) => row.v !== TURN_REQUEST_LEDGER_VERSION || row.turn !== turnId)) throw invalid(); + } catch { throw invalid(); } + return { usage: usage as string | null, requests }; +} + +/** Appends sealed lines on the first metering: no presence scan is needed, nothing was appended before the record existed. */ +export async function appendBrokerTurnLedger(lines: BrokerTurnLedgerLines, paths: Readonly<{ usageLedgerPath: string; requestLedgerPath: string }>): Promise { + if (lines.usage !== null) await recordLedgerLines(paths.usageLedgerPath, lines.usage); + await recordLedgerLines(paths.requestLedgerPath, lines.requests); +} + +/** On replay: append each stream's sealed lines only when that stream (current file or its `.1`) holds no row for this turn. Never rejects. */ +export async function ensureBrokerTurnLedgered(lines: BrokerTurnLedgerLines, turnId: string, paths: Readonly<{ usageLedgerPath: string; requestLedgerPath: string }>): Promise { + try { + if (lines.usage !== null && !await ledgerHasTurn(paths.usageLedgerPath, turnId)) await recordLedgerLines(paths.usageLedgerPath, lines.usage); + if (lines.requests.length > 0 && !await ledgerHasTurn(paths.requestLedgerPath, turnId)) await recordLedgerLines(paths.requestLedgerPath, lines.requests); + } catch { /* advisory: a replay never fails on its ledger */ } +} + +async function ledgerHasTurn(file: string, turnId: string): Promise { + for (const candidate of [`${file}.1`, file]) { + let text: string; + try { text = await readFile(candidate, "utf8"); } catch { continue; } + for (const line of text.split("\n")) { + if (!line.includes(turnId)) continue; + try { if ((JSON.parse(line) as { turn?: unknown }).turn === turnId) return true; } catch { /* a torn line is not this turn's row */ } + } + } + return false; +} diff --git a/src/runtime/grokEngineBrokerMetering.ts b/src/runtime/grokEngineBrokerMetering.ts index 12c25a4..8b8fb31 100644 --- a/src/runtime/grokEngineBrokerMetering.ts +++ b/src/runtime/grokEngineBrokerMetering.ts @@ -1,7 +1,8 @@ import type { EngineBrokerRequest, EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; import type { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; -import { recordGrokTurnRequests, type GrokTurnRequest } from "./turnRequestLedger.js"; -import { recordTurnUsage, type TurnUsageFailureReason } from "./turnUsageLedger.js"; +import { appendBrokerTurnLedger, renderBrokerTurnLedger } from "./grokEngineBrokerLedger.js"; +import type { GrokTurnRequest } from "./turnRequestLedger.js"; +import type { TurnUsageFailureReason } from "./turnUsageLedger.js"; export type BrokerTurnMetering = Readonly<{ usageLedgerPath: string; @@ -14,30 +15,32 @@ export type BrokerTurnMeteringDetail = Readonly<{ notionalUsd: number; complete: /** * Seal a terminal turn, then meter it. The broker is the single writer. * - * Order is load-bearing. `turns.finish` publishes the durable terminal record - * *with* its accounting; only after that are the advisory ledger rows appended. - * A replayed turn returns before the broker's `try` block and never reaches - * here, so a crash-recovered or repeated turn cannot double-count; every row - * also carries the turn id as `turn`, so a reader that sees one twice counts - * it once. + * Order is load-bearing. The ledger bytes are rendered first and sealed into + * the durable terminal record together with its accounting + * (`grokEngineBrokerLedger.ts`); only after `turns.finish` published that + * record are the same bytes appended. A replayed turn returns before the + * broker's `try` block and never meters again — it only completes an append a + * crash interrupted (`ensureBrokerTurnLedgered`), and every row carries the + * turn id as `turn`, so a reader that sees one twice counts it once. + * + * Remaining window, documented rather than closed: a crash before the record's + * rename (while the turn is still `active`, including mid-turn) makes the next + * boot seal that turn `failed` with `usage: null`, so its spend is unmetered. + * Closing it needs the running proxy usage checkpointed into the active record + * on every request (an fsync'd rewrite per model request); not done here. * * Both terminal kinds meter: a failed turn spent real tokens, so its partial * usage is written with `outcome: failed` and its closed `limitReason`. A turn * with no usage at all (`usage: null`) writes nothing — a zero row is * byte-identical to a measured zero. * - * `recordTurnUsage`/`recordGrokTurnRequests` never reject, so an append failure - * cannot escape into the caller's `catch` and rewrite a completed turn as failed. + * Appends never reject, so an append failure cannot escape into the caller's + * `catch` and rewrite a completed turn as failed; the caller also refuses to + * re-seal a turn this helper already sealed. */ -export async function finishBrokerTurnWithUsage(turns: EngineBrokerTurnRegistry, request: Extract, terminal: EngineBrokerTerminalResponse, metering: BrokerTurnMetering, detail: BrokerTurnMeteringDetail): Promise { - await turns.finish(request, terminal); - if (terminal.usage === null) return; - const { usage } = terminal; - await recordTurnUsage(metering.usageLedgerPath, { - agent: metering.agentId, wake: metering.wakeId, engine: "grok", - usage: { input: usage.input, output: usage.output, cacheRead: usage.cacheRead, cacheWrite: usage.cacheWrite, total: usage.total, calls: terminal.requests, notionalUsd: detail.notionalUsd, complete: detail.complete }, - outcome: terminal.kind === "completed" ? { status: "completed" } : { status: "failed", reason: detail.reason ?? "unknown" }, - turn: terminal.turnId, limitReason: terminal.limitReason, model: terminal.model, estimatedRequests: detail.estimatedRequests - }); - await recordGrokTurnRequests(metering.requestLedgerPath, { agent: metering.agentId, wake: metering.wakeId, turn: terminal.turnId, model: terminal.model, requests: detail.requests, requestCount: terminal.requests, ...(detail.session === undefined ? {} : { session: detail.session }) }); +export async function finishBrokerTurnWithUsage(turns: EngineBrokerTurnRegistry, request: Extract, terminal: EngineBrokerTerminalResponse, metering: BrokerTurnMetering, detail: BrokerTurnMeteringDetail, onSealed: () => void = () => undefined): Promise { + const lines = renderBrokerTurnLedger(terminal, { ...detail, agentId: metering.agentId, wakeId: metering.wakeId }); + await turns.finish(request, terminal, lines); + onSealed(); + await appendBrokerTurnLedger(lines, metering); } diff --git a/src/runtime/grokEngineBrokerTurn.ts b/src/runtime/grokEngineBrokerTurn.ts index 650c94b..d1ae705 100644 --- a/src/runtime/grokEngineBrokerTurn.ts +++ b/src/runtime/grokEngineBrokerTurn.ts @@ -7,6 +7,7 @@ import { lowerEngineBrokerTurnLimits, mapGrokReportedModel, type EngineBrokerTur import { NativeBrokerTurnFailure, type NativeBrokerDiagnostic, type NativeBrokerTurn, type NativeBrokerTurnResult } from "./engineBrokerNativeClient.js"; import type { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; import { engineBrokerRequestLedgerPathFor, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +import { ensureBrokerTurnLedgered } from "./grokEngineBrokerLedger.js"; import { finishBrokerTurnWithUsage, type BrokerTurnMetering, type BrokerTurnMeteringDetail } from "./grokEngineBrokerMetering.js"; import type { GrokBrokerProxyTurn } from "./grokBrokerProxy.js"; import { GrokBrokerTurnMeter, type GrokBrokerTurnMeterSnapshot } from "./grokBrokerTurnMeter.js"; @@ -51,13 +52,14 @@ export async function runGrokEngineBrokerTurn(deps: GrokEngineBrokerTurnDependen const turnId = createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"); const request = { version: ENGINE_BROKER_VERSION, kind: "start_turn", requestId: randomUUID(), turnId, agentId, wakeId, prompt, mcpEndpoint, ...(overrides === undefined ? {} : { limits: overrides }) } as const; const begun = await deps.turns.begin(request, declared); - if (begun !== "start") return replay(begun.replay); + const metering: BrokerTurnMetering = { usageLedgerPath: registration.usageLedgerPath, requestLedgerPath: engineBrokerRequestLedgerPathFor(registration.usageLedgerPath), agentId, wakeId }; + if (begun !== "start") { await ensureBrokerTurnLedgered(begun.ledger, turnId, metering); return replay(begun.replay); } const controller = new AbortController(); const meter = new GrokBrokerTurnMeter(limits, () => controller.abort()); const onAbort = () => controller.abort(); signal?.addEventListener("abort", onAbort, { once: true }); if (signal?.aborted) controller.abort(); const timer = setTimeout(() => meter.trip("timeout"), limits.timeoutMs); timer.unref?.(); - const metering: BrokerTurnMetering = { usageLedgerPath: registration.usageLedgerPath, requestLedgerPath: engineBrokerRequestLedgerPathFor(registration.usageLedgerPath), agentId, wakeId }; - let nativeDiagnostic: NativeBrokerDiagnostic | undefined, attested = false, output: string | undefined, rejected = false; + + let nativeDiagnostic: NativeBrokerDiagnostic | undefined, attested = false, output: string | undefined, rejected = false, sealed: GrokEngineBrokerTurnResult | undefined; try { const isolationGuard = await deps.prepareIsolation(registration); deps.proxy.registerIsolationGuard(turnId, isolationGuard); @@ -76,8 +78,9 @@ export async function runGrokEngineBrokerTurn(deps: GrokEngineBrokerTurnDependen const usage = decoded.usage === undefined ? streamOrMeterUsage(stream, snapshot) : usageOf(decoded.usage); const accounting = { outcome: "completed", usage, model: declared, requests: requestCount(stream, snapshot), limitReason: "none" } as const; const completed = { version: request.version, kind: "completed", requestId: request.requestId, turnId, text: decoded.text, workerPid: result.workerPid, workerUid: result.workerUid, workerStartTime: result.startTicks.toString(), ...accounting } as const; - await finishBrokerTurnWithUsage(deps.turns, request, completed, metering, { notionalUsd: decoded.usage?.notionalUsd ?? 0, complete: decoded.usage?.complete ?? false, estimatedRequests: snapshot.estimatedRequests, requests: requestRows(stream, snapshot), ...(stream.sessionId === undefined ? {} : { session: stream.sessionId }) }); - return { text: completed.text, workerPid: completed.workerPid, workerUid: completed.workerUid, workerStartTime: completed.workerStartTime, ...accounting }; + const result_ = { text: completed.text, workerPid: completed.workerPid, workerUid: completed.workerUid, workerStartTime: completed.workerStartTime, ...accounting }; + await finishBrokerTurnWithUsage(deps.turns, request, completed, metering, { notionalUsd: decoded.usage?.notionalUsd ?? 0, complete: decoded.usage?.complete ?? false, estimatedRequests: snapshot.estimatedRequests, requests: requestRows(stream, snapshot), ...(stream.sessionId === undefined ? {} : { session: stream.sessionId }) }, () => { sealed = result_; }); + return result_; } catch (error) { const snapshot = meter.snapshot(); const code: EngineBrokerTurnFailure["code"] = snapshot.limitReason !== "none" ? "limit_exceeded" : deps.credentialStale() ? "auth_stale" : controller.signal.aborted ? "cancelled" : "engine_failed"; diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index 9dec696..ce36973 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; -import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; import { request as httpRequest } from "node:http"; import os from "node:os"; import path from "node:path"; @@ -159,6 +159,23 @@ test("a turn whose stream reports an undeclared model fails as rejected but is s }); }); +test("a crash between sealing and appending is completed by the replay exactly once, with the sealed bytes", async () => { + await withBroker(async ({ root, turn, usageRows, requestRows }) => { + await turn("wake-8", twoRequests); + const [sealedUsage] = await usageRows(); const sealedRequests = await requestRows(); + // Simulate the crash window: the record is published but the append never happened. + await rm(path.join(root, "usage.jsonl")); await rm(path.join(root, "requests.jsonl")); + // Mutation guard: a replay that never ensures its ledger leaves this spend unmetered. + assert.equal((await turn("wake-8", async () => { throw new Error("a replay runs no worker"); })).outcome, "completed"); + assert.deepEqual(await usageRows(), [sealedUsage]); + assert.deepEqual(await requestRows(), sealedRequests); + // A replay after the rows exist writes nothing further. + await turn("wake-8", async () => { throw new Error("a replay runs no worker"); }); + assert.equal((await usageRows()).length, 1); + assert.equal((await requestRows()).length, 2); + }); +}); + test("an unwritable ledger leaves the turn recorded as completed, not failed", async () => { await withBroker(async ({ root, turn }) => { assert.equal((await turn("wake-7", twoRequests)).outcome, "completed"); diff --git a/src/runtime/turnRequestLedger.ts b/src/runtime/turnRequestLedger.ts index 6cd2844..bb33992 100644 --- a/src/runtime/turnRequestLedger.ts +++ b/src/runtime/turnRequestLedger.ts @@ -150,6 +150,16 @@ export const renderGrokTurnRequestLines = (entry: GrokTurnRequestEntry): string })}\n`).join(""); }; +/** + * Append already-rendered, newline-terminated ledger lines in one write, with the + * same rotation and file mode as both ledgers. Advisory: never rejects. The + * broker uses it to append the exact bytes it sealed into a turn record. + */ +export const recordLedgerLines = async (file: string, lines: string): Promise => { + if (lines.length === 0) return false; + try { await rotate(file); await appendLines(file, lines); return true; } catch { return false; } +}; + /** Advisory and never rejects, like {@link recordTurnRequests}; an empty turn writes nothing. */ export const recordGrokTurnRequests = async (file: string, entry: GrokTurnRequestEntry): Promise => { if (entry.requests.length === 0) return false; From f1f95f01bef9900a5f4ad303181febb140e30836 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:42:13 +0200 Subject: [PATCH 043/124] fix: never re-seal a completed Grok turn when metering after the seal fails --- src/runtime/grokEngineBrokerTurn.ts | 4 ++++ src/runtime/grokEngineBrokerUsage.test.ts | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/runtime/grokEngineBrokerTurn.ts b/src/runtime/grokEngineBrokerTurn.ts index d1ae705..674b572 100644 --- a/src/runtime/grokEngineBrokerTurn.ts +++ b/src/runtime/grokEngineBrokerTurn.ts @@ -82,6 +82,10 @@ export async function runGrokEngineBrokerTurn(deps: GrokEngineBrokerTurnDependen await finishBrokerTurnWithUsage(deps.turns, request, completed, metering, { notionalUsd: decoded.usage?.notionalUsd ?? 0, complete: decoded.usage?.complete ?? false, estimatedRequests: snapshot.estimatedRequests, requests: requestRows(stream, snapshot), ...(stream.sessionId === undefined ? {} : { session: stream.sessionId }) }, () => { sealed = result_; }); return result_; } catch (error) { + // Once the completed record is published it is the durable truth: anything + // failing after that (metering) must neither re-seal the turn as failed nor + // append a second row. + if (sealed !== undefined) return sealed; const snapshot = meter.snapshot(); const code: EngineBrokerTurnFailure["code"] = snapshot.limitReason !== "none" ? "limit_exceeded" : deps.credentialStale() ? "auth_stale" : controller.signal.aborted ? "cancelled" : "engine_failed"; const diagnostic = error instanceof NativeBrokerTurnFailure ? error.diagnostic : nativeDiagnostic && !attested ? { ...nativeDiagnostic, status: "worker_failed" as const, stage: "attestation" as const, failureClass: error instanceof GrokWorkerAttestationFailure ? error.failureClass : "profile_invalid" as const, profileApplied: false } : undefined; diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index ce36973..2c3fbbc 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -176,6 +176,17 @@ test("a crash between sealing and appending is completed by the replay exactly o }); }); +test("a ledger append that fails after the turn was sealed leaves it completed and appends nothing twice", async () => { + await withBroker(async ({ root, turn, usageRows }) => { + // The request stream cannot be written (its path is a directory); the usage stream can. + await mkdir(path.join(root, "requests.jsonl")); + assert.equal((await turn("wake-9", twoRequests)).outcome, "completed"); + assert.deepEqual((await usageRows()).map((row) => [row.outcome, row.turn]), [["completed", turnIdFor("foreman", "wake-9")]]); + assert.equal((await turn("wake-9", twoRequests, undefined, undefined, path.join(root, "turns"))).outcome, "completed", "the sealed record was never rewritten as failed"); + assert.equal((await usageRows()).length, 1); + }); +}); + test("an unwritable ledger leaves the turn recorded as completed, not failed", async () => { await withBroker(async ({ root, turn }) => { assert.equal((await turn("wake-7", twoRequests)).outcome, "completed"); From c9dbf3c1a18a6c0f20c8973b79c0f13bfae5d767 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:42:25 +0200 Subject: [PATCH 044/124] test: refuse a slot preflight receipt carrying a canary the projection does not deny --- .../receipt.extra-canary.json | 42 +++++++++++++++++++ src/runtime/grokSlotPreflightReceipt.test.ts | 2 + 2 files changed, 44 insertions(+) create mode 100644 src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json new file mode 100644 index 0000000..719bdc8 --- /dev/null +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json @@ -0,0 +1,42 @@ +{ + "version": "noopolis.daimon.grok-slot-preflight.v1", + "slot": 0, + "worker_uid": 2200, + "projection_sha256": "b41a6952dfe3319f014866e7468df69ca2e77cfd28d781ac39cfdf0d3d587bd8", + "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", + "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", + "canaries": [ + { + "path": "/run/paideia", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/paideia/control", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/training/inputs", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-bootstrap-auth", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-subscription-realm", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/not-projected", + "method": "sandboxed-read", + "result": "denied" + } + ], + "created_at": "2026-09-17T12:00:00.000Z" +} diff --git a/src/runtime/grokSlotPreflightReceipt.test.ts b/src/runtime/grokSlotPreflightReceipt.test.ts index 77d80c1..d6fb4cd 100644 --- a/src/runtime/grokSlotPreflightReceipt.test.ts +++ b/src/runtime/grokSlotPreflightReceipt.test.ts @@ -38,6 +38,8 @@ test("a receipt for a different projection, slot, profile or deny set is refused // Mutation guard: dropping the digest comparison accepts this fixture. await assert.rejects(async () => verifyGrokSlotPreflightReceipt(await fixture("receipt.projection-mismatch.json"), projected), /projection_sha256/u); await assert.rejects(async () => verifyGrokSlotPreflightReceipt(await fixture("receipt.missing-canary.json"), projected), /canaries/u); + // Exact match, both halves: a canary for a path the projection does not deny is as wrong as a missing one. + await assert.rejects(async () => verifyGrokSlotPreflightReceipt(await fixture("receipt.extra-canary.json"), projected), /canaries/u); assert.throws(() => verifyGrokSlotPreflightReceipt(valid, { ...projected, limits: { ...projected.limits, maxTokens: 1 } }), /projection_sha256/u); assert.throws(() => verifyGrokSlotPreflightReceipt({ ...valid, sandbox_profile_sha256: "1".repeat(64) }, projected), /sandbox_profile_sha256/u); assert.throws(() => verifyGrokSlotPreflightReceipt({ ...valid, slot: 1 }, projected), /slot/u); From a5fd91bfd2b93d22e2f23d184dd1a87db7feee12 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:43:12 +0200 Subject: [PATCH 045/124] feat: bind slot preflight receipts to the projected seccomp profile and bubblewrap runtime --- .../grok-slot-preflight/projection-input.json | 34 ++++++++++++++++--- .../receipt.extra-canary.json | 3 +- .../receipt.missing-canary.json | 3 +- .../receipt.projection-mismatch.json | 1 + .../receipt.readable-canary.json | 3 +- .../receipt.unknown-member.json | 3 +- .../grok-slot-preflight/receipt.valid.v1.json | 3 +- src/runtime/grokBrokerProjection.test.ts | 6 ++-- src/runtime/grokBrokerProjection.ts | 17 ++++++++-- src/runtime/grokSlotPreflightReceipt.test.ts | 4 +++ src/runtime/grokSlotPreflightReceipt.ts | 7 +++- 11 files changed, 69 insertions(+), 15 deletions(-) diff --git a/src/runtime/fixtures/grok-slot-preflight/projection-input.json b/src/runtime/fixtures/grok-slot-preflight/projection-input.json index b75a102..303abd0 100644 --- a/src/runtime/fixtures/grok-slot-preflight/projection-input.json +++ b/src/runtime/fixtures/grok-slot-preflight/projection-input.json @@ -1,9 +1,27 @@ { "config": { "version": "noopolis.daimon.organization-runtime.v2", - "host": { "bindHost": "127.0.0.1", "port": 19700, "controlTokenEnv": "DAIMON_CONTROL_TOKEN" }, + "host": { + "bindHost": "127.0.0.1", + "port": 19700, + "controlTokenEnv": "DAIMON_CONTROL_TOKEN" + }, "agents": [ - { "id": "foreman", "name": "Foreman", "instructions": "Fixture agent.", "workspacePath": "/var/lib/spawnfile/instance/workspace/agents/foreman", "runtimeHomePath": "/var/lib/spawnfile/instance/homes/foreman", "schedule": { "kind": "disabled" }, "engine": { "kind": "grok", "model": "grok-4.6", "reasoningEffort": "low" } } + { + "id": "foreman", + "name": "Foreman", + "instructions": "Fixture agent.", + "workspacePath": "/var/lib/spawnfile/instance/workspace/agents/foreman", + "runtimeHomePath": "/var/lib/spawnfile/instance/homes/foreman", + "schedule": { + "kind": "disabled" + }, + "engine": { + "kind": "grok", + "model": "grok-4.6", + "reasoningEffort": "low" + } + } ] }, "agentId": "foreman", @@ -13,8 +31,16 @@ "workerHomePath": "/var/lib/daimon-workers/2200", "architecture": "arm64", "usageLedgerPath": "/run/daimon-slots/0/usage/usage.jsonl", - "limits": { "maxRequests": 24, "maxTokens": 400000, "timeoutMs": 480000 }, + "limits": { + "maxRequests": 24, + "maxTokens": 400000, + "timeoutMs": 480000 + }, "acceptanceStorePath": "/run/paideia/control", - "denyPaths": ["/run/paideia", "/run/training/inputs"] + "denyPaths": [ + "/run/paideia", + "/run/training/inputs" + ], + "seccompProfileSha256": "7777777777777777777777777777777777777777777777777777777777777777" } } diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json index 719bdc8..f694edb 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json @@ -2,9 +2,10 @@ "version": "noopolis.daimon.grok-slot-preflight.v1", "slot": 0, "worker_uid": 2200, - "projection_sha256": "b41a6952dfe3319f014866e7468df69ca2e77cfd28d781ac39cfdf0d3d587bd8", + "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "sandbox_runtime": "bubblewrap", "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", "canaries": [ { diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json index 6671c48..974b321 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json @@ -2,9 +2,10 @@ "version": "noopolis.daimon.grok-slot-preflight.v1", "slot": 0, "worker_uid": 2200, - "projection_sha256": "b41a6952dfe3319f014866e7468df69ca2e77cfd28d781ac39cfdf0d3d587bd8", + "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "sandbox_runtime": "bubblewrap", "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", "canaries": [ { diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.projection-mismatch.json b/src/runtime/fixtures/grok-slot-preflight/receipt.projection-mismatch.json index cdcb916..c8729e3 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.projection-mismatch.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.projection-mismatch.json @@ -5,6 +5,7 @@ "projection_sha256": "0000000000000000000000000000000000000000000000000000000000000000", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "sandbox_runtime": "bubblewrap", "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", "canaries": [ { diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json index 34d05ff..8f5d407 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json @@ -2,9 +2,10 @@ "version": "noopolis.daimon.grok-slot-preflight.v1", "slot": 0, "worker_uid": 2200, - "projection_sha256": "b41a6952dfe3319f014866e7468df69ca2e77cfd28d781ac39cfdf0d3d587bd8", + "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "sandbox_runtime": "bubblewrap", "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", "canaries": [ { diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json b/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json index c69658a..2e50e6a 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json @@ -2,9 +2,10 @@ "version": "noopolis.daimon.grok-slot-preflight.v1", "slot": 0, "worker_uid": 2200, - "projection_sha256": "b41a6952dfe3319f014866e7468df69ca2e77cfd28d781ac39cfdf0d3d587bd8", + "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "sandbox_runtime": "bubblewrap", "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", "canaries": [ { diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v1.json b/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v1.json index 46954bc..3ccf85c 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v1.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v1.json @@ -2,9 +2,10 @@ "version": "noopolis.daimon.grok-slot-preflight.v1", "slot": 0, "worker_uid": 2200, - "projection_sha256": "b41a6952dfe3319f014866e7468df69ca2e77cfd28d781ac39cfdf0d3d587bd8", + "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "sandbox_runtime": "bubblewrap", "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", "canaries": [ { diff --git a/src/runtime/grokBrokerProjection.test.ts b/src/runtime/grokBrokerProjection.test.ts index aa8689a..10794bb 100644 --- a/src/runtime/grokBrokerProjection.test.ts +++ b/src/runtime/grokBrokerProjection.test.ts @@ -11,7 +11,7 @@ const agent = (id: string, engine: Record) => ({ id, name: id, const config = { version: "noopolis.daimon.organization-runtime.v2", host: { bindHost: "127.0.0.1", port: 19700, controlTokenEnv: "UNIT_CONTROL_TOKEN" }, agents: [agent("foreman", { kind: "grok", model: "grok-4.6", reasoningEffort: "low" }), agent("peer", { kind: "codex" })] }; const options = { slot: 0, workerUid: 2_200, workerHomePath: "/var/lib/daimon-workers/2200", architecture: "arm64", usageLedgerPath: "/run/slots/0/usage/usage.jsonl", - limits: { maxRequests: 24, maxTokens: 400_000, timeoutMs: 480_000 }, acceptanceStorePath: "/run/paideia/control", denyPaths: ["/run/paideia", "/run/training/inputs"] } as const; + limits: { maxRequests: 24, maxTokens: 400_000, timeoutMs: 480_000 }, acceptanceStorePath: "/run/paideia/control", denyPaths: ["/run/paideia", "/run/training/inputs"], seccompProfileSha256: "7".repeat(64) } as const; test("the projection is Daimon's own renderers and collectors, fully declared and deterministic", () => { const projection = resolveOrganizationGrokBrokerProjection(config, "foreman", options); @@ -23,7 +23,8 @@ test("the projection is Daimon's own renderers and collectors, fully declared an workerConfigSha256: grokBrokerWorkerConfigSha256({ model: "grok-4.6", reasoningEffort: "low" }), systemPromptSha256: GROK_ENGINE_BROKER.worker.systemPromptSha256, grokCliVersion: "1.0.34", grokExecutableSha256: GROK_ENGINE_BROKER.grokCliArtifacts.arm64.sha256, nativeAbiVersion: GROK_ENGINE_BROKER.nativeAbiVersion, model: "grok-4.6", reasoningEffort: "low", limits: options.limits, usageLedgerPath: options.usageLedgerPath, - attestation: { platform: "linux/landlock", enforced: true, restrictNetwork: true, profileName: "daimon-strict", eventsPath: "/var/lib/daimon-workers/2200/.grok/sessions/sandbox-events.jsonl" } + seccompProfileSha256: "7".repeat(64), + attestation: { platform: "linux/landlock", enforced: true, restrictNetwork: true, profileName: "daimon-strict", sandboxRuntime: "bubblewrap", eventsPath: "/var/lib/daimon-workers/2200/.grok/sessions/sandbox-events.jsonl" } }); assert.equal(grokBrokerProjectionSha256(resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, denyPaths: [...options.denyPaths].reverse() })), grokBrokerProjectionSha256(projection)); assert.match(grokBrokerProjectionSha256(projection), /^[a-f0-9]{64}$/u); @@ -32,6 +33,7 @@ test("the projection is Daimon's own renderers and collectors, fully declared an test("the projection refuses undeclared models, non-Grok agents, and a profile digest it did not render", () => { assert.throws(() => resolveOrganizationGrokBrokerProjection({ ...config, agents: [agent("foreman", { kind: "grok" })] }, "foreman", options), /declared model/u); assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "peer", options), /known Grok agent/u); + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, seccompProfileSha256: "not-a-digest" }), /seccomp profile sha256/u); assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "missing", options), /known Grok agent/u); // Mutation guard: skipping the digest comparison accepts a weaker profile's digest. assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, profileSha256: grokWorkerSandboxProfileSha256([]) }), /profile digest mismatch/u); diff --git a/src/runtime/grokBrokerProjection.ts b/src/runtime/grokBrokerProjection.ts index ac1e5f6..4052e41 100644 --- a/src/runtime/grokBrokerProjection.ts +++ b/src/runtime/grokBrokerProjection.ts @@ -39,7 +39,9 @@ export type OrganizationGrokBrokerProjection = Readonly<{ reasoningEffort: GrokBrokerReasoningEffort; limits: EngineBrokerTurnLimits; usageLedgerPath: string; - attestation: Readonly<{ platform: "linux/landlock"; enforced: true; restrictNetwork: true; profileName: typeof GROK_WORKER_SANDBOX_PROFILE; eventsPath: string }>; + /** The container seccomp profile the worker must run under (the pinned default-plus-userns profile bubblewrap needs). */ + seccompProfileSha256: string; + attestation: Readonly<{ platform: "linux/landlock"; enforced: true; restrictNetwork: true; profileName: typeof GROK_WORKER_SANDBOX_PROFILE; sandboxRuntime: "bubblewrap"; eventsPath: string }>; }>; export type OrganizationGrokBrokerProjectionOptions = Readonly<{ @@ -55,6 +57,8 @@ export type OrganizationGrokBrokerProjectionOptions = Readonly<{ acceptanceStorePath: string; /** Evaluator and host-bind paths the deployment must keep from the worker (R4). */ denyPaths?: readonly string[]; + /** sha256 of the seccomp profile bytes the deployment runs the worker under. */ + seccompProfileSha256: string; /** When the caller already holds a rendered profile digest, it must equal Daimon's. */ profileSha256?: string; }>; @@ -62,6 +66,12 @@ export type OrganizationGrokBrokerProjectionOptions = Readonly<{ /** * Resolve the public Grok broker projection for one agent. * + * Paths are taken as given, never resolved: the caller (Spawnfile provisioning) + * must supply canonical, non-symlink paths — the fixed tmpfs/workspace roots it + * creates — and its provisioning must verify they are not symlinks before a + * slot is used; the broker's own attestation re-checks the worker home at + * every turn. + * * Deterministic and I/O-free on purpose: its digest * ({@link grokBrokerProjectionSha256}) is what the slot preflight receipt * binds, so the supervisor that writes the receipt and the evaluator that reads @@ -81,6 +91,7 @@ export function resolveOrganizationGrokBrokerProjection(config: unknown, agentId for (const [label, value] of [["workerHomePath", options.workerHomePath], ["acceptanceStorePath", options.acceptanceStorePath]] as const) { if (!path.posix.isAbsolute(value) || path.posix.normalize(value) !== value || value === "/" || value.endsWith("/")) throw new Error(`Grok broker projection requires a canonical absolute ${label}`); } + if (!/^[a-f0-9]{64}$/u.test(options.seccompProfileSha256)) throw new Error("Grok broker projection requires a seccomp profile sha256"); const model = agent.engine.model as GrokBrokerModel, reasoningEffort = agent.engine.reasoningEffort as GrokBrokerReasoningEffort; const denyPaths = [...new Set([...grokSandboxProtectedPaths(agent.id, parsed.agents, [options.acceptanceStorePath]), ...(options.denyPaths ?? [])])].sort(); renderGrokWorkerSandboxProfile(denyPaths); @@ -96,8 +107,8 @@ export function resolveOrganizationGrokBrokerProjection(config: unknown, agentId version: GROK_BROKER_PROJECTION_VERSION, agentId, workspacePath: agent.workspacePath, runtimeHomePath: agent.runtimeHomePath, workerUid: options.workerUid, slot: options.slot, profilePath, profileSha256, denyPaths, workerConfigSha256, systemPromptSha256: GROK_ENGINE_BROKER.worker.systemPromptSha256, grokCliVersion: GROK_ENGINE_BROKER.grokCliVersion, grokExecutableSha256: artifact.sha256, - nativeAbiVersion: GROK_ENGINE_BROKER.nativeAbiVersion, model, reasoningEffort, limits: parseEngineBrokerTurnLimits(options.limits), usageLedgerPath: options.usageLedgerPath, - attestation: { platform: "linux/landlock", enforced: true, restrictNetwork: true, profileName: GROK_WORKER_SANDBOX_PROFILE, eventsPath: grokWorkerEventsPathFor(profilePath) } + nativeAbiVersion: GROK_ENGINE_BROKER.nativeAbiVersion, model, reasoningEffort, limits: parseEngineBrokerTurnLimits(options.limits), usageLedgerPath: options.usageLedgerPath, seccompProfileSha256: options.seccompProfileSha256, + attestation: { platform: "linux/landlock", enforced: true, restrictNetwork: true, profileName: GROK_WORKER_SANDBOX_PROFILE, sandboxRuntime: "bubblewrap", eventsPath: grokWorkerEventsPathFor(profilePath) } }; // The registration this projection implies must itself be a valid v2 service.json entry. grokBrokerServiceRegistrationFor(projection); diff --git a/src/runtime/grokSlotPreflightReceipt.test.ts b/src/runtime/grokSlotPreflightReceipt.test.ts index d6fb4cd..b141060 100644 --- a/src/runtime/grokSlotPreflightReceipt.test.ts +++ b/src/runtime/grokSlotPreflightReceipt.test.ts @@ -43,4 +43,8 @@ test("a receipt for a different projection, slot, profile or deny set is refused assert.throws(() => verifyGrokSlotPreflightReceipt(valid, { ...projected, limits: { ...projected.limits, maxTokens: 1 } }), /projection_sha256/u); assert.throws(() => verifyGrokSlotPreflightReceipt({ ...valid, sandbox_profile_sha256: "1".repeat(64) }, projected), /sandbox_profile_sha256/u); assert.throws(() => verifyGrokSlotPreflightReceipt({ ...valid, slot: 1 }, projected), /slot/u); + // Mutation guard: never comparing the seccomp digest accepts a receipt taken under another profile. + assert.throws(() => verifyGrokSlotPreflightReceipt({ ...valid, seccomp_profile_sha256: "8".repeat(64) }, projected), /seccomp_profile_sha256/u); + assert.throws(() => parseGrokSlotPreflightReceipt({ ...valid, sandbox_runtime: "none" }), /invalid Grok slot preflight receipt/u); + assert.throws(() => parseGrokSlotPreflightReceipt((({ sandbox_runtime: _omit, ...rest }) => rest)(valid)), /invalid Grok slot preflight receipt/u); }); diff --git a/src/runtime/grokSlotPreflightReceipt.ts b/src/runtime/grokSlotPreflightReceipt.ts index 3bed48d..7658581 100644 --- a/src/runtime/grokSlotPreflightReceipt.ts +++ b/src/runtime/grokSlotPreflightReceipt.ts @@ -38,6 +38,8 @@ export const grokSlotPreflightReceiptSchema = z.strictObject({ sandbox_profile_sha256: sha256, /** The container seccomp profile the worker ran under. */ seccomp_profile_sha256: sha256, + /** Grok 1.0.34 runs every profile inside bubblewrap; the supervisor observed it present and working. */ + sandbox_runtime: z.literal("bubblewrap"), grok_executable_sha256: sha256, canaries: z.array(grokSlotPreflightCanarySchema).min(1).max(256), created_at: z.string().regex(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/u).refine((value) => !Number.isNaN(Date.parse(value)) && new Date(value).toISOString() === value, "exact RFC3339 timestamp") @@ -58,7 +60,8 @@ export function parseGrokSlotPreflightReceipt(value: unknown): GrokSlotPreflight /** * Parse a receipt and require that it proves *this* projection's slot: same * digest, slot, worker uid, profile and executable, and a denied canary for - * exactly every projected deny path (no more, no fewer). + * exactly every projected deny path (no more, no fewer), under the projected + * seccomp profile and sandbox runtime. */ export function verifyGrokSlotPreflightReceipt(value: unknown, projection: OrganizationGrokBrokerProjection): GrokSlotPreflightReceipt { const receipt = parseGrokSlotPreflightReceipt(value); @@ -68,6 +71,8 @@ export function verifyGrokSlotPreflightReceipt(value: unknown, projection: Organ if (receipt.worker_uid !== projection.workerUid) mismatch("worker_uid"); if (receipt.sandbox_profile_sha256 !== projection.profileSha256) mismatch("sandbox_profile_sha256"); if (receipt.grok_executable_sha256 !== projection.grokExecutableSha256) mismatch("grok_executable_sha256"); + if (receipt.seccomp_profile_sha256 !== projection.seccompProfileSha256) mismatch("seccomp_profile_sha256"); + if (receipt.sandbox_runtime !== projection.attestation.sandboxRuntime) mismatch("sandbox_runtime"); const denied = receipt.canaries.map((canary) => canary.path).sort(); if (denied.length !== projection.denyPaths.length || denied.some((entry, index) => entry !== projection.denyPaths[index])) mismatch("canaries"); return receipt; From 3a1fac813d879012a21460e2933fd68d8eec8cc6 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:43:12 +0200 Subject: [PATCH 046/124] docs: record the Grok in-flight gate, usage estimates, sealed ledger bytes and projection path contract --- src/runtime/AGENTS.md | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index ca8eb55..472352f 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -37,9 +37,14 @@ upstream-reported running total (prompt tokens *including* cached, plus completion) has reached `maxTokens` — HTTP 429, and the tripped limit aborts the worker through the ordinary cancel/kill path. The token ceiling is checked between requests, so a turn overshoots it by at most the last admitted -request; if an upstream body carries no `usage`, only `maxRequests` and -`timeoutMs` bound that turn mid-flight. A broker timer also trips `timeout` -for a worker that is mid-request. Limits come from `service.json` v2 +request. That bound holds only because a turn has at most one upstream request +in flight: an overlapping request is refused (429, uncounted), and Grok's loop +is sequential in every live capture. A per-request usage block above +`turnLimits.requestUsageMaxTokens` (500k) is invalid, and a response without +valid usage is charged `ceil(bodyBytes/2) + 4096` tokens (rows say +`usage_source: "estimated"`, usage rows `estimated_requests`), so a missing +`usage` never disables the ceiling. A broker timer also trips `timeout` for a +worker that is mid-request, and any trip aborts the in-flight upstream call. Limits come from `service.json` v2 (`engineBrokerServiceConfig.ts`; v1 gets `GROK_ENGINE_BROKER.turnLimits.v1Defaults`) and a wake may only lower them: a raise is refused as `invalid_request`, never clamped. @@ -49,9 +54,13 @@ seals every terminal turn — completed, failed, limit, cancelled — through `finishBrokerTurnWithUsage` (`grokEngineBrokerMetering.ts`): the turn registry record v2 stores the control-protocol v2 terminal response *with* its numeric-only accounting (`usage`, `outcome`, declared `model`, `requests`, -closed `limitReason`), and only then are ledger rows appended. A replay -returns the sealed accounting and never meters again; v1 records still replay -(upgraded with `usage: null`). Completed usage is the terminal `result.usage`; +closed `limitReason`) *and the exact ledger bytes it owes*, and only then are +those bytes appended. A replay returns the sealed accounting and never meters +again; it only appends the sealed bytes when the ledger has no row for that +`turn` (a crash between seal and append). The window not closed: a crash +before the record's rename seals the turn `failed` with `usage: null` on the +next boot. Once a completed record is sealed, nothing after it can re-seal the +turn as failed. v1 records still replay (upgraded with `usage: null`). Completed usage is the terminal `result.usage`; a failed turn's partial usage is its per-request stream frames (`../pi/grokStreamUsage.ts`) when output arrived, else the upstream usage the proxy saw. Usage rows carry `turn` (the idempotency key readers dedupe on — @@ -67,7 +76,10 @@ Grok agent's slot (`noopolis.daimon.grok-broker-projection.v1`): Daimon's own deny collectors plus the caller's evaluator paths, profile/config/prompt digests, pinned executable, model, limits and ledger. A Grok agent must declare `model` and `reasoningEffort` for it; nothing is defaulted, and a supplied -profile digest that differs is refused. `grokSlotPreflightReceipt.ts` is the +profile digest that differs is refused. Paths are never resolved: Spawnfile +must supply canonical non-symlink paths (its fixed tmpfs and workspace roots) +and verify that during provisioning. The projection also carries the seccomp +profile digest and the `bubblewrap` sandbox runtime a receipt must match. `grokSlotPreflightReceipt.ts` is the zod schema a root slot supervisor's receipt must satisfy (`noopolis.daimon.grok-slot-preflight.v1`, fixtures under `fixtures/grok-slot-preflight/`); `verifyGrokSlotPreflightReceipt` binds it to From 97784b40d82e367a74d32ac73adb8f88f99c4539 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:51:23 +0200 Subject: [PATCH 047/124] fix: treat a Grok turn record as sealed once renamed and report a failed directory sync instead of rejecting --- src/runtime/engineBrokerTurnRegistry.ts | 21 ++++++++++++++++----- src/runtime/grokEngineBrokerUsage.test.ts | 23 ++++++++++++++++++++--- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/src/runtime/engineBrokerTurnRegistry.ts b/src/runtime/engineBrokerTurnRegistry.ts index d59bab7..43ff351 100644 --- a/src/runtime/engineBrokerTurnRegistry.ts +++ b/src/runtime/engineBrokerTurnRegistry.ts @@ -23,12 +23,13 @@ type Observed = Readonly<{ version: typeof ENGINE_BROKER_TURN_RECORD_V1 | typeof * only on the path that returned `"start"`. */ export class EngineBrokerTurnRegistry { - constructor(private readonly root: string,private readonly bootId:string=randomUUID()) {} + /** `syncDirectoryOf` is injectable only so the post-publish failure path can be exercised under test. */ + constructor(private readonly root: string,private readonly bootId:string=randomUUID(),private readonly syncDirectoryOf:(directory:string)=>Promise=syncDirectory) {} /** `model` is the registration's declared model, used only to upgrade a v1 record on replay. */ async begin(request: Start, model: GrokBrokerModel): Promise<"start" | { replay: Terminal; ledger: BrokerTurnLedgerLines }> { await mkdir(this.root, { recursive: true, mode: 0o700 }); const file = path.join(this.root, safe(request.turnId)); const expected = digest(request); - try { const handle = await open(file, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); try { await handle.writeFile(`${JSON.stringify({ version: ENGINE_BROKER_TURN_RECORD_V2, digest: expected, state: "active",bootId:this.bootId })}\n`); await handle.sync(); } finally { await handle.close(); } await syncDirectory(this.root); return "start"; } + try { const handle = await open(file, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); try { await handle.writeFile(`${JSON.stringify({ version: ENGINE_BROKER_TURN_RECORD_V2, digest: expected, state: "active",bootId:this.bootId })}\n`); await handle.sync(); } finally { await handle.close(); } await this.syncDirectoryOf(this.root); return "start"; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw new Error("broker turn registry unavailable"); } const observed = parseEngineBrokerTurnRecord(await readFile(file, "utf8"), model); if (observed.digest !== expected) throw new Error("broker turn conflict"); @@ -36,12 +37,22 @@ export class EngineBrokerTurnRegistry { if(observed.state==="active"&&observed.bootId!==this.bootId){const response={version:request.version,kind:"failed",requestId:request.requestId,turnId:request.turnId,code:"engine_failed",outcome:"failed",usage:null,model,requests:0,limitReason:"none"} as const;await this.finish(request,response);return {replay:response,ledger:EMPTY_BROKER_TURN_LEDGER};} throw new Error("broker turn already active"); } - /** `ledger` is the exact ledger bytes this turn owes, sealed with it so a replay can finish an interrupted append. */ - async finish(request: Start, response: Terminal, ledger: BrokerTurnLedgerLines = EMPTY_BROKER_TURN_LEDGER): Promise { + /** + * `ledger` is the exact ledger bytes this turn owes, sealed with it so a replay can finish an interrupted append. + * + * The rename is the publish point. A failure before it rejects (nothing was + * published); a failure after it — the directory fsync — must not, because + * the terminal record is already the visible truth and a caller that saw a + * rejection would believe the turn unsealed and write a contradicting + * record over it. That durability gap is reported as `directorySynced: false` + * instead: the record is published but may not survive a power loss. + */ + async finish(request: Start, response: Terminal, ledger: BrokerTurnLedgerLines = EMPTY_BROKER_TURN_LEDGER): Promise> { const file = path.join(this.root, safe(request.turnId)); const temporary = `${file}.${randomUUID()}.tmp`; const handle = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); try { await handle.writeFile(`${JSON.stringify({ version: ENGINE_BROKER_TURN_RECORD_V2, digest: digest(request), state: "terminal",bootId:this.bootId, response, ledger })}\n`); await handle.sync(); } finally { await handle.close(); } - try { await rename(temporary, file); await syncDirectory(this.root); } finally { await unlink(temporary).catch(() => undefined); } + try { await rename(temporary, file); } finally { await unlink(temporary).catch(() => undefined); } + try { await this.syncDirectoryOf(this.root); return { directorySynced: true }; } catch { return { directorySynced: false }; } } } diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index 2c3fbbc..b850069 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -46,7 +46,7 @@ const untilAborted = (signal: AbortSignal): Promise => new Promise((_reso * talks to the proxy exactly as the native worker does (capability bearer, * pinned client version, lean body). Only the launcher and attestation are fakes. */ -const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId: string, worker: Worker, overrides?: Parameters[6], limits?: EngineBrokerServiceRegistration["limits"], turnStore?: string) => ReturnType; usageRows: () => Promise[]>; requestRows: () => Promise[]>; upstreamCalls: () => number; upstreamAborts: () => number }>) => Promise, usageLedgerPath?: string, upstreamDelayMs: (call: number) => number = () => 15): Promise => { +const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId: string, worker: Worker, overrides?: Parameters[6], limits?: EngineBrokerServiceRegistration["limits"], turnStore?: string, syncDirectory?: (directory: string) => Promise) => ReturnType; usageRows: () => Promise[]>; requestRows: () => Promise[]>; upstreamCalls: () => number; upstreamAborts: () => number }>) => Promise, usageLedgerPath?: string, upstreamDelayMs: (call: number) => number = () => 15): Promise => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-usage-")); let calls = 0, aborted = 0; const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async (_request, signal) => { calls++; await new Promise((resolve, reject) => { const timer = setTimeout(resolve, upstreamDelayMs(calls)); signal?.addEventListener("abort", () => { clearTimeout(timer); aborted++; reject(new Error("aborted")); }, { once: true }); }); return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(`data: ${JSON.stringify({ choices: [], usage: upstreamUsage })}\n\ndata: [DONE]\n\n`) }; }, undefined, 0); @@ -55,10 +55,10 @@ const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId try { await body({ root, - turn: (wakeId, worker, overrides, limits = { maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }, turnStore = path.join(root, "turns")) => { + turn: (wakeId, worker, overrides, limits = { maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }, turnStore = path.join(root, "turns"), syncDirectory = undefined) => { const registration: EngineBrokerServiceRegistration = { agentId: "foreman", slot: 0, workerUid: 2_200, workspace: "/workspace", profilePath: "/workers/0/.grok/sandbox.toml", eventsPath: "/workers/0/.grok/sessions/sandbox-events.jsonl", profileSha256: "a".repeat(64), usageLedgerPath: ledger, limits, model: { model: "grok-4.6", reasoningEffort: "low" } }; const deps: GrokEngineBrokerTurnDependencies = { - turns: new EngineBrokerTurnRegistry(turnStore), proxy, credentialStale: () => false, + turns: syncDirectory === undefined ? new EngineBrokerTurnRegistry(turnStore) : new EngineBrokerTurnRegistry(turnStore, undefined, syncDirectory), proxy, credentialStale: () => false, mcp: { register: () => "mcp-capability-0123456789abcdef", revoke: () => undefined }, prepareIsolation: async () => async () => undefined, runNative: async (input: NativeBrokerTurn, signal: AbortSignal) => nativeResult(await worker(() => post(proxy.port, input.providerCapability), signal)) @@ -187,6 +187,23 @@ test("a ledger append that fails after the turn was sealed leaves it completed a }); }); +test("a directory-sync failure after the completed record is published never re-seals the turn as failed", async () => { + await withBroker(async ({ root, turn, usageRows, requestRows }) => { + let syncs = 0; + // Sync 1 is begin()'s active record; sync 2 follows the completed record's rename. + const failAfterPublish = async (): Promise => { syncs += 1; if (syncs === 2) throw new Error("EIO"); }; + // Mutation guard: letting the post-rename failure reject makes the turn's catch write `failed` over the published record. + const result = await turn("wake-10", twoRequests, undefined, undefined, path.join(root, "turns"), failAfterPublish); + assert.equal(result.outcome, "completed"); + assert.equal(syncs, 2); + assert.deepEqual((await usageRows()).map((row) => [row.outcome, row.turn]), [["completed", turnIdFor("foreman", "wake-10")]]); + assert.equal((await requestRows()).length, 2); + const replayed = await turn("wake-10", async () => { throw new Error("a replay runs no worker"); }); + assert.deepEqual(replayed, result); + assert.equal((await usageRows()).length, 1); + }); +}); + test("an unwritable ledger leaves the turn recorded as completed, not failed", async () => { await withBroker(async ({ root, turn }) => { assert.equal((await turn("wake-7", twoRequests)).outcome, "completed"); From 3ab4e4a7d513094cbf6ffd72bd1b0f0ea1d45a87 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:51:23 +0200 Subject: [PATCH 048/124] test: prove concurrent replays of one Grok turn are counted once and require readers to dedupe on turn --- src/runtime/AGENTS.md | 8 ++++++- src/runtime/grokEngineBrokerUsage.test.ts | 29 +++++++++++++++++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 472352f..182a3fa 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -57,7 +57,13 @@ numeric-only accounting (`usage`, `outcome`, declared `model`, `requests`, closed `limitReason`) *and the exact ledger bytes it owes*, and only then are those bytes appended. A replay returns the sealed accounting and never meters again; it only appends the sealed bytes when the ledger has no row for that -`turn` (a crash between seal and append). The window not closed: a crash +`turn` (a crash between seal and append). Two replays of one sealed turn in +the same broker may both append those identical bytes (a second broker cannot +exist: the realm lease is an exclusive lock), so **every ledger consumer — +`wakeFuse.ts`, Spawnfile's reader (P3), Paideia's evidence reader (P4) — MUST +dedupe usage rows by `turn`** (`dedupeTurnUsageRows`). The turn record's rename +is its publish point: a directory-sync failure after it is reported, never +raised, so a published completed turn is never re-sealed as failed. The window not closed: a crash before the record's rename seals the turn `failed` with `usage: null` on the next boot. Once a completed record is sealed, nothing after it can re-seal the turn as failed. v1 records still replay (upgraded with `usage: null`). Completed usage is the terminal `result.usage`; diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index b850069..9964bd9 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; -import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { request as httpRequest } from "node:http"; import os from "node:os"; import path from "node:path"; @@ -13,7 +13,8 @@ import { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; import { EngineBrokerTurnFailure, runGrokEngineBrokerTurn, type GrokEngineBrokerTurnDependencies } from "./grokEngineBrokerTurn.js"; import { TURN_REQUEST_LEDGER_VERSION } from "./turnRequestLedger.js"; -import { TURN_USAGE_LEDGER_VERSION } from "./turnUsageLedger.js"; +import { dedupeTurnUsageRows, TURN_USAGE_LEDGER_VERSION } from "./turnUsageLedger.js"; +import { WakeFuse } from "./wakeFuse.js"; const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); const leanBody = JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: lean }); @@ -204,6 +205,30 @@ test("a directory-sync failure after the completed record is published never re- }); }); +test("two concurrent replays of one sealed turn may both append, and every reader still counts the turn once", async () => { + await withBroker(async ({ root, turn, usageRows }) => { + await turn("wake-11", twoRequests); + const [sealed] = await usageRows(); + await rm(path.join(root, "usage.jsonl")); await rm(path.join(root, "requests.jsonl")); + const noWorker = async (): Promise => { throw new Error("a replay runs no worker"); }; + const replays = await Promise.all([turn("wake-11", noWorker), turn("wake-11", noWorker), turn("wake-11", noWorker)]); + assert.ok(replays.every((replayed) => replayed.outcome === "completed")); + const rows = await usageRows(); + assert.ok(rows.length >= 1 && rows.every((row) => row.turn === sealed!.turn && row.total === sealed!.total), "duplicates, if any, are byte-equal sealed rows"); + // Readers dedupe on `turn`: the ledger helper and the wake fuse's sum both count it once. + assert.deepEqual(dedupeTurnUsageRows(rows).map((row) => row.total), [sealed!.total]); + const fuseDirectory = path.join(root, "fuse"); await mkdir(fuseDirectory); + const fuse = await WakeFuse.open({ organizationKey: "org", now: () => new Date(Date.parse(String(sealed!.at)) - 1), environment: { + DAIMON_WAKE_FUSE_DIRECTORY: fuseDirectory, DAIMON_WAKE_FUSE_EPOCH: "replay", DAIMON_WAKE_FUSE_MAX_WAKES: "10", + DAIMON_WAKE_FUSE_MAX_TOKENS: String(Number(sealed!.total) + 1), DAIMON_TURN_USAGE_LEDGER_PATH: path.join(root, "usage.jsonl") + } }); + // Counted once the turn is below the ceiling by one token; counted twice it would trip. + const concurrentRows = [...rows, ...rows]; + await writeFile(path.join(root, "usage.jsonl"), concurrentRows.map((row) => JSON.stringify(row)).join("\n") + "\n"); + assert.deepEqual(await fuse.admit("foreman", "next"), { state: "admitted" }); + }); +}); + test("an unwritable ledger leaves the turn recorded as completed, not failed", async () => { await withBroker(async ({ root, turn }) => { assert.equal((await turn("wake-7", twoRequests)).outcome, "completed"); From 3174716be9b3f5a5005e57f070aaf3727852d791 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 06:51:37 +0200 Subject: [PATCH 049/124] fix: charge the estimate when a Grok response's final usage block is invalid --- src/runtime/grokBrokerTurnMeter.test.ts | 3 +++ src/runtime/grokBrokerTurnMeter.ts | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/runtime/grokBrokerTurnMeter.test.ts b/src/runtime/grokBrokerTurnMeter.test.ts index f64911c..0f28a88 100644 --- a/src/runtime/grokBrokerTurnMeter.test.ts +++ b/src/runtime/grokBrokerTurnMeter.test.ts @@ -103,6 +103,9 @@ test("upstream usage parsing takes the last usage block and never zero-fills", ( assert.equal(parseGrokUpstreamUsage(sse({ prompt_tokens: "4", completion_tokens: 1 }), "text/event-stream"), undefined); assert.equal(parseGrokUpstreamUsage(sse({ prompt_tokens: 4, completion_tokens: 1, prompt_tokens_details: { cached_tokens: 9 } }), "text/event-stream"), undefined); assert.equal(parseGrokUpstreamUsage(Buffer.from("not json"), "application/json"), undefined); + // Mutation guard: keeping the earlier valid block under-reports a request whose final usage is implausible. + const twoBlocks = Buffer.from([{ choices: [], usage: { prompt_tokens: 5, completion_tokens: 1 } }, { choices: [], usage: { prompt_tokens: 900_000, completion_tokens: 1 } }].map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("")); + assert.equal(parseGrokUpstreamUsage(twoBlocks, "text/event-stream"), undefined); }); test("at most one upstream request is in flight per turn: an overlapping request is refused, uncounted", async () => { diff --git a/src/runtime/grokBrokerTurnMeter.ts b/src/runtime/grokBrokerTurnMeter.ts index d5b03f4..3c14b7c 100644 --- a/src/runtime/grokBrokerTurnMeter.ts +++ b/src/runtime/grokBrokerTurnMeter.ts @@ -133,8 +133,10 @@ export function parseGrokUpstreamUsage(body: Uint8Array, contentType: string | u let found: EngineBrokerTurnUsage | undefined; for (const candidate of candidates) { if (!isRecord(candidate) || !isRecord(candidate.usage)) continue; - const decoded = decodeOpenAiUsage(candidate.usage); - if (decoded !== undefined) found = decoded; + // Last usage block wins even when invalid: an implausible final report + // must not fall back to an earlier, smaller block (the request is then + // charged the estimate instead). + found = decodeOpenAiUsage(candidate.usage); } return found; } From 3dc64e5416b18dec6229d6d60b6465b09effe915 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:00:51 +0200 Subject: [PATCH 050/124] feat: add evaluator inference grants as a distinct metered proxy kind with a judge-shaped body gate --- src/contracts/runtimeContractManifest.ts | 30 +++++ src/runtime/grokBrokerProxy.ts | 20 +++- src/runtime/grokInferenceGrants.test.ts | 94 ++++++++++++++++ src/runtime/grokInferenceGrants.ts | 114 +++++++++++++++++++ src/runtime/grokInferenceProxy.test.ts | 134 +++++++++++++++++++++++ src/runtime/grokInferenceProxy.ts | 65 +++++++++++ src/runtime/grokInferenceProxyRequest.ts | 54 +++++++++ src/runtime/inferenceUsageLedger.ts | 63 +++++++++++ 8 files changed, 569 insertions(+), 5 deletions(-) create mode 100644 src/runtime/grokInferenceGrants.test.ts create mode 100644 src/runtime/grokInferenceGrants.ts create mode 100644 src/runtime/grokInferenceProxy.test.ts create mode 100644 src/runtime/grokInferenceProxy.ts create mode 100644 src/runtime/grokInferenceProxyRequest.ts create mode 100644 src/runtime/inferenceUsageLedger.ts diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 8aebfb1..3738843 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -87,6 +87,36 @@ export const GROK_ENGINE_BROKER = { missingUsageEstimate: { inputBytesPerToken: 2, outputTokens: 4_096 } }, wakeLimitEnvironment: { timeoutMs: "DAIMON_ENGINE_WAKE_TIMEOUT_MS", maxTokens: "DAIMON_ENGINE_WAKE_TOKEN_CEILING" }, + // Evaluator inference grants (P2c). Judges and the optimizer (organization uid + // only, over the control socket) borrow the broker's Grok credential through + // the provider proxy; they never hold it, and their spend never reaches the + // subject usage ledger or wake fuse. + inferenceGrants: { + requestKinds: ["request_inference_grant", "release_inference_grant"], + purposes: ["judge", "optimizer"], + tokenPrefix: "inference_", + ttlMs: 600_000, + limits: { maxRequests: 64, maxTokens: 2_000_000 }, + maxLiveGrants: 8, + maxInFlightRequestsPerGrant: 1, + // Top-level request members Grok 1.0.34 sends for a Paideia judge/optimizer call + // (live stub capture); `tools` and `tool_choice` are refused outright. + bodyMembers: ["messages", "model", "reasoning_effort", "response_format", "stream", "stream_options"], + messageRoles: ["system", "user", "assistant"], + failureCodes: ["auth_stale", "grant_limit", "invalid_request", "unavailable"], + ledgerVersion: "noopolis.daimon.inference-usage.v1", + ledgerDedupeKey: ["grant", "request"], + client: { + modelId: "daimon-inference-grok", + envKey: "DAIMON_INFERENCE_GRANT", + // sha256 of `renderGrokInferenceClientConfig` for the production proxy base URL and this env key. + configSha256: { + "grok-4.6": { low: "", medium: "", high: "" }, + "grok-4.5": { low: "", medium: "", high: "" }, + "grok-build": { low: "", medium: "", high: "" } + } + } + }, projectionVersion: "noopolis.daimon.grok-broker-projection.v1", slotPreflightVersion: "noopolis.daimon.grok-slot-preflight.v1", artifacts: { diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index 187bf7a..4d1e6d0 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -5,11 +5,14 @@ import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; import { DEFAULT_GROK_BROKER_MODEL_POLICY, parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; import { authorizeGrokBrokerProxyRequest } from "./grokBrokerProxyRequest.js"; import { parseGrokUpstreamUsage, type GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import { serveGrokInferenceGrant } from "./grokInferenceProxy.js"; +import type { GrokInferenceGrants } from "./grokInferenceGrants.js"; /** One running turn as the proxy sees it: its declared model/effort and its spend gate. */ export type GrokBrokerProxyTurn = Readonly<{ policy: GrokBrokerModelPolicy; meter: GrokBrokerTurnMeter }>; -export type GrokBrokerCredentialAuthority = Readonly<{ accessToken(forceRefresh: boolean): Promise; refreshAfterRejection?(rejectedTokenDigest:string):Promise; markRejected(rejectedTokenDigest?:string): Promise }>; +export type GrokBrokerCredentialAuthority = Readonly<{ accessToken(forceRefresh: boolean): Promise; refreshAfterRejection?(rejectedTokenDigest:string):Promise; markRejected(rejectedTokenDigest?:string): Promise; isStale?(): boolean }>; export type GrokBrokerUpstream = (request: ReturnType, signal?: AbortSignal) => Promise>; body: Uint8Array }>>; /** @@ -17,20 +20,27 @@ export type GrokBrokerUpstream = (request: ReturnTypePromise):void; revokeIsolationGuard(turnId:string):void; registerTurn(turnId:string,turn:GrokBrokerProxyTurn):void; revokeTurn(turnId:string):void; close(): Promise }>> { +export async function startGrokBrokerProxy(authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream = defaultUpstream, policy: GrokBrokerModelPolicy = DEFAULT_GROK_BROKER_MODEL_POLICY, listenPort = 43_123, grants?: GrokInferenceGrants): PromisePromise):void; revokeIsolationGuard(turnId:string):void; registerTurn(turnId:string,turn:GrokBrokerProxyTurn):void; revokeTurn(turnId:string):void; close(): Promise }>> { const declared = parseGrokBrokerModelPolicy(policy); const capabilities = new EngineBrokerCapabilities(); - const guards=new MapPromise>();const turns=new Map();const server = createServer((request, response) => { void serve(request, response, authority, upstream, capabilities,guards,turns,declared); }); + const guards=new MapPromise>();const turns=new Map();const server = createServer((request, response) => { void serve(request, response, authority, upstream, capabilities,guards,turns,declared,grants); }); await new Promise((resolve, reject) => { server.once("error", reject); server.listen(listenPort, "127.0.0.1", () => { server.off("error", reject); resolve(); }); }); const address = server.address() as AddressInfo; return { port: address.port, capabilities,registerIsolationGuard(turnId,guard){guards.set(turnId,guard);},revokeIsolationGuard(turnId){guards.delete(turnId);},registerTurn(turnId,turn){turns.set(turnId,{policy:parseGrokBrokerModelPolicy(turn.policy),meter:turn.meter});},revokeTurn(turnId){turns.delete(turnId);}, close: () => new Promise((resolve, reject) => server.close((error) => error === undefined ? resolve() : reject(error))) }; } -async function serve(request: IncomingMessage, response: ServerResponse, authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream, capabilities: EngineBrokerCapabilities,guards:MapPromise>,turns:Map,fallback:GrokBrokerModelPolicy): Promise { +async function serve(request: IncomingMessage, response: ServerResponse, authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream, capabilities: EngineBrokerCapabilities,guards:MapPromise>,turns:Map,fallback:GrokBrokerModelPolicy,grants?:GrokInferenceGrants): Promise { let settle:((usage:ReturnType)=>void)|undefined; try { const body = await readBody(request); const headers = Object.fromEntries(Object.entries(request.headers).map(([key, value]) => [key, Array.isArray(value) ? value[0] : value])); - const match=headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u),scope=match?capabilities.inspectToken(match[1]!):undefined;if(!scope)throw new Error();const guard=guards.get(scope.turnId),turn=turns.get(scope.turnId);if(!guard||!turn)throw new Error();await guard(); + const match=headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u); + if(match&&match[1]!.startsWith(GROK_ENGINE_BROKER.inferenceGrants.tokenPrefix)){if(!grants)throw new Error();return await serveGrokInferenceGrant({method:request.method??"",pathname:new URL(request.url??"/","http://127.0.0.1").pathname,headers,body,token:match[1]!},response,grants,authority,upstream);} + const scope=match?capabilities.inspectToken(match[1]!):undefined;if(!scope)throw new Error();const guard=guards.get(scope.turnId),turn=turns.get(scope.turnId);if(!guard||!turn)throw new Error();await guard(); let token = await authority.accessToken(false);const rejectedDigest=createHash("sha256").update(token).digest("hex"); let prepared = authorizeGrokBrokerProxyRequest({ method: request.method ?? "", pathname: new URL(request.url ?? "/", "http://127.0.0.1").pathname, headers, body }, capabilities, token, turn.policy ?? fallback); token = ""; // The spend gate runs after the body is proven a real lean worker request // (a refused session-title body never counts) and before any upstream call. diff --git a/src/runtime/grokInferenceGrants.test.ts b/src/runtime/grokInferenceGrants.test.ts new file mode 100644 index 0000000..8fb97e1 --- /dev/null +++ b/src/runtime/grokInferenceGrants.test.ts @@ -0,0 +1,94 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; +import { GrokInferenceGrantRefused, GrokInferenceGrants } from "./grokInferenceGrants.js"; +import type { InferenceUsageEntry } from "./inferenceUsageLedger.js"; + +const judge = { model: "grok-4.6", reasoningEffort: "low", purpose: "judge" } as const; +const refusedWith = (code: string) => (error: unknown) => error instanceof GrokInferenceGrantRefused && error.code === code; + +test("a grant is scoped to its declared model, effort and purpose and carries the manifest limits", () => { + const grants = new GrokInferenceGrants(); + try { + const issued = grants.issue({ model: "grok-4.5", reasoningEffort: "medium", purpose: "optimizer" }); + assert.match(issued.token, /^inference_[A-Za-z0-9_-]{43}$/u); assert.match(issued.grantId, /^[a-f0-9]{32}$/u); + assert.deepEqual(issued.limits, { maxRequests: 64, maxTokens: 2_000_000, timeoutMs: 600_000 }); + const grant = grants.authorize(issued.token); + assert.deepEqual(grant?.policy, { model: "grok-4.5", reasoningEffort: "medium" }); assert.equal(grant?.purpose, "optimizer"); + assert.equal(grants.authorize(issued.token.replace(/.$/u, (last) => last === "A" ? "B" : "A")), undefined); + } finally { grants.close(); } +}); + +test("a grant request naming an undeclared model, effort or purpose, or omitting one, is refused", () => { + const grants = new GrokInferenceGrants(); + for (const request of [{ ...judge, model: "grok-3" }, { ...judge, reasoningEffort: "xhigh" }, { ...judge, purpose: "subject" }, { ...judge, model: undefined }, { ...judge, reasoningEffort: undefined }]) { + assert.throws(() => grants.issue(request), refusedWith("invalid_request")); + } + assert.equal(grants.live(), 0); +}); + +test("the grant TTL can never exceed ten minutes", () => { + assert.equal(GROK_ENGINE_BROKER.inferenceGrants.ttlMs <= 600_000, true); + assert.throws(() => new GrokInferenceGrants({ ttlMs: 600_001 }), /invalid inference grant policy/u); +}); + +test("an expired grant is refused and frees its slot", () => { + let now = 1_000_000; + const grants = new GrokInferenceGrants({ now: () => now, maxLiveGrants: 1 }); + try { + const issued = grants.issue(judge); + now += 599_999; assert.ok(grants.authorize(issued.token)); + now += 1; assert.equal(grants.authorize(issued.token), undefined); + assert.equal(grants.live(), 0); assert.ok(grants.issue(judge)); + } finally { grants.close(); } +}); + +test("live grants are capped and a release frees a slot", () => { + const grants = new GrokInferenceGrants(); + try { + const issued = Array.from({ length: GROK_ENGINE_BROKER.inferenceGrants.maxLiveGrants }, () => grants.issue(judge)); + assert.throws(() => grants.issue(judge), refusedWith("grant_limit")); + assert.equal(grants.release(issued[3]!.grantId), true); assert.equal(grants.authorize(issued[3]!.token), undefined); + assert.ok(grants.issue(judge)); assert.throws(() => grants.issue(judge), refusedWith("grant_limit")); + } finally { grants.close(); } +}); + +test("grants never share a key space with turn capabilities", () => { + const turnId = "0123456789abcdef0123456789abcdef"; + const capabilities = new EngineBrokerCapabilities(); const turnToken = capabilities.issue("agent-a", turnId); + const grants = new GrokInferenceGrants({ grantId: () => turnId }); + try { + const issued = grants.issue(judge); + assert.equal(issued.grantId, turnId); + assert.deepEqual(capabilities.inspectToken(turnToken), { agentId: "agent-a", turnId }); + assert.equal(capabilities.inspectToken(issued.token), undefined); + assert.equal(grants.authorize(turnToken), undefined); + capabilities.revoke(turnId); assert.ok(grants.authorize(issued.token)); + grants.release(turnId); assert.equal(capabilities.inspectToken(turnToken), undefined); + } finally { grants.close(); } +}); + +test("a grant meters one request at a time and emits one row per settled request, estimated when usage is missing", () => { + const rows: InferenceUsageEntry[] = []; + const grants = new GrokInferenceGrants({ onSettled: (row) => rows.push(row) }); + try { + const grant = grants.authorize(grants.issue(judge).token)!; + const first = grant.meter.admit(); assert.ok("index" in first); + assert.deepEqual(grant.meter.admit(), { busy: true }); + grants.settle(grant, first.index, { input: 90, cacheRead: 10, cacheWrite: 0, output: 5, total: 105 }, 400); + grants.settle(grant, first.index, { input: 1, cacheRead: 0, cacheWrite: 0, output: 1, total: 2 }, 400); + const second = grant.meter.admit(); assert.ok("index" in second); + grants.settle(grant, second.index, undefined, 1_000); + assert.deepEqual(rows.map((row) => [row.request, row.usage.total, row.usageSource, row.purpose, row.model]), [[0, 105, "upstream", "judge", "grok-4.6"], [1, 4_596, "estimated", "judge", "grok-4.6"]]); + } finally { grants.close(); } +}); + +test("releasing a grant aborts its in-flight upstream request", () => { + const grants = new GrokInferenceGrants(); + const issued = grants.issue(judge); const grant = grants.authorize(issued.token)!; + const admission = grant.meter.admit(); assert.ok("signal" in admission); + grants.release(issued.grantId); + assert.equal(admission.signal.aborted, true); +}); diff --git a/src/runtime/grokInferenceGrants.ts b/src/runtime/grokInferenceGrants.ts new file mode 100644 index 0000000..14e1ac4 --- /dev/null +++ b/src/runtime/grokInferenceGrants.ts @@ -0,0 +1,114 @@ +import { createHash, randomBytes, timingSafeEqual } from "node:crypto"; + +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import type { EngineBrokerTurnLimits, EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; +import { parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; +import { GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; +import { GROK_INFERENCE_PURPOSES, type GrokInferencePurpose, type InferenceUsageEntry } from "./inferenceUsageLedger.js"; + +const SPEC = GROK_ENGINE_BROKER.inferenceGrants; + +/** One live evaluator grant as the proxy sees it. */ +export type GrokInferenceGrant = Readonly<{ grantId: string; purpose: GrokInferencePurpose; policy: GrokBrokerModelPolicy; expiresAt: number; meter: GrokBrokerTurnMeter }>; +export type GrokInferenceGrantIssued = Readonly<{ grantId: string; token: string; purpose: GrokInferencePurpose; policy: GrokBrokerModelPolicy; expiresAt: number; limits: EngineBrokerTurnLimits }>; +export type GrokInferenceGrantRequest = Readonly<{ model: unknown; reasoningEffort: unknown; purpose: unknown }>; + +export class GrokInferenceGrantRefused extends Error { + constructor(readonly code: "grant_limit" | "invalid_request") { super(`inference grant refused (${code})`); } +} + +type Entry = { grant: GrokInferenceGrant; digest: Buffer; timer: NodeJS.Timeout }; +export type GrokInferenceGrantsOptions = Readonly<{ + now?: () => number; + /** Test seam: a fixed grant id proves grants never share a key space with turn capabilities. */ + grantId?: () => string; + onSettled?: (entry: InferenceUsageEntry) => void; + ttlMs?: number; + maxLiveGrants?: number; +}>; + +/** + * Evaluator inference grants: a distinct kind beside subject turn capabilities. + * + * Grants live in their own map keyed by a random grant id; turn capabilities + * (`engineBrokerCapabilities.ts`) are keyed by turn id and never consulted + * here, and grant tokens carry `inference_` so the proxy routes a bearer to + * exactly one of the two lookups. A grant has no worker isolation guard (it is + * issued only to the organization uid, the trusted evaluator side), but it + * carries the same spend gate as a subject turn: a {@link GrokBrokerTurnMeter} + * with one request in flight, the request ceiling, the between-requests token + * ceiling and the estimate for a response without usage. Its lifetime is the + * meter's time limit: past `expiresAt` it is gone from the map, and a request + * still in flight at expiry is aborted. + * + * At most `maxLiveGrants` grants exist at once; a caller frees a slot early + * with {@link release}. Nothing here is durable — a broker restart drops every + * grant, and callers request a new one. + */ +export class GrokInferenceGrants { + private readonly grants = new Map(); + private readonly now: () => number; + private readonly ttlMs: number; + private readonly maxLive: number; + constructor(private readonly options: GrokInferenceGrantsOptions = {}) { + this.now = options.now ?? Date.now; + this.ttlMs = options.ttlMs ?? SPEC.ttlMs; + this.maxLive = options.maxLiveGrants ?? SPEC.maxLiveGrants; + if (!Number.isSafeInteger(this.ttlMs) || this.ttlMs < 1 || this.ttlMs > SPEC.ttlMs || !Number.isSafeInteger(this.maxLive) || this.maxLive < 1 || this.maxLive > SPEC.maxLiveGrants) throw new TypeError("invalid inference grant policy"); + } + + issue(request: GrokInferenceGrantRequest): GrokInferenceGrantIssued { + let policy: GrokBrokerModelPolicy; + try { policy = parseGrokBrokerModelPolicy({ model: request.model, reasoningEffort: request.reasoningEffort }); } catch { throw new GrokInferenceGrantRefused("invalid_request"); } + // Nothing is defaulted: the model parser fills an absent member, a grant must name both. + if (request.model !== policy.model || request.reasoningEffort !== policy.reasoningEffort || !(GROK_INFERENCE_PURPOSES as readonly unknown[]).includes(request.purpose)) throw new GrokInferenceGrantRefused("invalid_request"); + this.prune(); + if (this.grants.size >= this.maxLive) throw new GrokInferenceGrantRefused("grant_limit"); + const grantId = this.options.grantId?.() ?? randomBytes(16).toString("hex"); + if (!/^[a-f0-9]{32}$/u.test(grantId) || this.grants.has(grantId)) throw new GrokInferenceGrantRefused("grant_limit"); + const limits: EngineBrokerTurnLimits = Object.freeze({ maxRequests: SPEC.limits.maxRequests, maxTokens: SPEC.limits.maxTokens, timeoutMs: this.ttlMs }); + const token = `${SPEC.tokenPrefix}${randomBytes(32).toString("base64url")}`; + const grant: GrokInferenceGrant = Object.freeze({ grantId, purpose: request.purpose as GrokInferencePurpose, policy, expiresAt: this.now() + this.ttlMs, meter: new GrokBrokerTurnMeter(limits, () => undefined, this.now) }); + const timer = setTimeout(() => this.release(grantId), this.ttlMs); timer.unref?.(); + this.grants.set(grantId, { grant, digest: digest(token), timer }); + return Object.freeze({ grantId, token, purpose: grant.purpose, policy, expiresAt: grant.expiresAt, limits }); + } + + /** The live, unexpired grant a bearer names, or `undefined`. Never counts a request. */ + authorize(token: string): GrokInferenceGrant | undefined { + if (!token.startsWith(SPEC.tokenPrefix)) return undefined; + this.prune(); + const candidate = digest(token); + for (const entry of this.grants.values()) if (timingSafeEqual(entry.digest, candidate)) return entry.grant; + return undefined; + } + + /** Records one admitted request's end on the grant's meter and emits its ledger row. */ + settle(grant: GrokInferenceGrant, index: number, usage: EngineBrokerTurnUsage | undefined, requestBytes: number): void { + const before = grant.meter.snapshot().timings[index]; + if (before === undefined || before.endedAt !== undefined) return; + grant.meter.settle(index, usage, requestBytes); + const timing = grant.meter.snapshot().timings[index]; + if (timing?.usage === undefined || timing.endedAt === undefined) return; + this.options.onSettled?.({ grant: grant.grantId, purpose: grant.purpose, model: grant.policy.model, request: index, usage: timing.usage, usageSource: timing.estimated === true ? "estimated" : "upstream", startedAt: timing.startedAt, endedAt: timing.endedAt }); + } + + /** Revokes a grant and aborts its in-flight request. Returns whether it was live. */ + release(grantId: string): boolean { + const entry = this.grants.get(grantId); + if (entry === undefined) return false; + clearTimeout(entry.timer); entry.grant.meter.trip("timeout"); entry.digest.fill(0); this.grants.delete(grantId); + return true; + } + + live(): number { this.prune(); return this.grants.size; } + + close(): void { for (const grantId of [...this.grants.keys()]) this.release(grantId); } + + private prune(): void { + const now = this.now(); + for (const [grantId, entry] of this.grants) if (entry.grant.expiresAt <= now) this.release(grantId); + } +} + +const digest = (value: string): Buffer => createHash("sha256").update(value).digest(); diff --git a/src/runtime/grokInferenceProxy.test.ts b/src/runtime/grokInferenceProxy.test.ts new file mode 100644 index 0000000..4874a6b --- /dev/null +++ b/src/runtime/grokInferenceProxy.test.ts @@ -0,0 +1,134 @@ +import assert from "node:assert/strict"; +import { request as httpRequest } from "node:http"; +import test from "node:test"; + +import { startGrokBrokerProxy, type GrokBrokerCredentialAuthority, type GrokBrokerUpstream } from "./grokBrokerProxy.js"; +import { GROK_SESSION_TITLE_SINK_KEY } from "./grokBrokerWorkerConfig.js"; +import { GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; +import { GrokInferenceGrants } from "./grokInferenceGrants.js"; +import { GROK_INFERENCE_AUTH_STALE_BODY } from "./grokInferenceProxy.js"; +import type { InferenceUsageEntry } from "./inferenceUsageLedger.js"; + +// The judge main request Grok 1.0.34 sends (live capture, `--json-schema` variant). +const judgeBody = (overrides: Record = {}): string => JSON.stringify({ + messages: [{ role: "system", content: "You are a strict judge." }, { role: "user", content: "..." }, { role: "user", content: "Rate the answer." }], + model: "grok-4.6", reasoning_effort: "low", + response_format: { type: "json_schema", json_schema: { name: "structured_output", schema: { type: "object", properties: { score: { type: "number" } }, required: ["score"], additionalProperties: false }, strict: true } }, + stream: true, stream_options: { include_usage: true }, ...overrides +}); +// The per-call session_title request the same CLI sends first (live capture). +const titleBody = JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", temperature: 1, max_tokens: 100, messages: [{ role: "system", content: "title" }, { role: "user", content: "Rate the answer." }], tools: [{ type: "function", function: { name: "session_title", description: "", parameters: {} } }], tool_choice: { type: "function", function: { name: "session_title" } }, stream: true, stream_options: { include_usage: true } }); +const usageStream = (total: number) => Buffer.from(`data: ${JSON.stringify({ choices: [{ index: 0, delta: { content: "{\"score\":1}" } }] })}\n\ndata: ${JSON.stringify({ choices: [], usage: { prompt_tokens: total - 5, completion_tokens: 5, total_tokens: total } })}\n\ndata: [DONE]\n\n`); + +type Harness = Readonly<{ port: number; grants: GrokInferenceGrants; rows: InferenceUsageEntry[]; bodies: Record[]; proxy: Awaited> }>; +async function withProxy(run: (harness: Harness) => Promise, options: Readonly<{ authority?: GrokBrokerCredentialAuthority; upstream?: GrokBrokerUpstream }> = {}): Promise { + const rows: InferenceUsageEntry[] = [], bodies: Record[] = []; + const grants = new GrokInferenceGrants({ onSettled: (row) => rows.push(row) }); + const upstream: GrokBrokerUpstream = options.upstream ?? (async (request) => { bodies.push(JSON.parse(Buffer.from(request.body).toString("utf8")) as Record); return { status: 200, headers: { "content-type": "text/event-stream" }, body: usageStream(105) }; }); + const proxy = await startGrokBrokerProxy(options.authority ?? { accessToken: async () => "provider-token", markRejected: async () => undefined }, upstream, undefined, 0, grants); + try { await run({ port: proxy.port, grants, rows, bodies, proxy }); } finally { grants.close(); await proxy.close(); } +} + +function post(port: number, bearer: string, body: string, version = "1.0.34"): Promise> { + return new Promise((resolve, reject) => { + const req = httpRequest({ host: "127.0.0.1", port, path: "/v1/chat/completions", method: "POST", agent: false, headers: { authorization: `Bearer ${bearer}`, "x-grok-client-version": version, "content-type": "application/json" } }, (response) => { + const chunks: Buffer[] = []; response.on("data", (chunk: Buffer) => chunks.push(chunk)); response.on("end", () => resolve({ status: response.statusCode ?? 0, text: Buffer.concat(chunks).toString("utf8") })); + }); + req.on("error", reject); req.end(body); + }); +} + +test("a grant forwards the captured judge request re-serialized under the declared model and meters it into the inference rows only", async () => { + await withProxy(async ({ port, grants, rows, bodies }) => { + const issued = grants.issue({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }); + const result = await post(port, issued.token, judgeBody().replace('"model":', '"model":"grok-4.5","model":')); + assert.equal(result.status, 200); assert.match(result.text, /score/u); + assert.equal(bodies.length, 1); assert.equal(bodies[0]!.model, "grok-4.6"); + assert.deepEqual(rows.map((row) => [row.grant, row.request, row.usage.total, row.usageSource, row.purpose]), [[issued.grantId, 0, 105, "upstream", "judge"]]); + assert.equal((await post(port, issued.token, judgeBody({ response_format: undefined }))).status, 200); + assert.equal(rows.length, 2); + }); +}); + +test("a grant refuses any tools member, the session_title request, and undeclared model or effort, before any upstream call or row", async () => { + await withProxy(async ({ port, grants, rows, bodies }) => { + const { token } = grants.issue({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }); + const refused = [ + judgeBody({ tools: [] }), judgeBody({ tools: [{ type: "function", function: { name: "read_file" } }] }), judgeBody({ tool_choice: "none" }), titleBody, + judgeBody({ model: "grok-4.5" }), judgeBody({ reasoning_effort: "high" }), judgeBody({ reasoning_effort: undefined }), + judgeBody({ stream: false }), judgeBody({ stream_options: undefined }), judgeBody({ temperature: 1 }), + judgeBody({ messages: [{ role: "tool", content: "x" }] }), judgeBody({ messages: [{ role: "assistant", content: null, tool_calls: [] }] }), + judgeBody({ response_format: { type: "json_object" } }) + ]; + for (const body of refused) assert.equal((await post(port, token, body)).status, 503, body.slice(0, 120)); + assert.equal((await post(port, token, judgeBody(), "1.0.30")).status, 503); + assert.equal((await post(port, GROK_SESSION_TITLE_SINK_KEY, titleBody)).status, 503); + assert.equal(bodies.length, 0); assert.equal(rows.length, 0); + }); +}); + +test("an expired, released or unknown grant is refused", async () => { + await withProxy(async ({ port, grants, bodies }) => { + const released = grants.issue({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }); grants.release(released.grantId); + assert.equal((await post(port, released.token, judgeBody())).status, 503); + assert.equal((await post(port, `inference_${"A".repeat(43)}`, judgeBody())).status, 503); + assert.equal(bodies.length, 0); + }); + let now = 5_000; + const grants = new GrokInferenceGrants({ now: () => now }); + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => ({ status: 200, headers: { "content-type": "text/event-stream" }, body: usageStream(10) }), undefined, 0, grants); + try { + const { token } = grants.issue({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }); + assert.equal((await post(proxy.port, token, judgeBody())).status, 200); + now += 600_000; + assert.equal((await post(proxy.port, token, judgeBody())).status, 503); + } finally { grants.close(); await proxy.close(); } +}); + +test("a grant token never authorizes a subject turn and a turn capability never authorizes a grant request", async () => { + await withProxy(async ({ port, grants, proxy, bodies }) => { + const { token: grantToken } = grants.issue({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }); + const turnToken = proxy.capabilities.issue("agent-a", "turn-a"); + proxy.registerIsolationGuard("turn-a", async () => undefined); + proxy.registerTurn("turn-a", { policy: { model: "grok-4.6", reasoningEffort: "low" }, meter: new GrokBrokerTurnMeter({ maxRequests: 4, maxTokens: 10_000, timeoutMs: 60_000 }) }); + const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); + const leanBody = JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: lean }); + assert.equal((await post(port, grantToken, leanBody)).status, 503); + assert.equal((await post(port, turnToken, judgeBody())).status, 503); + assert.equal(bodies.length, 0); + assert.equal((await post(port, turnToken, leanBody)).status, 200); + assert.equal((await post(port, grantToken, judgeBody())).status, 200); + }); +}); + +test("a stale realm answers a grant request with the distinct auth_stale failure, and a rejected refresh too", async () => { + let stale = true; + await withProxy(async ({ port, grants, bodies }) => { + const { token } = grants.issue({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }); + const result = await post(port, token, judgeBody()); + assert.equal(result.status, 401); assert.equal(result.text, GROK_INFERENCE_AUTH_STALE_BODY); assert.equal(bodies.length, 0); + }, { authority: { accessToken: async () => { if (stale) throw new Error("stale"); return "t"; }, markRejected: async () => undefined, isStale: () => stale } }); + stale = false; let rejected = 0; + await withProxy(async ({ port, grants, rows }) => { + const { token } = grants.issue({ model: "grok-4.6", reasoningEffort: "low", purpose: "optimizer" }); + const result = await post(port, token, judgeBody()); + assert.equal(result.status, 401); assert.equal(result.text, GROK_INFERENCE_AUTH_STALE_BODY); assert.equal(rejected, 1); + assert.deepEqual(rows.map((row) => row.usageSource), ["estimated"]); + }, { authority: { accessToken: async (force) => force ? "second" : "first", markRejected: async () => { rejected++; stale = true; throw new Error("stale"); }, isStale: () => stale }, upstream: async () => ({ status: 401, headers: { "content-type": "application/json" }, body: new Uint8Array() }) }); +}); + +test("a grant's request ceiling and one-in-flight rule hold on the wire", async () => { + let release!: () => void; const gate = new Promise((resolve) => { release = resolve; }); + let calls = 0; + await withProxy(async ({ port, grants, rows }) => { + const { token } = grants.issue({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }); + const first = post(port, token, judgeBody()); + while (calls === 0) await new Promise((resolve) => setTimeout(resolve, 5)); + const busy = await post(port, token, judgeBody()); assert.equal(busy.status, 429); assert.match(busy.text, /in flight/u); + release(); assert.equal((await first).status, 200); + const grant = grants.authorize(token)!; + for (let index = 1; index < 64; index++) { const admission = grant.meter.admit(); assert.ok("index" in admission); grants.settle(grant, admission.index, { input: 1, cacheRead: 0, cacheWrite: 0, output: 1, total: 2 }, 10); } + const over = await post(port, token, judgeBody()); assert.equal(over.status, 429); assert.match(over.text, /requests/u); + assert.equal(calls, 1); assert.equal(rows.length, 64); + }, { upstream: async () => { calls++; await gate; return { status: 200, headers: { "content-type": "text/event-stream" }, body: usageStream(50) }; } }); +}); diff --git a/src/runtime/grokInferenceProxy.ts b/src/runtime/grokInferenceProxy.ts new file mode 100644 index 0000000..95169c1 --- /dev/null +++ b/src/runtime/grokInferenceProxy.ts @@ -0,0 +1,65 @@ +import { createHash } from "node:crypto"; +import type { ServerResponse } from "node:http"; + +import type { GrokBrokerCredentialAuthority, GrokBrokerUpstream } from "./grokBrokerProxy.js"; +import { parseGrokUpstreamUsage } from "./grokBrokerTurnMeter.js"; +import type { GrokInferenceGrants } from "./grokInferenceGrants.js"; +import { authorizeGrokInferenceProxyRequest } from "./grokInferenceProxyRequest.js"; + +export type GrokInferenceProxyInput = Readonly<{ method: string; pathname: string; headers: Readonly>; body: Buffer; token: string }>; + +/** HTTP 401 with a fixed code: the broker realm is stale, distinct from a generic 503 broker failure. */ +export const GROK_INFERENCE_AUTH_STALE_BODY = '{"error":"auth_stale"}'; + +/** + * Serves one evaluator grant request; never throws. + * + * Order mirrors the subject path: the body is proven a grant-shaped request + * (declared model/effort, no tools) before the credential is read, the grant + * meter admits it before any upstream call, and exactly one ledger row is + * written once an admitted request settles. No worker isolation guard applies: + * grants are issued only to the organization uid. + * + * Stale realm: the grant shares the subject's credential authority, so a + * stale realm fails judges and subject turns alike (accepted shared fate). A + * grant request that finds the realm stale — before the upstream call or + * after a rejected refresh — is answered 401 `{"error":"auth_stale"}`, never + * the generic 503, so the evaluator can report it as a credential failure. + */ +export async function serveGrokInferenceGrant(input: GrokInferenceProxyInput, response: ServerResponse, grants: GrokInferenceGrants, authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream): Promise { + let settle: ((usage: ReturnType) => void) | undefined; + try { + const grant = grants.authorize(input.token); + if (grant === undefined) throw new Error("inference grant unavailable"); + let prepared = authorizeGrokInferenceProxyRequest(input, "pending", grant.policy); + let token = await authority.accessToken(false); const rejectedDigest = createHash("sha256").update(token).digest("hex"); + prepared = withBearer(prepared, token); token = ""; + const admission = grant.meter.admit(); + if ("refused" in admission) return json(response, 429, JSON.stringify({ error: "grant limit reached", limit: admission.refused })); + if ("busy" in admission) return json(response, 429, '{"error":"grant request in flight"}'); + settle = (usage) => { settle = undefined; grants.settle(grant, admission.index, usage, input.body.byteLength); }; + let result = await upstream(prepared, admission.signal); + if (result.status === 401) { + token = authority.refreshAfterRejection ? await authority.refreshAfterRejection(rejectedDigest) : await authority.accessToken(true); + const refreshedDigest = createHash("sha256").update(token).digest("hex"); prepared = withBearer(prepared, token); token = ""; + result = await upstream(prepared, admission.signal); + if (result.status === 401) await authority.markRejected(refreshedDigest); + } + settle?.(parseGrokUpstreamUsage(result.body, result.headers["content-type"])); + json(response, result.status, result.body, result.headers["content-type"]); + } catch { + settle?.(undefined); + if (authority.isStale?.() === true) json(response, 401, GROK_INFERENCE_AUTH_STALE_BODY); + else json(response, 503, '{"error":"broker unavailable"}'); + } +} + +const withBearer = (prepared: ReturnType, token: string): ReturnType => { + if (!token || /[\r\n]/u.test(token)) throw new Error("broker credential authority unavailable"); + return { ...prepared, headers: { ...prepared.headers, authorization: `Bearer ${token}` } }; +}; + +function json(response: ServerResponse, status: number, body: string | Uint8Array, contentType = "application/json"): void { + if (response.headersSent) return; + response.writeHead(status, { "content-type": contentType, "cache-control": "no-store" }); response.end(body); +} diff --git a/src/runtime/grokInferenceProxyRequest.ts b/src/runtime/grokInferenceProxyRequest.ts new file mode 100644 index 0000000..51796cf --- /dev/null +++ b/src/runtime/grokInferenceProxyRequest.ts @@ -0,0 +1,54 @@ +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import type { GrokBrokerProxyInput, GrokBrokerUpstreamRequest } from "./grokBrokerProxyRequest.js"; +import { parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; + +const MAX_BODY = 2 * 1024 * 1024; +const SPEC = GROK_ENGINE_BROKER.inferenceGrants; +const BODY_MEMBERS: ReadonlySet = new Set(SPEC.bodyMembers); +const ROLES: ReadonlySet = new Set(SPEC.messageRoles); +type JsonRecord = Record; +const plain = (value: unknown): value is JsonRecord => value !== null && typeof value === "object" && !Array.isArray(value); +const rejected = (): Error => new Error("inference grant request rejected"); + +/** + * Authorizes one evaluator grant request body and rebuilds it for the provider. + * + * The accepted shape is exactly what Grok CLI 1.0.34 sends for a Paideia + * judge/optimizer call (`--tools read_file --disallowed-tools + * read_file,search_tool,use_tool --max-turns 1`, with and without + * `--system-prompt-override` and `--json-schema`; live stub capture): + * + * - `stream: true` with `stream_options: {include_usage: true}` — the CLI never + * sends a non-streaming request, so none is accepted; + * - `model` and `reasoning_effort` equal to the grant's declaration; + * - `messages` of plain `{role, content}` string turns, roles system/user/assistant; + * - optional `response_format` `{type: "json_schema", json_schema: {name, schema, strict}}`; + * - no `tools` and no `tool_choice` member at all, not even an empty one. The + * CLI's per-call `session_title` request carries both and is refused here. + * + * The client version must be the pinned CLI, and the forwarded body is the + * re-serialized parse, never the caller's bytes. + */ +export function authorizeGrokInferenceProxyRequest(input: Omit, bearer: string, policy: GrokBrokerModelPolicy): GrokBrokerUpstreamRequest { + const declared = parseGrokBrokerModelPolicy(policy); + if (input.method !== "POST" || input.pathname !== "/v1/chat/completions" || input.body.byteLength < 2 || input.body.byteLength > MAX_BODY) throw rejected(); + if (!bearer || /[\r\n]/u.test(bearer)) throw new Error("broker credential authority unavailable"); + const clientVersion = input.headers["x-grok-client-version"]; + if (clientVersion !== GROK_ENGINE_BROKER.grokCliVersion) throw rejected(); + let parsed: unknown; + try { parsed = JSON.parse(Buffer.from(input.body).toString("utf8"), (key, value: unknown) => { if (key === "__proto__") throw new Error(); return value; }); } catch { throw rejected(); } + if (!plain(parsed) || Object.keys(parsed).some((key) => !BODY_MEMBERS.has(key))) throw rejected(); + if (parsed.model !== declared.model || parsed.reasoning_effort !== declared.reasoningEffort || parsed.stream !== true) throw rejected(); + if (!plain(parsed.stream_options) || Object.keys(parsed.stream_options).length !== 1 || parsed.stream_options.include_usage !== true) throw rejected(); + if (!Array.isArray(parsed.messages) || parsed.messages.length === 0 || !parsed.messages.every(plainMessage)) throw rejected(); + if (parsed.response_format !== undefined && !jsonSchemaFormat(parsed.response_format)) throw rejected(); + return { url: "https://cli-chat-proxy.grok.com/v1/chat/completions", headers: { authorization: `Bearer ${bearer}`, "content-type": "application/json", "x-xai-token-auth": "xai-grok-cli", "x-grok-model-override": declared.model, "x-grok-client-version": clientVersion, "x-grok-client-identifier": "grok-shell" }, body: Buffer.from(JSON.stringify(parsed)) }; +} + +const plainMessage = (message: unknown): boolean => plain(message) && Object.keys(message).length === 2 && ROLES.has(message.role as string) && typeof message.content === "string"; + +const jsonSchemaFormat = (value: unknown): boolean => { + if (!plain(value) || Object.keys(value).length !== 2 || value.type !== "json_schema" || !plain(value.json_schema)) return false; + const schema = value.json_schema; + return Object.keys(schema).every((key) => key === "name" || key === "schema" || key === "strict") && typeof schema.name === "string" && plain(schema.schema) && (schema.strict === undefined || typeof schema.strict === "boolean"); +}; diff --git a/src/runtime/inferenceUsageLedger.ts b/src/runtime/inferenceUsageLedger.ts new file mode 100644 index 0000000..7a389c3 --- /dev/null +++ b/src/runtime/inferenceUsageLedger.ts @@ -0,0 +1,63 @@ +import { GROK_BROKER_MODELS } from "../contracts/grokWorkerContract.js"; +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import type { EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; +import type { GrokBrokerModel } from "./grokBrokerModelPolicy.js"; +import { recordLedgerLines } from "./turnRequestLedger.js"; + +/** + * Evaluator inference spend, one row per upstream model request of one grant. + * + * This is a separate stream at a separate path (`service.json` + * `inferenceLedgerPath`), never the subject usage ledger: the org wake fuse and + * Spawnfile's subject accounting sum that ledger, and a judge's tokens are not + * a subject wake's. Rows carry `kind: "inference"` so a reader that is pointed + * at the wrong file can still tell them apart (`wakeFuse.ts` skips them). + * + * Grants are not sealed in a durable registry: each request is appended when + * it settles. `(grant, request)` identifies a row; readers dedupe on it + * ({@link dedupeInferenceUsageRows}). + */ +export const INFERENCE_USAGE_LEDGER_VERSION = GROK_ENGINE_BROKER.inferenceGrants.ledgerVersion; +export const GROK_INFERENCE_PURPOSES = GROK_ENGINE_BROKER.inferenceGrants.purposes; +export type GrokInferencePurpose = (typeof GROK_INFERENCE_PURPOSES)[number]; + +export type InferenceUsageEntry = Readonly<{ + grant: string; + purpose: GrokInferencePurpose; + model: GrokBrokerModel; + request: number; + usage: EngineBrokerTurnUsage; + usageSource: "upstream" | "estimated"; + startedAt: string; + endedAt: string; + at?: string; +}>; + +export const renderInferenceUsageLine = (entry: InferenceUsageEntry): string => { + if (!/^[a-f0-9]{32}$/u.test(entry.grant) || !(GROK_INFERENCE_PURPOSES as readonly string[]).includes(entry.purpose) || !(GROK_BROKER_MODELS as readonly string[]).includes(entry.model) || !Number.isSafeInteger(entry.request) || entry.request < 0) throw new TypeError("invalid inference usage entry"); + const { usage } = entry; + return `${JSON.stringify({ + v: INFERENCE_USAGE_LEDGER_VERSION, kind: "inference", purpose: entry.purpose, grant: entry.grant, request: entry.request, + at: entry.at ?? new Date().toISOString(), started_at: entry.startedAt, ended_at: entry.endedAt, model: entry.model, + input: usage.input, cache_read: usage.cacheRead, cache_write: usage.cacheWrite, output: usage.output, total: usage.total, + usage_source: entry.usageSource + })}\n`; +}; + +/** Advisory, never rejects: an evaluator request that already spent tokens must not fail on its ledger. */ +export const recordInferenceUsage = async (file: string, entry: InferenceUsageEntry): Promise => { + let line: string; + try { line = renderInferenceUsageLine(entry); } catch { return false; } + return recordLedgerLines(file, line); +}; + +/** Keeps the first row of each `(grant, request)`; rows without both keys are not inference rows and are dropped. */ +export const dedupeInferenceUsageRows = >(rows: readonly T[]): T[] => { + const seen = new Set(); + return rows.filter((row) => { + if (typeof row.grant !== "string" || typeof row.request !== "number") return false; + const key = `${row.grant}\0${row.request}`; + if (seen.has(key)) return false; + seen.add(key); return true; + }); +}; From 5834da14ed42d8c255bf59efd2d4502e11b73d8d Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:04:10 +0200 Subject: [PATCH 051/124] feat: serve inference grant verbs over the control socket and meter grants into a separate inference ledger --- src/runtime/engineBrokerControlClient.ts | 30 +++++++- src/runtime/engineBrokerInferenceProtocol.ts | 77 +++++++++++++++++++ .../engineBrokerInferenceService.test.ts | 66 ++++++++++++++++ src/runtime/engineBrokerProtocol.ts | 9 ++- src/runtime/engineBrokerService.ts | 17 +++- src/runtime/engineBrokerServiceCli.test.ts | 10 +++ src/runtime/engineBrokerServiceCli.ts | 2 +- src/runtime/engineBrokerServiceConfig.ts | 25 +++++- src/runtime/grokEngineBroker.ts | 22 +++++- src/runtime/grokInferenceGrants.ts | 9 ++- src/runtime/grokInferenceLedger.test.ts | 74 ++++++++++++++++++ src/runtime/wakeFuse.ts | 4 +- 12 files changed, 330 insertions(+), 15 deletions(-) create mode 100644 src/runtime/engineBrokerInferenceProtocol.ts create mode 100644 src/runtime/engineBrokerInferenceService.test.ts create mode 100644 src/runtime/grokInferenceLedger.test.ts diff --git a/src/runtime/engineBrokerControlClient.ts b/src/runtime/engineBrokerControlClient.ts index 5054030..6d5c845 100644 --- a/src/runtime/engineBrokerControlClient.ts +++ b/src/runtime/engineBrokerControlClient.ts @@ -1,7 +1,16 @@ import { createHash, randomUUID } from "node:crypto"; import { createConnection } from "node:net"; import { ENGINE_BROKER_VERSION, encodeEngineBrokerFrame,EngineBrokerFrameDecoder,parseEngineBrokerResponse } from "./engineBrokerProtocol.js"; +import type { EngineBrokerInferenceFailureCode, EngineBrokerInferenceRequest, EngineBrokerInferenceResponse } from "./engineBrokerInferenceProtocol.js"; import type { EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; +import type { GrokBrokerModel, GrokBrokerReasoningEffort } from "./grokBrokerModelPolicy.js"; +import type { GrokInferencePurpose } from "./inferenceUsageLedger.js"; + +export type EngineBrokerInferenceGrant = Omit, "version" | "kind" | "requestId">; +/** A refused grant request; `code` is closed (`auth_stale` is the stale shared realm, `grant_limit` the live-grant cap). */ +export class EngineBrokerInferenceGrantRefused extends Error { + constructor(readonly code: EngineBrokerInferenceFailureCode) { super(`engine broker inference grant refused (${code})`); } +} /** * What the organization runtime asks of a brokered turn beyond the prompt: @@ -13,9 +22,28 @@ export interface EngineBrokerTurnClient { turn(agentId:string,wakeId:string,prom export class EngineBrokerControlClient implements EngineBrokerTurnClient { constructor(private readonly socketPath="/run/daimon-engine-broker/control.sock"){} async ready():Promise{const requestId=randomUUID(),socket=createConnection({path:this.socketPath}),decoder=new EngineBrokerFrameDecoder();await new Promise((resolve,reject)=>{let settled=false;const fail=()=>{if(settled)return;settled=true;socket.destroy();reject(new Error("engine broker unavailable"));};socket.once("error",fail);socket.once("close",fail);socket.once("connect",()=>socket.write(encodeEngineBrokerFrame({version:ENGINE_BROKER_VERSION,kind:"health",requestId})));socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const response=parseEngineBrokerResponse(value);if(response.kind!=="ready"||response.requestId!==requestId||settled)throw new Error();settled=true;socket.destroy();resolve();}}catch{fail();}});});} + /** + * Evaluator side (organization uid only; the native relay enforces it): + * borrow the broker credential for one sequential lane of judge or optimizer + * requests. Refusals reject with {@link EngineBrokerInferenceGrantRefused}; + * a broker that cannot answer rejects with `engine broker unavailable`. + */ + async requestInferenceGrant(request:Readonly<{model:GrokBrokerModel;reasoningEffort:GrokBrokerReasoningEffort;purpose:GrokInferencePurpose}>):Promise{ + const response=await this.exchange({version:ENGINE_BROKER_VERSION,kind:"request_inference_grant",requestId:randomUUID(),model:request.model,reasoningEffort:request.reasoningEffort,purpose:request.purpose}); + if(response.kind==="inference_grant_refused")throw new EngineBrokerInferenceGrantRefused(response.code); + if(response.kind!=="inference_grant"||response.model!==request.model||response.reasoningEffort!==request.reasoningEffort||response.purpose!==request.purpose)throw new Error("engine broker unavailable"); + const {version:_version,kind:_kind,requestId:_requestId,...grant}=response;return grant; + } + async releaseInferenceGrant(grantId:string):Promise{ + const response=await this.exchange({version:ENGINE_BROKER_VERSION,kind:"release_inference_grant",requestId:randomUUID(),grantId}); + if(response.kind==="inference_grant_refused")throw new EngineBrokerInferenceGrantRefused(response.code); + if(response.kind!=="inference_grant_released"||response.grantId!==grantId)throw new Error("engine broker unavailable"); + return response.released; + } + private exchange(request:EngineBrokerInferenceRequest):Promise{const socket=createConnection({path:this.socketPath}),decoder=new EngineBrokerFrameDecoder();return new Promise((resolve,reject)=>{let settled=false;const fail=()=>{if(settled)return;settled=true;socket.destroy();reject(new Error("engine broker unavailable"));};socket.once("error",fail);socket.once("close",fail);socket.once("connect",()=>socket.write(encodeEngineBrokerFrame(request)));socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const response=parseEngineBrokerResponse(value);if(settled||response.requestId!==request.requestId||(response.kind!=="inference_grant"&&response.kind!=="inference_grant_released"&&response.kind!=="inference_grant_refused"))throw new Error();settled=true;socket.destroy();resolve(response);}}catch{fail();}});});} async turn(agentId:string,wakeId:string,prompt:string,mcpEndpoint:string,signal?:AbortSignal,options:EngineBrokerTurnOptions={}):Promise{ const turnId=createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"),requestId=randomUUID();const request={version:ENGINE_BROKER_VERSION,kind:"start_turn",requestId,turnId,agentId,wakeId,prompt,mcpEndpoint,...(options.limits===undefined?{}:{limits:options.limits})} as const;const socket=createConnection({path:this.socketPath});const decoder=new EngineBrokerFrameDecoder(); - return new Promise((resolve,reject)=>{let accepted=false,settled=false;const fail=()=>{if(settled)return;settled=true;cleanup();reject(new Error("engine broker unavailable"));};const cleanup=()=>{signal?.removeEventListener("abort",abort);socket.destroy();};const abort=()=>fail();signal?.addEventListener("abort",abort,{once:true});if(signal?.aborted)return abort();socket.once("connect",()=>socket.write(encodeEngineBrokerFrame(request)));socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const response=parseEngineBrokerResponse(value);if(response.kind==="ready"||response.requestId!==requestId||response.turnId!==turnId)throw new Error();if(response.kind==="accepted"){if(accepted)throw new Error();accepted=true;continue;}if(!accepted||settled)throw new Error();settled=true;cleanup(); + return new Promise((resolve,reject)=>{let accepted=false,settled=false;const fail=()=>{if(settled)return;settled=true;cleanup();reject(new Error("engine broker unavailable"));};const cleanup=()=>{signal?.removeEventListener("abort",abort);socket.destroy();};const abort=()=>fail();signal?.addEventListener("abort",abort,{once:true});if(signal?.aborted)return abort();socket.once("connect",()=>socket.write(encodeEngineBrokerFrame(request)));socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const response=parseEngineBrokerResponse(value);if((response.kind!=="accepted"&&response.kind!=="completed"&&response.kind!=="failed")||response.requestId!==requestId||response.turnId!==turnId)throw new Error();if(response.kind==="accepted"){if(accepted)throw new Error();accepted=true;continue;}if(!accepted||settled)throw new Error();settled=true;cleanup(); if(options.model!==undefined&&response.model!==options.model){reject(new Error(`engine broker turn used model ${response.model}, not the declared ${options.model}`));return;} if(response.kind==="completed")resolve(response.text);else reject(new Error(`engine broker turn failed (${response.code}${response.limitReason==="none"?"":`; limit=${response.limitReason}`}${response.diagnostic ? `; ${response.diagnostic.stage}/${response.diagnostic.failureClass}; exit=${response.diagnostic.exitCode}; signal=${response.diagnostic.termSignal}` : ""})`));}}catch{fail();}});socket.once("error",fail);socket.once("close",()=>{if(!settled)fail();});}); } diff --git a/src/runtime/engineBrokerInferenceProtocol.ts b/src/runtime/engineBrokerInferenceProtocol.ts new file mode 100644 index 0000000..02310b9 --- /dev/null +++ b/src/runtime/engineBrokerInferenceProtocol.ts @@ -0,0 +1,77 @@ +import { GROK_BROKER_MODELS, GROK_BROKER_REASONING_EFFORTS } from "../contracts/grokWorkerContract.js"; +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import type { EngineBrokerTurnLimits } from "./engineBrokerTurnAccounting.js"; +import type { GrokBrokerModel, GrokBrokerReasoningEffort } from "./grokBrokerModelPolicy.js"; +import type { GrokInferencePurpose } from "./inferenceUsageLedger.js"; + +/** + * Evaluator inference grant frames, additive kinds of control protocol v2. + * + * Both ends ship in this package (the organization-side client and the broker + * service), so the kinds join v2 without a version bump; a broker that + * predates them refuses the unknown kind and closes the connection, which the + * client reports as `unavailable`. The native relay forwards frames opaquely + * and admits only the organization uid on `control.sock` (`SO_PEERCRED`), so + * that check is what limits grants to the evaluator side. + * + * The grant `token` is the bearer the evaluator's Grok CLI presents to the + * provider proxy (`env_key`). It never carries the broker credential. + */ +const SPEC = GROK_ENGINE_BROKER.inferenceGrants; +export const ENGINE_BROKER_INFERENCE_FAILURE_CODES = SPEC.failureCodes; +export type EngineBrokerInferenceFailureCode = (typeof ENGINE_BROKER_INFERENCE_FAILURE_CODES)[number]; +export const GROK_INFERENCE_PROXY_BASE_URL = `http://${GROK_ENGINE_BROKER.providerProxy.host}:${GROK_ENGINE_BROKER.providerProxy.port}/v1` as const; + +type V = "noopolis.daimon.engine-broker.v2"; +export type EngineBrokerInferenceRequest = + | Readonly<{ version: V; kind: "request_inference_grant"; requestId: string; model: GrokBrokerModel; reasoningEffort: GrokBrokerReasoningEffort; purpose: GrokInferencePurpose }> + | Readonly<{ version: V; kind: "release_inference_grant"; requestId: string; grantId: string }>; +export type EngineBrokerInferenceResponse = + | Readonly<{ version: V; kind: "inference_grant"; requestId: string; grantId: string; token: string; baseUrl: typeof GROK_INFERENCE_PROXY_BASE_URL; model: GrokBrokerModel; reasoningEffort: GrokBrokerReasoningEffort; purpose: GrokInferencePurpose; expiresAt: string; limits: EngineBrokerTurnLimits }> + | Readonly<{ version: V; kind: "inference_grant_released"; requestId: string; grantId: string; released: boolean }> + | Readonly<{ version: V; kind: "inference_grant_refused"; requestId: string; code: EngineBrokerInferenceFailureCode }>; + +type JsonRecord = Record; +const invalid = (): TypeError => new TypeError("invalid broker frame"); +const exact = (value: JsonRecord, fields: readonly string[]): void => { if (Object.keys(value).length !== fields.length || fields.some((field) => !Object.hasOwn(value, field))) throw invalid(); }; +const member = (list: readonly T[], value: unknown): T => { if (!(list as readonly unknown[]).includes(value)) throw invalid(); return value as T; }; +const grantId = (value: unknown): string => { if (typeof value !== "string" || !/^[a-f0-9]{32}$/u.test(value)) throw invalid(); return value; }; +const TOKEN = new RegExp(`^${SPEC.tokenPrefix}[A-Za-z0-9_-]{43}$`, "u"); + +/** `input` has already passed the v2 envelope checks; `requestId` is the parsed id. */ +export function parseEngineBrokerInferenceRequest(input: JsonRecord, requestId: string, version: V): EngineBrokerInferenceRequest { + if (input.kind === "request_inference_grant") { + exact(input, ["version", "kind", "requestId", "model", "reasoningEffort", "purpose"]); + return { version, kind: "request_inference_grant", requestId, model: member(GROK_BROKER_MODELS, input.model), reasoningEffort: member(GROK_BROKER_REASONING_EFFORTS, input.reasoningEffort), purpose: member(SPEC.purposes, input.purpose) }; + } + if (input.kind === "release_inference_grant") { + exact(input, ["version", "kind", "requestId", "grantId"]); + return { version, kind: "release_inference_grant", requestId, grantId: grantId(input.grantId) }; + } + throw invalid(); +} + +export function parseEngineBrokerInferenceResponse(input: JsonRecord, requestId: string, version: V): EngineBrokerInferenceResponse { + if (input.kind === "inference_grant") { + exact(input, ["version", "kind", "requestId", "grantId", "token", "baseUrl", "model", "reasoningEffort", "purpose", "expiresAt", "limits"]); + if (typeof input.token !== "string" || !TOKEN.test(input.token) || input.baseUrl !== GROK_INFERENCE_PROXY_BASE_URL || typeof input.expiresAt !== "string" || Number.isNaN(Date.parse(input.expiresAt)) || new Date(input.expiresAt).toISOString() !== input.expiresAt) throw invalid(); + const limits = input.limits as JsonRecord; + if (limits === null || typeof limits !== "object" || Array.isArray(limits)) throw invalid(); + exact(limits, ["maxRequests", "maxTokens", "timeoutMs"]); + if (!(Number.isSafeInteger(limits.maxRequests) && (limits.maxRequests as number) >= 1 && (limits.maxRequests as number) <= SPEC.limits.maxRequests && Number.isSafeInteger(limits.maxTokens) && (limits.maxTokens as number) >= 1 && (limits.maxTokens as number) <= SPEC.limits.maxTokens && Number.isSafeInteger(limits.timeoutMs) && (limits.timeoutMs as number) >= 1 && (limits.timeoutMs as number) <= SPEC.ttlMs)) throw invalid(); + return { version, kind: "inference_grant", requestId, grantId: grantId(input.grantId), token: input.token, baseUrl: GROK_INFERENCE_PROXY_BASE_URL, model: member(GROK_BROKER_MODELS, input.model), reasoningEffort: member(GROK_BROKER_REASONING_EFFORTS, input.reasoningEffort), purpose: member(SPEC.purposes, input.purpose), expiresAt: input.expiresAt, limits: { maxRequests: limits.maxRequests as number, maxTokens: limits.maxTokens as number, timeoutMs: limits.timeoutMs as number } }; + } + if (input.kind === "inference_grant_released") { + exact(input, ["version", "kind", "requestId", "grantId", "released"]); + if (typeof input.released !== "boolean") throw invalid(); + return { version, kind: "inference_grant_released", requestId, grantId: grantId(input.grantId), released: input.released }; + } + if (input.kind === "inference_grant_refused") { + exact(input, ["version", "kind", "requestId", "code"]); + return { version, kind: "inference_grant_refused", requestId, code: member(ENGINE_BROKER_INFERENCE_FAILURE_CODES, input.code) }; + } + throw invalid(); +} + +export const isEngineBrokerInferenceRequestKind = (kind: unknown): boolean => kind === "request_inference_grant" || kind === "release_inference_grant"; +export const isEngineBrokerInferenceResponseKind = (kind: unknown): boolean => kind === "inference_grant" || kind === "inference_grant_released" || kind === "inference_grant_refused"; diff --git a/src/runtime/engineBrokerInferenceService.test.ts b/src/runtime/engineBrokerInferenceService.test.ts new file mode 100644 index 0000000..27eed5a --- /dev/null +++ b/src/runtime/engineBrokerInferenceService.test.ts @@ -0,0 +1,66 @@ +import assert from "node:assert/strict"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { EngineBrokerControlClient, EngineBrokerInferenceGrantRefused } from "./engineBrokerControlClient.js"; +import { GROK_INFERENCE_PROXY_BASE_URL } from "./engineBrokerInferenceProtocol.js"; +import { parseEngineBrokerRequest, parseEngineBrokerResponse } from "./engineBrokerProtocol.js"; +import { startEngineBrokerServiceWithIdentity, type EngineBrokerServiceEngine } from "./engineBrokerService.js"; +import { GrokInferenceGrantRefused, GrokInferenceGrants } from "./grokInferenceGrants.js"; + +const V = "noopolis.daimon.engine-broker.v2"; +const baseEngine = (): EngineBrokerServiceEngine => ({ turn: async () => { throw new Error("no turns"); }, readiness: () => ({ providerProxyPort: 43123, mcpFacadePort: 43124, registrations: 1, credentialStale: false, realmLease: true, workerIsolation: true }), close: async () => undefined }); + +async function withService(engine: EngineBrokerServiceEngine, run: (client: EngineBrokerControlClient) => Promise): Promise { + const directory = await mkdtemp(path.join(tmpdir(), "daimon-broker-grants-")), socketPath = path.join(directory, "broker.sock"); + const service = await startEngineBrokerServiceWithIdentity(engine, socketPath, process.getuid!()); + try { await run(new EngineBrokerControlClient(socketPath)); } finally { await service.close(); await rm(directory, { recursive: true, force: true }); } +} +const refusedWith = (code: string) => (error: unknown) => error instanceof EngineBrokerInferenceGrantRefused && error.code === code; + +test("the control socket issues, refuses and releases inference grants", async () => { + const grants = new GrokInferenceGrants({ maxLiveGrants: 1 }); + let stale = false; + const engine: EngineBrokerServiceEngine = { ...baseEngine(), requestInferenceGrant: (request) => { if (stale) throw new GrokInferenceGrantRefused("auth_stale"); return grants.issue(request); }, releaseInferenceGrant: (grantId) => grants.release(grantId) }; + try { + await withService(engine, async (client) => { + const grant = await client.requestInferenceGrant({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }); + assert.equal(grant.baseUrl, GROK_INFERENCE_PROXY_BASE_URL); assert.equal(grant.baseUrl, "http://127.0.0.1:43123/v1"); + assert.match(grant.token, /^inference_/u); assert.deepEqual(grant.limits, { maxRequests: 64, maxTokens: 2_000_000, timeoutMs: 600_000 }); + assert.ok(Date.parse(grant.expiresAt) - Date.now() <= 600_000); + assert.ok(grants.authorize(grant.token)); + await assert.rejects(client.requestInferenceGrant({ model: "grok-4.6", reasoningEffort: "low", purpose: "optimizer" }), refusedWith("grant_limit")); + assert.equal(await client.releaseInferenceGrant(grant.grantId), true); + assert.equal(await client.releaseInferenceGrant(grant.grantId), false); + assert.equal(grants.authorize(grant.token), undefined); + stale = true; + await assert.rejects(client.requestInferenceGrant({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }), refusedWith("auth_stale")); + }); + } finally { grants.close(); } +}); + +test("an engine without grants refuses them as unavailable, and an off-list model never reaches the engine", async () => { + await withService(baseEngine(), async (client) => { + await assert.rejects(client.requestInferenceGrant({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }), refusedWith("unavailable")); + }); + let called = false; + await withService({ ...baseEngine(), requestInferenceGrant: () => { called = true; throw new Error("unreachable"); } }, async (client) => { + await assert.rejects(client.requestInferenceGrant({ model: "grok-3" as "grok-4.6", reasoningEffort: "low", purpose: "judge" }), /unavailable/u); + await assert.rejects(client.requestInferenceGrant({ model: "grok-4.6", reasoningEffort: "xhigh" as "low", purpose: "judge" }), /unavailable/u); + await assert.rejects(client.requestInferenceGrant({ model: "grok-4.6", reasoningEffort: "low", purpose: "subject" as "judge" }), /unavailable/u); + }); + assert.equal(called, false); +}); + +test("grant frames are closed and never carry tools or undeclared members", () => { + const request = { version: V, kind: "request_inference_grant", requestId: "r1", model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }; + assert.deepEqual(parseEngineBrokerRequest(request), request); + for (const bad of [{ ...request, tools: [] }, { ...request, purpose: "subject" }, { ...request, model: "grok-3" }, { ...request, limits: { maxRequests: 1 } }, { ...request, version: "noopolis.daimon.engine-broker.v1" }]) assert.throws(() => parseEngineBrokerRequest(bad), /invalid broker frame/u); + assert.throws(() => parseEngineBrokerRequest({ version: V, kind: "release_inference_grant", requestId: "r1", grantId: "not-hex" }), /invalid broker frame/u); + const grant = { version: V, kind: "inference_grant", requestId: "r1", grantId: "a".repeat(32), token: `inference_${"A".repeat(43)}`, baseUrl: GROK_INFERENCE_PROXY_BASE_URL, model: "grok-4.6", reasoningEffort: "low", purpose: "judge", expiresAt: "2026-09-17T05:00:00.000Z", limits: { maxRequests: 64, maxTokens: 2_000_000, timeoutMs: 600_000 } }; + assert.deepEqual(parseEngineBrokerResponse(grant), grant); + for (const bad of [{ ...grant, baseUrl: "http://evil:43123/v1" }, { ...grant, token: "A".repeat(53) }, { ...grant, limits: { ...grant.limits, timeoutMs: 600_001 } }, { ...grant, limits: { ...grant.limits, maxRequests: 65 } }, { ...grant, credential: "x" }]) assert.throws(() => parseEngineBrokerResponse(bad), /invalid broker frame/u); + assert.throws(() => parseEngineBrokerResponse({ version: V, kind: "inference_grant_refused", requestId: "r1", code: "because" }), /invalid broker frame/u); +}); diff --git a/src/runtime/engineBrokerProtocol.ts b/src/runtime/engineBrokerProtocol.ts index d5de2f0..9b9f644 100644 --- a/src/runtime/engineBrokerProtocol.ts +++ b/src/runtime/engineBrokerProtocol.ts @@ -1,3 +1,4 @@ +import { isEngineBrokerInferenceRequestKind, isEngineBrokerInferenceResponseKind, parseEngineBrokerInferenceRequest, parseEngineBrokerInferenceResponse, type EngineBrokerInferenceRequest, type EngineBrokerInferenceResponse } from "./engineBrokerInferenceProtocol.js"; import { parseEngineBrokerTurnAccounting, parseEngineBrokerTurnLimitOverrides, type EngineBrokerTurnAccounting, type EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; /** @@ -16,7 +17,8 @@ const ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; export type EngineBrokerRequest = | Readonly<{ version: typeof VERSION; kind: "health"; requestId: string }> | Readonly<{ version: typeof VERSION; kind: "start_turn"; requestId: string; turnId: string; agentId: string; wakeId: string; prompt: string; mcpEndpoint: string; limits?: EngineBrokerTurnLimitOverrides }> - | Readonly<{ version: typeof VERSION; kind: "cancel_turn"; requestId: string; turnId: string }>; + | Readonly<{ version: typeof VERSION; kind: "cancel_turn"; requestId: string; turnId: string }> + | EngineBrokerInferenceRequest; export interface EngineBrokerFailureDiagnostic { status:string;stage:string;failureClass:string;profileApplied:boolean;exitCode:number;termSignal:number;workerPid:number;workerUid:number;startTicks:string } @@ -24,7 +26,8 @@ export type EngineBrokerResponse = | Readonly<{ version: typeof VERSION; kind: "ready"; requestId: string; brokerUid: 2100; providerProxyPort: 43123; mcpFacadePort: 43124; registrations: number; credentialStale: false; realmLease: true; workerIsolation: true }> | Readonly<{ version: typeof VERSION; kind: "accepted"; requestId: string; turnId: string }> | (Readonly<{ version: typeof VERSION; kind: "completed"; requestId: string; turnId: string; text: string; workerPid: number; workerUid: number; workerStartTime: string }> & EngineBrokerTurnAccounting) - | (Readonly<{ version: typeof VERSION; kind: "failed"; requestId: string; turnId: string; code: EngineBrokerFailureCode; diagnostic?: EngineBrokerFailureDiagnostic }> & EngineBrokerTurnAccounting); + | (Readonly<{ version: typeof VERSION; kind: "failed"; requestId: string; turnId: string; code: EngineBrokerFailureCode; diagnostic?: EngineBrokerFailureDiagnostic }> & EngineBrokerTurnAccounting) + | EngineBrokerInferenceResponse; export const ENGINE_BROKER_FAILURE_CODES = ["auth_stale", "cancelled", "engine_failed", "invalid_request", "limit_exceeded", "turn_conflict", "unavailable"] as const; export type EngineBrokerFailureCode = (typeof ENGINE_BROKER_FAILURE_CODES)[number]; export type EngineBrokerTerminalResponse = Extract; @@ -63,6 +66,7 @@ export function parseEngineBrokerRequest(value: unknown): EngineBrokerRequest { exact(input, ["version", "kind", "requestId", "turnId"]); return { version: VERSION, kind: "cancel_turn", requestId: id(input.requestId), turnId: id(input.turnId) }; } + if (isEngineBrokerInferenceRequestKind(input.kind)) return parseEngineBrokerInferenceRequest(input, id(input.requestId), VERSION); throw new TypeError("invalid broker frame"); } @@ -74,6 +78,7 @@ export function parseEngineBrokerResponse(value: unknown): EngineBrokerResponse return { version: VERSION, kind: "accepted", requestId: id(input.requestId), turnId: id(input.turnId) }; } if (input.kind === "completed" || input.kind === "failed") return parseTerminal(input, VERSION) as EngineBrokerTerminalResponse; + if (isEngineBrokerInferenceResponseKind(input.kind)) return parseEngineBrokerInferenceResponse(input, id(input.requestId), VERSION); throw new TypeError("invalid broker frame"); } diff --git a/src/runtime/engineBrokerService.ts b/src/runtime/engineBrokerService.ts index 2b3f4c5..aef6e33 100644 --- a/src/runtime/engineBrokerService.ts +++ b/src/runtime/engineBrokerService.ts @@ -3,10 +3,15 @@ import { createServer, type Server, type Socket } from "node:net"; import { encodeEngineBrokerFrame,EngineBrokerFrameDecoder,parseEngineBrokerRequest,type EngineBrokerResponse } from "./engineBrokerProtocol.js"; import type { EngineBrokerTurnAccounting, EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; import { EngineBrokerTurnFailure } from "./grokEngineBroker.js"; +import { GROK_INFERENCE_PROXY_BASE_URL, type EngineBrokerInferenceRequest } from "./engineBrokerInferenceProtocol.js"; +import { GrokInferenceGrantRefused, type GrokInferenceGrantIssued, type GrokInferenceGrantRequest } from "./grokInferenceGrants.js"; export interface EngineBrokerServiceEngine { turn(agentId:string,wakeId:string,prompt:string,mcpEndpoint:string,signal?:AbortSignal,limits?:EngineBrokerTurnLimitOverrides):Promise&EngineBrokerTurnAccounting>; readiness():Readonly<{providerProxyPort:number;mcpFacadePort:number;registrations:number;credentialStale:boolean;realmLease:boolean;workerIsolation:boolean}>; close():Promise; + /** Evaluator inference grants; an engine without them refuses every grant request as `unavailable`. Refusals throw {@link GrokInferenceGrantRefused}. */ + requestInferenceGrant?(request:GrokInferenceGrantRequest):GrokInferenceGrantIssued; + releaseInferenceGrant?(grantId:string):boolean; } export function startEngineBrokerService(broker:EngineBrokerServiceEngine,socketPath="/run/daimon-engine-broker/backend.sock"){ @@ -37,7 +42,7 @@ export async function startEngineBrokerServiceWithIdentity(broker:EngineBrokerSe * reader has disconnected. */ function handleSocketError(socket:Socket,owned:()=>Readonly<{turnId:string;controller:AbortController}>|undefined):void{socket.on("error",()=>{owned()?.controller.abort();socket.destroy();});} -function handle(socket:Socket,broker:EngineBrokerServiceEngine,active:Map):void{const decoder=new EngineBrokerFrameDecoder();let started=false,owned:Readonly<{turnId:string;controller:AbortController}>|undefined;handleSocketError(socket,()=>owned);socket.once("close",()=>owned?.controller.abort());socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const request=parseEngineBrokerRequest(value);if(request.kind==="health"){if(started)throw new Error();started=true;const ready=broker.readiness();if(ready.providerProxyPort!==43123||ready.mcpFacadePort!==43124||ready.registrations<1||ready.credentialStale||!ready.realmLease||!ready.workerIsolation)throw new Error();return send(socket,{version:request.version,kind:"ready",requestId:request.requestId,brokerUid:2100,providerProxyPort:43123,mcpFacadePort:43124,registrations:ready.registrations,credentialStale:false,realmLease:true,workerIsolation:true});}if(request.kind==="cancel_turn"){active.get(request.turnId)?.abort();continue;}if(started||active.has(request.turnId))throw new Error();started=true;const controller=new AbortController();owned={turnId:request.turnId,controller};active.set(request.turnId,controller);socket.write(encodeEngineBrokerFrame({version:request.version,kind:"accepted",requestId:request.requestId,turnId:request.turnId}));void broker.turn(request.agentId,request.wakeId,request.prompt,request.mcpEndpoint,controller.signal,request.limits).then((result)=>send(socket,{version:request.version,kind:"completed",requestId:request.requestId,turnId:request.turnId,text:result.text,workerPid:result.workerPid,workerUid:result.workerUid,workerStartTime:result.workerStartTime,outcome:"completed",usage:result.usage,model:result.model,requests:result.requests,limitReason:"none"}),(error:unknown)=>failed(socket,request,error,controller.signal.aborted)).finally(()=>{if(active.get(request.turnId)===controller)active.delete(request.turnId);owned=undefined;});}}catch{socket.destroy();}});} +function handle(socket:Socket,broker:EngineBrokerServiceEngine,active:Map):void{const decoder=new EngineBrokerFrameDecoder();let started=false,owned:Readonly<{turnId:string;controller:AbortController}>|undefined;handleSocketError(socket,()=>owned);socket.once("close",()=>owned?.controller.abort());socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const request=parseEngineBrokerRequest(value);if(request.kind==="health"){if(started)throw new Error();started=true;const ready=broker.readiness();if(ready.providerProxyPort!==43123||ready.mcpFacadePort!==43124||ready.registrations<1||ready.credentialStale||!ready.realmLease||!ready.workerIsolation)throw new Error();return send(socket,{version:request.version,kind:"ready",requestId:request.requestId,brokerUid:2100,providerProxyPort:43123,mcpFacadePort:43124,registrations:ready.registrations,credentialStale:false,realmLease:true,workerIsolation:true});}if(request.kind==="cancel_turn"){active.get(request.turnId)?.abort();continue;}if(request.kind==="request_inference_grant"||request.kind==="release_inference_grant"){if(started)throw new Error();started=true;serveInferenceGrant(socket,broker,request);continue;}if(started||active.has(request.turnId))throw new Error();started=true;const controller=new AbortController();owned={turnId:request.turnId,controller};active.set(request.turnId,controller);socket.write(encodeEngineBrokerFrame({version:request.version,kind:"accepted",requestId:request.requestId,turnId:request.turnId}));void broker.turn(request.agentId,request.wakeId,request.prompt,request.mcpEndpoint,controller.signal,request.limits).then((result)=>send(socket,{version:request.version,kind:"completed",requestId:request.requestId,turnId:request.turnId,text:result.text,workerPid:result.workerPid,workerUid:result.workerUid,workerStartTime:result.workerStartTime,outcome:"completed",usage:result.usage,model:result.model,requests:result.requests,limitReason:"none"}),(error:unknown)=>failed(socket,request,error,controller.signal.aborted)).finally(()=>{if(active.get(request.turnId)===controller)active.delete(request.turnId);owned=undefined;});}}catch{socket.destroy();}});} /** * Every failure the broker raises for a known registration carries its * accounting (a wake that tried to raise a limit: `usage: null`, zero @@ -51,6 +56,16 @@ function failed(socket:Socket,request:Extractsend(socket,{version:request.version,kind:"inference_grant_refused",requestId:request.requestId,code}); + try{ + if(request.kind==="release_inference_grant"){if(!broker.releaseInferenceGrant)return refuse("unavailable");return send(socket,{version:request.version,kind:"inference_grant_released",requestId:request.requestId,grantId:request.grantId,released:broker.releaseInferenceGrant(request.grantId)});} + if(!broker.requestInferenceGrant)return refuse("unavailable"); + const grant=broker.requestInferenceGrant({model:request.model,reasoningEffort:request.reasoningEffort,purpose:request.purpose}); + send(socket,{version:request.version,kind:"inference_grant",requestId:request.requestId,grantId:grant.grantId,token:grant.token,baseUrl:GROK_INFERENCE_PROXY_BASE_URL,model:grant.policy.model,reasoningEffort:grant.policy.reasoningEffort,purpose:grant.purpose,expiresAt:new Date(grant.expiresAt).toISOString(),limits:grant.limits}); + }catch(error){refuse(error instanceof GrokInferenceGrantRefused?error.code:"unavailable");} +} function send(socket:Socket,response:EngineBrokerResponse):void{if(!socket.destroyed)socket.end(encodeEngineBrokerFrame(response));} async function removeOwnedSocket(file:string,uid:number):Promise{try{const entry=await lstat(file);if(!entry.isSocket()||Number(entry.uid)!==uid)throw new Error("unsafe broker socket");await unlink(file);}catch(error){if((error as NodeJS.ErrnoException).code!=="ENOENT")throw error;}} async function verifySocket(file:string,uid:number):Promise{const entry=await lstat(file);if(!entry.isSocket()||Number(entry.uid)!==uid||(Number(entry.mode)&0o777)!==0o600)throw new Error("unsafe broker socket");} diff --git a/src/runtime/engineBrokerServiceCli.test.ts b/src/runtime/engineBrokerServiceCli.test.ts index db7771d..20078ae 100644 --- a/src/runtime/engineBrokerServiceCli.test.ts +++ b/src/runtime/engineBrokerServiceCli.test.ts @@ -53,3 +53,13 @@ test("rejects caller-selected commands, duplicate identities, and traversal", () assert.throws(() => parseEngineBrokerServiceConfig({ ...base, registrations: [{ ...reg("agent-a", 0), eventsPath: "/workers/0/.grok/sandbox-events.jsonl" }] })); assert.throws(() => parseEngineBrokerServiceConfig({ ...base, registrations: [{ ...reg("agent-a", 0), eventsPath: "/workers/1/.grok/sessions/sandbox-events.jsonl" }] })); }); + +test("v2 may declare an evaluator inference ledger that is never a subject ledger", () => { + const base = config("v2", [v2("agent-a", 0), v2("agent-b", 1)]); + assert.equal(parseEngineBrokerServiceConfig(base).inferenceLedgerPath, undefined); + assert.equal(parseEngineBrokerServiceConfig({ ...base, inferenceLedgerPath: "/run/paideia-inference/inference.jsonl" }).inferenceLedgerPath, "/run/paideia-inference/inference.jsonl"); + for (const inferenceLedgerPath of ["/run/slots/0/usage/usage.jsonl", "/run/slots/1/usage/requests.jsonl", "/var/lib/spawnfile/daimon/usage/usage.jsonl", "/var/lib/spawnfile/daimon/usage/requests.jsonl", "relative.jsonl", "/run/x/../inference.jsonl", "/run/inference.json", 7]) { + assert.throws(() => parseEngineBrokerServiceConfig({ ...base, inferenceLedgerPath }), /invalid engine broker service config/u, String(inferenceLedgerPath)); + } + assert.throws(() => parseEngineBrokerServiceConfig({ ...config("v1", [reg("agent-a", 0)]), inferenceLedgerPath: "/run/paideia-inference/inference.jsonl" }), /invalid engine broker service config/u); +}); diff --git a/src/runtime/engineBrokerServiceCli.ts b/src/runtime/engineBrokerServiceCli.ts index 2a3967a..8661b02 100644 --- a/src/runtime/engineBrokerServiceCli.ts +++ b/src/runtime/engineBrokerServiceCli.ts @@ -12,7 +12,7 @@ const MAX_CONFIG_BYTES=65_536; export async function runEngineBrokerServiceCli():Promise{ if(process.getuid?.()!==2100)throw new Error("engine broker service requires broker identity"); const config=parseEngineBrokerServiceConfig(await readRootConfig(ENGINE_BROKER_SERVICE_CONFIG)); - const broker=await startGrokEngineBroker({grokCommand:"/usr/local/bin/grok",nativeClient:"/opt/daimon/bin/daimon-engine-broker",credentialHome:config.credentialHome,turnStore:config.turnStore,registrations:config.registrations}); + const broker=await startGrokEngineBroker({grokCommand:"/usr/local/bin/grok",nativeClient:"/opt/daimon/bin/daimon-engine-broker",credentialHome:config.credentialHome,turnStore:config.turnStore,registrations:config.registrations,...(config.inferenceLedgerPath===undefined?{}:{inferenceLedgerPath:config.inferenceLedgerPath})}); const service=await startEngineBrokerService(broker);let stopping:Promise|undefined; const stop=()=>{stopping??=service.close();return stopping;}; const onSignal=()=>{void stop().catch(()=>{process.exitCode=1;});};process.once("SIGINT",onSignal);process.once("SIGTERM",onSignal); diff --git a/src/runtime/engineBrokerServiceConfig.ts b/src/runtime/engineBrokerServiceConfig.ts index 836963c..742e2a5 100644 --- a/src/runtime/engineBrokerServiceConfig.ts +++ b/src/runtime/engineBrokerServiceConfig.ts @@ -3,6 +3,7 @@ import path from "node:path"; import { DEFAULT_GROK_BROKER_TURN_LIMITS, parseEngineBrokerTurnLimits, type EngineBrokerTurnLimits } from "./engineBrokerTurnAccounting.js"; import { DEFAULT_GROK_BROKER_MODEL_POLICY, GROK_BROKER_MODELS, GROK_BROKER_REASONING_EFFORTS, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; import { grokWorkerEventsPathFor } from "./grokWorkerSandboxProfile.js"; +import { TURN_REQUEST_LEDGER } from "./turnRequestLedger.js"; import { TURN_USAGE_LEDGER } from "./turnUsageLedger.js"; export const ENGINE_BROKER_SERVICE_V1 = "noopolis.daimon.engine-broker-service.v1" as const; @@ -16,7 +17,14 @@ export type EngineBrokerServiceRegistration = Readonly<{ limits: EngineBrokerTurnLimits; model: GrokBrokerModelPolicy; }>; -export type EngineBrokerServiceConfig = Readonly<{ credentialHome: string; turnStore: string; registrations: readonly EngineBrokerServiceRegistration[] }>; +/** + * `inferenceLedgerPath` (v2, optional) is where evaluator inference grants + * append their rows (`inferenceUsageLedger.ts`). Without it the broker refuses + * every grant request. It can never be a subject ledger: not any + * registration's usage ledger or its `requests.jsonl`, and not the container + * ledger the wake fuse sums. + */ +export type EngineBrokerServiceConfig = Readonly<{ credentialHome: string; turnStore: string; registrations: readonly EngineBrokerServiceRegistration[]; inferenceLedgerPath?: string }>; const V1_REGISTRATION = ["agentId", "slot", "workerUid", "workspace", "profilePath", "eventsPath", "profileSha256"] as const; const V2_REGISTRATION = [...V1_REGISTRATION, "usageLedgerPath", "limits", "model"] as const; @@ -41,7 +49,8 @@ export function parseEngineBrokerServiceConfig(value: unknown): EngineBrokerServ if (!plain(value)) throw invalid(); const v2 = value.version === ENGINE_BROKER_SERVICE_V2; if (!v2 && value.version !== ENGINE_BROKER_SERVICE_V1) throw invalid(); - exact(value, ["version", "credentialHome", "turnStore", "registrations"]); + const top = ["version", "credentialHome", "turnStore", "registrations"]; + exact(value, v2 && Object.hasOwn(value, "inferenceLedgerPath") ? [...top, "inferenceLedgerPath"] : top); if (!absolute(value.credentialHome) || !absolute(value.turnStore) || !Array.isArray(value.registrations) || value.registrations.length === 0) throw invalid(); const seen = new Set(), slots = new Set(); const registrations = value.registrations.map((entry: unknown): EngineBrokerServiceRegistration => { @@ -53,14 +62,22 @@ export function parseEngineBrokerServiceConfig(value: unknown): EngineBrokerServ const base = { agentId, slot: slot as number, workerUid: workerUid as number, workspace, profilePath, eventsPath, profileSha256 }; if (!v2) return { ...base, usageLedgerPath: TURN_USAGE_LEDGER.filePath, limits: DEFAULT_GROK_BROKER_TURN_LIMITS, model: DEFAULT_GROK_BROKER_MODEL_POLICY }; const usageLedgerPath = entry.usageLedgerPath; - if (!absolute(usageLedgerPath) || !usageLedgerPath.endsWith(".jsonl") || usageLedgerPath === engineBrokerRequestLedgerPathFor(usageLedgerPath) || path.posix.normalize(usageLedgerPath) !== usageLedgerPath) throw invalid(); + if (!ledgerPath(usageLedgerPath) || usageLedgerPath === engineBrokerRequestLedgerPathFor(usageLedgerPath)) throw invalid(); let limits: EngineBrokerTurnLimits; try { limits = parseEngineBrokerTurnLimits(entry.limits); } catch { throw invalid(); } return { ...base, usageLedgerPath, limits, model: parseServiceModel(entry.model) }; }); - return { credentialHome: value.credentialHome, turnStore: value.turnStore, registrations }; + const base = { credentialHome: value.credentialHome, turnStore: value.turnStore, registrations }; + if (!Object.hasOwn(value, "inferenceLedgerPath")) return base; + const inferenceLedgerPath = value.inferenceLedgerPath; + if (!ledgerPath(inferenceLedgerPath)) throw invalid(); + const subject = new Set([TURN_USAGE_LEDGER.filePath, TURN_REQUEST_LEDGER.filePath, ...registrations.flatMap((entry) => [entry.usageLedgerPath, engineBrokerRequestLedgerPathFor(entry.usageLedgerPath)])]); + if (subject.has(inferenceLedgerPath)) throw invalid(); + return { ...base, inferenceLedgerPath }; } +const ledgerPath = (item: unknown): item is string => absolute(item) && item.endsWith(".jsonl") && path.posix.normalize(item) === item; + function parseServiceModel(value: unknown): GrokBrokerModelPolicy { if (!plain(value)) throw invalid(); exact(value, ["id", "reasoningEffort"]); diff --git a/src/runtime/grokEngineBroker.ts b/src/runtime/grokEngineBroker.ts index d6d5125..aceb394 100644 --- a/src/runtime/grokEngineBroker.ts +++ b/src/runtime/grokEngineBroker.ts @@ -10,6 +10,7 @@ import { grokBrokerWorkerConfigSha256 } from "./grokBrokerWorkerConfig.js"; import { parseGrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; import { runGrokEngineBrokerTurn, type GrokEngineBrokerTurnResult } from "./grokEngineBrokerTurn.js"; import { createGrokWorkerIsolationGuard,prepareGrokWorkerAttestation } from "./grokWorkerAttestation.js"; +import { createLedgeredGrokInferenceGrants, GrokInferenceGrantRefused, type GrokInferenceGrantRequest } from "./grokInferenceGrants.js"; export { EngineBrokerTurnFailure, type GrokEngineBrokerTurnResult } from "./grokEngineBrokerTurn.js"; export { finishBrokerTurnWithUsage } from "./grokEngineBrokerMetering.js"; @@ -21,11 +22,17 @@ export type GrokEngineBroker = Awaited> * facade, and the root-provisioned registrations. Each registration declares * its own model/effort (whose worker config bytes are attested), usage ledger, * and turn limits (`engineBrokerServiceConfig.ts`). + * + * With an `inferenceLedgerPath` the broker also issues evaluator inference + * grants (`grokInferenceGrants.ts`) over the same credential authority and + * proxy; their rows go only to that ledger. A stale realm refuses a grant as + * `auth_stale`, exactly as it fails subject turns. */ -export async function startGrokEngineBroker(options: Readonly<{ grokCommand: string; nativeClient: string; credentialHome: string; turnStore: string; registrations: readonly GrokEngineBrokerRegistration[] }>) { +export async function startGrokEngineBroker(options: Readonly<{ grokCommand: string; nativeClient: string; credentialHome: string; turnStore: string; registrations: readonly GrokEngineBrokerRegistration[]; inferenceLedgerPath?: string }>) { const registrations = new Map(options.registrations.map((entry) => [entry.agentId, { ...entry, model: parseGrokBrokerModelPolicy(entry.model) }])); if (registrations.size !== options.registrations.length) throw new Error("engine broker registration conflict"); const attestationFor = (registration: GrokEngineBrokerRegistration) => ({ ...registration, brokerGid: 2100, configSha256: grokBrokerWorkerConfigSha256(registration.model) }); - const lease=await acquireGrokBrokerRealmLease(options.credentialHome);const authority = new DurableGrokBrokerCredentialAuthority(options.grokCommand, options.credentialHome);try{await authority.initialize();}catch(error){await lease.close();throw error;} let proxy:Awaited>;try{proxy=await startGrokBrokerProxy(authority);}catch(error){await lease.close();throw error;}let mcp:Awaited>|undefined;try{mcp=await startEngineBrokerMcpFacade();for(const registration of registrations.values())await prepareGrokWorkerAttestation(attestationFor(registration));}catch(error){if(mcp)await mcp.close().catch(()=>undefined);await proxy.close();await lease.close();throw error;}if(!mcp)throw new Error("engine broker unavailable");const turns = new EngineBrokerTurnRegistry(options.turnStore); const active = new Map }>(); let closed = false; + const inferenceLedgerPath=options.inferenceLedgerPath;const grants=inferenceLedgerPath===undefined?undefined:createLedgeredGrokInferenceGrants(inferenceLedgerPath); + const lease=await acquireGrokBrokerRealmLease(options.credentialHome);const authority = new DurableGrokBrokerCredentialAuthority(options.grokCommand, options.credentialHome);try{await authority.initialize();}catch(error){await lease.close();throw error;} let proxy:Awaited>;try{proxy=await startGrokBrokerProxy(authority,undefined,undefined,undefined,grants);}catch(error){await lease.close();throw error;}let mcp:Awaited>|undefined;try{mcp=await startEngineBrokerMcpFacade();for(const registration of registrations.values())await prepareGrokWorkerAttestation(attestationFor(registration));}catch(error){if(mcp)await mcp.close().catch(()=>undefined);await proxy.close();await lease.close();throw error;}if(!mcp)throw new Error("engine broker unavailable");const turns = new EngineBrokerTurnRegistry(options.turnStore); const active = new Map }>(); let closed = false; const facade = mcp; const deps = { turns, proxy, mcp: facade, credentialStale: () => authority.isStale(), @@ -40,7 +47,16 @@ export async function startGrokEngineBroker(options: Readonly<{ grokCommand: str active.set(key, { controller, done: done.then(() => undefined, () => undefined) }); try { return await done; } finally { signal?.removeEventListener("abort", onAbort); active.delete(key); } }, - async close(): Promise { if (closed) return; closed = true; const running=[...active.values()];for (const entry of running) entry.controller.abort();await Promise.allSettled(running.map((entry)=>entry.done));const results=await Promise.allSettled([facade.close(),proxy.close(),lease.close()]);const failures=results.flatMap((entry)=>entry.status==="rejected"?[entry.reason]:[]);if(failures.length)throw new AggregateError(failures,"engine broker shutdown failed"); }, + requestInferenceGrant(request: GrokInferenceGrantRequest) { + if (closed || grants === undefined) throw new GrokInferenceGrantRefused("unavailable"); + if (authority.isStale()) throw new GrokInferenceGrantRefused("auth_stale"); + return grants.issue(request); + }, + releaseInferenceGrant(grantId: string): boolean { + if (grants === undefined) throw new GrokInferenceGrantRefused("unavailable"); + return grants.release(grantId); + }, + async close(): Promise { if (closed) return; closed = true; grants?.close(); const running=[...active.values()];for (const entry of running) entry.controller.abort();await Promise.allSettled(running.map((entry)=>entry.done));const results=await Promise.allSettled([facade.close(),proxy.close(),lease.close()]);const failures=results.flatMap((entry)=>entry.status==="rejected"?[entry.reason]:[]);if(failures.length)throw new AggregateError(failures,"engine broker shutdown failed"); }, readiness: () => ({ providerProxyPort: proxy.port, mcpFacadePort:43_124, registrations: registrations.size,credentialStale:authority.isStale(),realmLease:true,workerIsolation:true }) }; } diff --git a/src/runtime/grokInferenceGrants.ts b/src/runtime/grokInferenceGrants.ts index 14e1ac4..a8c1b93 100644 --- a/src/runtime/grokInferenceGrants.ts +++ b/src/runtime/grokInferenceGrants.ts @@ -1,10 +1,11 @@ import { createHash, randomBytes, timingSafeEqual } from "node:crypto"; import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import type { EngineBrokerInferenceFailureCode } from "./engineBrokerInferenceProtocol.js"; import type { EngineBrokerTurnLimits, EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; import { parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; import { GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; -import { GROK_INFERENCE_PURPOSES, type GrokInferencePurpose, type InferenceUsageEntry } from "./inferenceUsageLedger.js"; +import { GROK_INFERENCE_PURPOSES, recordInferenceUsage, type GrokInferencePurpose, type InferenceUsageEntry } from "./inferenceUsageLedger.js"; const SPEC = GROK_ENGINE_BROKER.inferenceGrants; @@ -14,7 +15,7 @@ export type GrokInferenceGrantIssued = Readonly<{ grantId: string; token: string export type GrokInferenceGrantRequest = Readonly<{ model: unknown; reasoningEffort: unknown; purpose: unknown }>; export class GrokInferenceGrantRefused extends Error { - constructor(readonly code: "grant_limit" | "invalid_request") { super(`inference grant refused (${code})`); } + constructor(readonly code: EngineBrokerInferenceFailureCode) { super(`inference grant refused (${code})`); } } type Entry = { grant: GrokInferenceGrant; digest: Buffer; timer: NodeJS.Timeout }; @@ -111,4 +112,8 @@ export class GrokInferenceGrants { } } +/** The broker's grants: every settled request is appended to the evaluator inference ledger and nowhere else. */ +export const createLedgeredGrokInferenceGrants = (inferenceLedgerPath: string): GrokInferenceGrants => + new GrokInferenceGrants({ onSettled: (entry) => { void recordInferenceUsage(inferenceLedgerPath, entry); } }); + const digest = (value: string): Buffer => createHash("sha256").update(value).digest(); diff --git a/src/runtime/grokInferenceLedger.test.ts b/src/runtime/grokInferenceLedger.test.ts new file mode 100644 index 0000000..26fcaab --- /dev/null +++ b/src/runtime/grokInferenceLedger.test.ts @@ -0,0 +1,74 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { request as httpRequest } from "node:http"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import type { NativeBrokerTurn } from "./engineBrokerNativeClient.js"; +import type { EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +import { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; +import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; +import { runGrokEngineBrokerTurn, type GrokEngineBrokerTurnDependencies } from "./grokEngineBrokerTurn.js"; +import { createLedgeredGrokInferenceGrants } from "./grokInferenceGrants.js"; +import { dedupeInferenceUsageRows, INFERENCE_USAGE_LEDGER_VERSION } from "./inferenceUsageLedger.js"; +import { WakeFuse } from "./wakeFuse.js"; + +const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); +const leanBody = JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: lean }); +const judgeBody = JSON.stringify({ messages: [{ role: "system", content: "judge" }, { role: "user", content: "Rate." }], model: "grok-4.5", reasoning_effort: "medium", stream: true, stream_options: { include_usage: true } }); +const post = (port: number, bearer: string, body: string): Promise => new Promise((resolve, reject) => { + const req = httpRequest({ host: "127.0.0.1", port, path: "/v1/chat/completions", method: "POST", agent: false, headers: { authorization: `Bearer ${bearer}`, "x-grok-client-version": "1.0.34", "content-type": "application/json" } }, (response) => { response.resume(); response.on("end", () => resolve(response.statusCode ?? 0)); }); + req.on("error", reject); req.end(body); +}); +const rows = async (file: string): Promise[]> => (await readFile(file, "utf8").catch(() => "")).split("\n").filter(Boolean).map((line) => JSON.parse(line) as Record); +const eventually = async (check: () => Promise): Promise => { for (let attempt = 0; attempt < 100 && !await check(); attempt++) await new Promise((resolve) => setTimeout(resolve, 10)); }; + +test("a judge grant used while a subject turn runs meters only into the inference ledger and never into the subject turn, its ledger or the wake fuse", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-inference-ledger-")); + const usageLedger = path.join(root, "slot0", "usage.jsonl"), inferenceLedger = path.join(root, "evaluator", "inference.jsonl"); + await mkdir(path.dirname(usageLedger)); await mkdir(path.dirname(inferenceLedger)); + const grants = createLedgeredGrokInferenceGrants(inferenceLedger); + const upstreamUsage = (model: string) => ({ prompt_tokens: model === "grok-4.6" ? 1_000 : 40_000, completion_tokens: 50, total_tokens: model === "grok-4.6" ? 1_050 : 40_050 }); + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async (request) => { + const model = (JSON.parse(Buffer.from(request.body).toString("utf8")) as { model: string }).model; + return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(`data: ${JSON.stringify({ choices: [], usage: upstreamUsage(model) })}\n\ndata: [DONE]\n\n`) }; + }, undefined, 0, grants); + try { + const issued = grants.issue({ model: "grok-4.5", reasoningEffort: "medium", purpose: "judge" }); + const registration: EngineBrokerServiceRegistration = { agentId: "foreman", slot: 0, workerUid: 2_200, workspace: "/workspace", profilePath: "/workers/0/.grok/sandbox.toml", eventsPath: "/workers/0/.grok/sessions/sandbox-events.jsonl", profileSha256: "a".repeat(64), usageLedgerPath: usageLedger, limits: { maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }, model: { model: "grok-4.6", reasoningEffort: "low" } }; + let judgeStatus = 0; + const deps: GrokEngineBrokerTurnDependencies = { + turns: new EngineBrokerTurnRegistry(path.join(root, "turns")), proxy, credentialStale: () => false, + mcp: { register: () => "mcp-capability-0123456789abcdef", revoke: () => undefined }, + prepareIsolation: async () => async () => undefined, + runNative: async (input: NativeBrokerTurn) => { + assert.equal(await post(proxy.port, input.providerCapability, leanBody), 200); + judgeStatus = await post(proxy.port, issued.token, judgeBody); + assert.equal(await post(proxy.port, input.providerCapability, leanBody), 200); + return { text: "", workerPid: 4_242, workerUid: 2_200, startTicks: 99n, diagnostic: { status: "ok", stage: "output", failureClass: "none", profileApplied: false, exitCode: 0, termSignal: 0, workerPid: 4_242, workerUid: 2_200, startTicks: "99" } }; + } + }; + await assert.rejects(runGrokEngineBrokerTurn(deps, registration, "wake-1", "prompt", "http://127.0.0.1:43124/mcp")); + assert.equal(judgeStatus, 200); + await eventually(async () => (await rows(inferenceLedger)).length > 0); + + const subject = await rows(usageLedger), subjectRequests = await rows(path.join(path.dirname(usageLedger), "requests.jsonl")), inference = await rows(inferenceLedger); + assert.deepEqual(subject.map((row) => [row.requests ?? row.calls, row.total, row.model]), [[2, 2_100, "grok-4.6"]], "the subject turn counts only its own two requests"); + assert.deepEqual(subjectRequests.map((row) => row.total), [1_050, 1_050]); + assert.deepEqual(inference.map((row) => [row.v, row.kind, row.purpose, row.grant, row.request, row.model, row.total, row.usage_source]), [[INFERENCE_USAGE_LEDGER_VERSION, "inference", "judge", issued.grantId, 0, "grok-4.5", 40_050, "upstream"]]); + + // Even if an inference row reached the subject ledger, the wake fuse would not count it: + // 2,100 subject tokens are under a 2,101 ceiling; counted, the 40,050-token row would trip it. + await writeFile(usageLedger, [...subject, ...inference].map((row) => JSON.stringify(row)).join("\n") + "\n"); + const fuseDirectory = path.join(root, "fuse"); await mkdir(fuseDirectory); + const fuse = await WakeFuse.open({ organizationKey: "org", now: () => new Date(Date.parse(String(subject[0]!.at)) - 1), environment: { DAIMON_WAKE_FUSE_DIRECTORY: fuseDirectory, DAIMON_WAKE_FUSE_EPOCH: "grants", DAIMON_WAKE_FUSE_MAX_WAKES: "10", DAIMON_WAKE_FUSE_MAX_TOKENS: "2101", DAIMON_TURN_USAGE_LEDGER_PATH: usageLedger } }); + assert.deepEqual(await fuse.admit("foreman", "next"), { state: "admitted" }); + } finally { grants.close(); await proxy.close(); await rm(root, { recursive: true, force: true }); } +}); + +test("inference readers count each (grant, request) once", () => { + type Row = Readonly<{ grant?: string; request?: number; total: number }>; + const row = (grant: string, request: number, total: number): Row => ({ grant, request, total }); + assert.deepEqual(dedupeInferenceUsageRows([row("a", 0, 1), row("a", 1, 2), row("a", 0, 1), row("b", 0, 3), { total: 9 }]).map((value) => value.total), [1, 2, 3]); +}); diff --git a/src/runtime/wakeFuse.ts b/src/runtime/wakeFuse.ts index d5d00f4..04c8038 100644 --- a/src/runtime/wakeFuse.ts +++ b/src/runtime/wakeFuse.ts @@ -206,7 +206,9 @@ async function sumTokens(ledgerPath: string, since: string, agentId?: string): P for (const file of [`${ledgerPath}.1`, ledgerPath]) { for (const line of await lines(file)) { try { - const value = JSON.parse(line) as { at?: unknown; total?: unknown; agent?: unknown; turn?: unknown }; + const value = JSON.parse(line) as { at?: unknown; total?: unknown; agent?: unknown; turn?: unknown; kind?: unknown }; + // Evaluator inference rows belong to their own ledger; one here (a misconfigured path) is never subject spend. + if (value.kind === "inference") continue; if (typeof value.turn === "string") { if (turns.has(value.turn)) continue; turns.add(value.turn); } if ((agentId === undefined || value.agent === agentId) && typeof value.at === "string" && !Number.isNaN(Date.parse(value.at)) && value.at >= since && typeof value.total === "number" && Number.isFinite(value.total) && value.total >= 0) total += value.total; } catch { /* usage accounting is advisory input; malformed lines are skipped */ } From 90eab61e7f37b356528f66fd9f1a2e46e55b77ba Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:05:45 +0200 Subject: [PATCH 052/124] feat: add the pinned evaluator Grok client config renderer for inference grants --- src/contracts/runtimeContractManifest.ts | 6 +- src/runtime/grokInferenceClientConfig.test.ts | 43 ++++++++++++++ src/runtime/grokInferenceClientConfig.ts | 59 +++++++++++++++++++ src/runtime/index.ts | 4 ++ 4 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 src/runtime/grokInferenceClientConfig.test.ts create mode 100644 src/runtime/grokInferenceClientConfig.ts diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 3738843..79f7507 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -111,9 +111,9 @@ export const GROK_ENGINE_BROKER = { envKey: "DAIMON_INFERENCE_GRANT", // sha256 of `renderGrokInferenceClientConfig` for the production proxy base URL and this env key. configSha256: { - "grok-4.6": { low: "", medium: "", high: "" }, - "grok-4.5": { low: "", medium: "", high: "" }, - "grok-build": { low: "", medium: "", high: "" } + "grok-4.6": { low: "f52819340ec27180e75e0744f1cff9608bd8155a7e03881e24f2add77aaab311", medium: "c7fceb6d10d3c9848a282f80b0cf617172f598489092d5b98cbca098b7335605", high: "d612e5f6595ff76a15c2e33b03a7597086ea3556d760eacbb8eefb234d089fd9" }, + "grok-4.5": { low: "93f3c55b45843862782891cf9fd4e477532e43a4da9e9a09a003f49dc67bb9b3", medium: "b8213c439be60dc0268f5a62242cccd12922ab6e8e8d9212f8325848be76b1f2", high: "5ad93821eac55816afabfd1240b8b78508a707299ff903072c6984f24e2ee334" }, + "grok-build": { low: "1a26f5482aad0b872b6442c2fa026eb47c2113bbc3fdc1c580d5b2671695e3c6", medium: "fcb6f8b673e4179b086aa6cc009b43fda418e353927dd225a5f63facd8b4479a", high: "4bc5c6118612ecb32d8f79ad871ddc69e1b33d63ee3abef6b1b0618709d0cf21" } } } }, diff --git a/src/runtime/grokInferenceClientConfig.test.ts b/src/runtime/grokInferenceClientConfig.test.ts new file mode 100644 index 0000000..a1f3abd --- /dev/null +++ b/src/runtime/grokInferenceClientConfig.test.ts @@ -0,0 +1,43 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import { GROK_BROKER_MODELS, GROK_BROKER_REASONING_EFFORTS } from "./grokBrokerModelPolicy.js"; +import { GROK_1_0_34_BUNDLED_SKILLS, GROK_SESSION_TITLE_SINK_KEY } from "./grokBrokerWorkerConfig.js"; +import { GROK_INFERENCE_CLIENT_MODEL_ID, GROK_INFERENCE_GRANT_ENV, grokInferenceClientConfigSha256, renderGrokInferenceClientConfig, renderProductionGrokInferenceClientConfig } from "./grokInferenceClientConfig.js"; + +const production = { baseUrl: "http://127.0.0.1:43123/v1", model: "grok-4.6", reasoningEffort: "low", envKey: "DAIMON_INFERENCE_GRANT" } as const; + +test("the evaluator client config reaches only the grant proxy through env_key, with no MCP and no credential", () => { + const config = renderGrokInferenceClientConfig(production); + assert.match(config, /\[models\]\ndefault = "daimon-inference-grok"\ndefault_reasoning_effort = "low"\nsession_summary = "daimon-session-title-disabled"\n/u); + assert.match(config, /\[model\.daimon-inference-grok\]\nmodel = "grok-4\.6"\nbase_url = "http:\/\/127\.0\.0\.1:43123\/v1"\nenv_key = "DAIMON_INFERENCE_GRANT"\napi_backend = "chat_completions"\n/u); + assert.match(config, /\[\[model\.daimon-inference-grok\.reasoning_efforts\]\]\nvalue = "low"\nlabel = "Low"\ndefault = true\n/u); + assert.equal(config.match(/reasoning_efforts\]\]/gu)?.length, 1); + assert.doesNotMatch(config, /mcp_servers|auth_provider|access_token|refresh_token/u); + assert.equal(GROK_INFERENCE_CLIENT_MODEL_ID, "daimon-inference-grok"); assert.equal(GROK_INFERENCE_GRANT_ENV, "DAIMON_INFERENCE_GRANT"); +}); + +test("the evaluator client config mirrors the worker's lean settings and refuses the session title locally", () => { + const config = renderGrokInferenceClientConfig(production); + for (const skill of GROK_1_0_34_BUNDLED_SKILLS) assert.ok(config.includes(JSON.stringify(skill)), skill); + assert.match(config, /\[workflows\]\nenabled = false\n/u); assert.match(config, /\[managed_mcps\]\nenabled = false\n/u); assert.match(config, /auto_update = false/u); + assert.match(config, new RegExp(`\\[model\\.daimon-session-title-disabled\\]\\nmodel = "disabled"\\nbase_url = "http://127\\.0\\.0\\.1:43123/v1"\\napi_key = "${GROK_SESSION_TITLE_SINK_KEY}"\\nmax_retries = 0\\nhidden = true\\n`, "u")); + assert.ok(GROK_SESSION_TITLE_SINK_KEY.length < 40, "the placeholder can never pass the proxy bearer shape"); +}); + +test("the manifest pins the sha256 of every production evaluator client config", () => { + for (const model of GROK_BROKER_MODELS) for (const reasoningEffort of GROK_BROKER_REASONING_EFFORTS) { + const digest = grokInferenceClientConfigSha256({ ...production, model, reasoningEffort }); + assert.equal(GROK_ENGINE_BROKER.inferenceGrants.client.configSha256[model][reasoningEffort], digest, `${model}/${reasoningEffort}`); + assert.equal(renderProductionGrokInferenceClientConfig({ model, reasoningEffort }), renderGrokInferenceClientConfig({ ...production, model, reasoningEffort })); + } +}); + +test("the evaluator client config refuses non-loopback endpoints, injected keys and undeclared models", () => { + for (const bad of [ + { ...production, baseUrl: "https://cli-chat-proxy.grok.com/v1" }, { ...production, baseUrl: "http://127.0.0.1:43123/v1\"\nx = 1" }, { ...production, baseUrl: "http://127.0.0.1:99999/v1" }, + { ...production, envKey: "X\"\n[mcp_servers.evil]" }, { ...production, envKey: "lower" }, + { ...production, model: "grok-3" }, { ...production, reasoningEffort: "xhigh" } + ]) assert.throws(() => renderGrokInferenceClientConfig(bad as typeof production), /invalid Grok inference client configuration/u); +}); diff --git a/src/runtime/grokInferenceClientConfig.ts b/src/runtime/grokInferenceClientConfig.ts new file mode 100644 index 0000000..0841ab7 --- /dev/null +++ b/src/runtime/grokInferenceClientConfig.ts @@ -0,0 +1,59 @@ +import { createHash } from "node:crypto"; + +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; +import { GROK_INFERENCE_PROXY_BASE_URL } from "./engineBrokerInferenceProtocol.js"; +import { parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; +import { GROK_SESSION_TITLE_SINK_KEY, GROK_SESSION_TITLE_SINK_MODEL_ID, renderGrokLeanBaseConfig } from "./grokBrokerWorkerConfig.js"; + +const SPEC = GROK_ENGINE_BROKER.inferenceGrants.client; +/** The evaluator CLI's only model id: Paideia passes `--model daimon-inference-grok`, never a catalog id. */ +export const GROK_INFERENCE_CLIENT_MODEL_ID = SPEC.modelId; +/** The environment variable the evaluator CLI reads its grant token from. */ +export const GROK_INFERENCE_GRANT_ENV = SPEC.envKey; + +export type GrokInferenceClientConfigInput = Readonly<{ baseUrl: string; model: GrokBrokerModelPolicy["model"]; reasoningEffort: GrokBrokerModelPolicy["reasoningEffort"]; envKey: string }>; + +/** + * `config.toml` bytes for an evaluator Grok CLI (Paideia judge or optimizer, + * uid 2000) that reaches the broker proxy through an inference grant. + * + * Paideia writes it into a private `GROK_HOME` (`0700`, uid 2000) and sets the + * grant token in `envKey`; the CLI never holds the broker credential. Grok + * 1.0.34 ignores `[auth_provider.*]` helpers for a custom model, so the token + * travels through `env_key` exactly as the worker's turn capability does. + * + * It mirrors the worker renderer's lean settings: every bundled skill + * disabled, workflows off, the per-call `session_title` request pointed at a + * hidden model with a placeholder key the proxy refuses before any credential + * read or upstream call, and the declared effort as the model's single + * effort. There is no MCP server. `baseUrl` is the `baseUrl` of the grant (the + * loopback provider proxy); the manifest pins the sha256 for the production + * proxy URL and {@link GROK_INFERENCE_GRANT_ENV}. + * + * Paideia must also accept the init frame this produces: `apiKeySource` is + * `"user"` (not `"oauth"`), with `tools: []` and `mcp_servers: []` (live + * stub capture with the Paideia judge argv). + */ +export function renderGrokInferenceClientConfig(input: GrokInferenceClientConfigInput): string { + let declared: GrokBrokerModelPolicy; + try { declared = parseGrokBrokerModelPolicy({ model: input.model, reasoningEffort: input.reasoningEffort }); } catch { throw new TypeError("invalid Grok inference client configuration"); } + if (declared.model !== input.model || declared.reasoningEffort !== input.reasoningEffort) throw new TypeError("invalid Grok inference client configuration"); + if (typeof input.baseUrl !== "string" || !/^http:\/\/127\.0\.0\.1:([1-9][0-9]{0,4})\/v1$/u.test(input.baseUrl) || Number(input.baseUrl.slice(17, -3)) > 65_535) throw new TypeError("invalid Grok inference client configuration"); + if (typeof input.envKey !== "string" || !/^[A-Z][A-Z0-9_]{2,63}$/u.test(input.envKey)) throw new TypeError("invalid Grok inference client configuration"); + const label = `${declared.reasoningEffort[0]!.toUpperCase()}${declared.reasoningEffort.slice(1)}`; + return [ + renderGrokLeanBaseConfig(), + "[models]", `default = "${GROK_INFERENCE_CLIENT_MODEL_ID}"`, `default_reasoning_effort = "${declared.reasoningEffort}"`, `session_summary = "${GROK_SESSION_TITLE_SINK_MODEL_ID}"`, "", + `[model.${GROK_SESSION_TITLE_SINK_MODEL_ID}]`, 'model = "disabled"', `base_url = "${input.baseUrl}"`, `api_key = "${GROK_SESSION_TITLE_SINK_KEY}"`, "max_retries = 0", "hidden = true", "", + `[model.${GROK_INFERENCE_CLIENT_MODEL_ID}]`, `model = "${declared.model}"`, `base_url = "${input.baseUrl}"`, `env_key = "${input.envKey}"`, + 'api_backend = "chat_completions"', "context_window = 131072", "supports_backend_search = false", "", + `[[model.${GROK_INFERENCE_CLIENT_MODEL_ID}.reasoning_efforts]]`, `value = "${declared.reasoningEffort}"`, `label = "${label}"`, "default = true", "" + ].join("\n"); +} + +export const grokInferenceClientConfigSha256 = (input: GrokInferenceClientConfigInput): string => + createHash("sha256").update(renderGrokInferenceClientConfig(input)).digest("hex"); + +/** The production bytes for a declared model/effort: the grant's proxy URL and the canonical env key. */ +export const renderProductionGrokInferenceClientConfig = (policy: GrokBrokerModelPolicy): string => + renderGrokInferenceClientConfig({ baseUrl: GROK_INFERENCE_PROXY_BASE_URL, model: policy.model, reasoningEffort: policy.reasoningEffort, envKey: GROK_INFERENCE_GRANT_ENV }); diff --git a/src/runtime/index.ts b/src/runtime/index.ts index 83d00a4..75bbfbb 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -10,6 +10,10 @@ export { GROK_BROKER_PROJECTION_VERSION, grokBrokerProjectionSha256, grokBrokerS export { GROK_SLOT_PREFLIGHT_VERSION, grokSlotPreflightCanarySchema, grokSlotPreflightReceiptSchema, parseGrokSlotPreflightReceipt, verifyGrokSlotPreflightReceipt, type GrokSlotPreflightReceipt } from "./grokSlotPreflightReceipt.js"; export { grokBrokerWorkerConfigSha256, renderGrokBrokerWorkerConfig } from "./grokBrokerWorkerConfig.js"; +export { GROK_INFERENCE_CLIENT_MODEL_ID, GROK_INFERENCE_GRANT_ENV, grokInferenceClientConfigSha256, renderGrokInferenceClientConfig, renderProductionGrokInferenceClientConfig, type GrokInferenceClientConfigInput } from "./grokInferenceClientConfig.js"; +export { GROK_INFERENCE_PROXY_BASE_URL, ENGINE_BROKER_INFERENCE_FAILURE_CODES, type EngineBrokerInferenceFailureCode } from "./engineBrokerInferenceProtocol.js"; +export { dedupeInferenceUsageRows, INFERENCE_USAGE_LEDGER_VERSION, GROK_INFERENCE_PURPOSES, type GrokInferencePurpose } from "./inferenceUsageLedger.js"; +export { GROK_INFERENCE_AUTH_STALE_BODY } from "./grokInferenceProxy.js"; export { grokWorkerSandboxProfileSha256, renderGrokWorkerSandboxProfile } from "./grokWorkerSandboxProfile.js"; export { engineBrokerRequestLedgerPathFor, parseEngineBrokerServiceConfig, type EngineBrokerServiceConfig, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; export { DEFAULT_GROK_BROKER_TURN_LIMITS, ENGINE_BROKER_LIMIT_REASONS, type EngineBrokerLimitReason, type EngineBrokerTurnAccounting, From 8b38727b7fe582fa3a42459e059ea69899f45b30 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:06:02 +0200 Subject: [PATCH 053/124] feat: export the control client grant API for evaluator consumers --- src/runtime/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/runtime/index.ts b/src/runtime/index.ts index 75bbfbb..6278626 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -54,3 +54,4 @@ export { type WakeReceiptState } from "./wakeAcceptanceTypes.js"; export type { OrganizationRuntimeControlHost, OrganizationRuntimeControlOptions } from "./organizationRuntimeControl.js"; +export { EngineBrokerControlClient, EngineBrokerInferenceGrantRefused, type EngineBrokerInferenceGrant } from "./engineBrokerControlClient.js"; From 61149092fc7f986633597bd940041f3a5456bfb7 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:06:54 +0200 Subject: [PATCH 054/124] test: prove a grant id equal to a live turn id leaves the turn capability, policy and meter untouched --- src/runtime/grokInferenceProxy.test.ts | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/runtime/grokInferenceProxy.test.ts b/src/runtime/grokInferenceProxy.test.ts index 4874a6b..8fcb20e 100644 --- a/src/runtime/grokInferenceProxy.test.ts +++ b/src/runtime/grokInferenceProxy.test.ts @@ -132,3 +132,26 @@ test("a grant's request ceiling and one-in-flight rule hold on the wire", async assert.equal(calls, 1); assert.equal(rows.length, 64); }, { upstream: async () => { calls++; await gate; return { status: 200, headers: { "content-type": "text/event-stream" }, body: usageStream(50) }; } }); }); + +test("a grant whose id equals a live turn id leaves that turn's capability, policy and meter untouched", async () => { + const shared = "0123456789abcdef0123456789abcdef"; + const grants = new GrokInferenceGrants({ grantId: () => shared }); + let calls = 0; + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; return { status: 200, headers: { "content-type": "text/event-stream" }, body: usageStream(20) }; }, undefined, 0, grants); + try { + const turnToken = proxy.capabilities.issue("agent-a", shared); + const turnMeter = new GrokBrokerTurnMeter({ maxRequests: 4, maxTokens: 10_000, timeoutMs: 60_000 }); + proxy.registerIsolationGuard(shared, async () => undefined); + proxy.registerTurn(shared, { policy: { model: "grok-4.6", reasoningEffort: "low" }, meter: turnMeter }); + const issued = grants.issue({ model: "grok-4.5", reasoningEffort: "high", purpose: "judge" }); + assert.equal(issued.grantId, shared); + const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); + const leanBody = JSON.stringify({ model: "grok-4.6", reasoningEffort: undefined, reasoning_effort: "low", stream: true, messages: [], tools: lean }); + assert.equal((await post(proxy.port, turnToken, leanBody)).status, 200); + assert.equal((await post(proxy.port, issued.token, judgeBody({ model: "grok-4.5", reasoning_effort: "high" }))).status, 200); + assert.equal(turnMeter.snapshot().requests, 1); assert.equal(grants.authorize(issued.token)!.meter.snapshot().requests, 1); + grants.release(shared); + assert.equal((await post(proxy.port, turnToken, leanBody)).status, 200); + assert.equal(turnMeter.snapshot().requests, 2); assert.equal(calls, 3); + } finally { grants.close(); await proxy.close(); } +}); From a45a3dc58dc286c354bdfecae4f8d91e312bd28a Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:18:23 +0200 Subject: [PATCH 055/124] fix: render max_retries = 0 for the evaluator Grok client so a refused grant request fails fast instead of retrying --- src/contracts/runtimeContractManifest.ts | 6 +++--- src/runtime/grokInferenceClientConfig.test.ts | 2 +- src/runtime/grokInferenceClientConfig.ts | 8 ++++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 79f7507..cad2c99 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -111,9 +111,9 @@ export const GROK_ENGINE_BROKER = { envKey: "DAIMON_INFERENCE_GRANT", // sha256 of `renderGrokInferenceClientConfig` for the production proxy base URL and this env key. configSha256: { - "grok-4.6": { low: "f52819340ec27180e75e0744f1cff9608bd8155a7e03881e24f2add77aaab311", medium: "c7fceb6d10d3c9848a282f80b0cf617172f598489092d5b98cbca098b7335605", high: "d612e5f6595ff76a15c2e33b03a7597086ea3556d760eacbb8eefb234d089fd9" }, - "grok-4.5": { low: "93f3c55b45843862782891cf9fd4e477532e43a4da9e9a09a003f49dc67bb9b3", medium: "b8213c439be60dc0268f5a62242cccd12922ab6e8e8d9212f8325848be76b1f2", high: "5ad93821eac55816afabfd1240b8b78508a707299ff903072c6984f24e2ee334" }, - "grok-build": { low: "1a26f5482aad0b872b6442c2fa026eb47c2113bbc3fdc1c580d5b2671695e3c6", medium: "fcb6f8b673e4179b086aa6cc009b43fda418e353927dd225a5f63facd8b4479a", high: "4bc5c6118612ecb32d8f79ad871ddc69e1b33d63ee3abef6b1b0618709d0cf21" } + "grok-4.6": { low: "79314d039f787e4ebfec7dacf57adc969086b948f564dec008f0ed6367e6062f", medium: "6f538de0547c0c4e6a3f04ae08595ceadadabb06b75f6b6ee4c428744bb95cd8", high: "5652656effa82f0c4f09cf8226b16e6140332a5a358b571194bb5563312367ac" }, + "grok-4.5": { low: "a07f7436f1268bb399ec233c65d3b3d8fb99a11a1f175f8da1ca133c9367bc74", medium: "1f4c0d4dad1f3b09419b5739db6423a09e0049abc091594c64123a75dd53dfb9", high: "ffbc33728b821e9854fbc7c93601e599225da421ecfd6ebf10d314afcc28d6f2" }, + "grok-build": { low: "ca15c6a562a008227d39c51d3a3a83715663089b3784e8b46debb1fb67b3c4a1", medium: "01783fb6beadcf6f8486fff0836820ad43fab5662b812a907fe9cfdb83e9804d", high: "98d16f2b7d12f4eb540d625c853e51d227933e204923e43e8b9b4176f10aca2c" } } } }, diff --git a/src/runtime/grokInferenceClientConfig.test.ts b/src/runtime/grokInferenceClientConfig.test.ts index a1f3abd..d9ee6df 100644 --- a/src/runtime/grokInferenceClientConfig.test.ts +++ b/src/runtime/grokInferenceClientConfig.test.ts @@ -11,7 +11,7 @@ const production = { baseUrl: "http://127.0.0.1:43123/v1", model: "grok-4.6", re test("the evaluator client config reaches only the grant proxy through env_key, with no MCP and no credential", () => { const config = renderGrokInferenceClientConfig(production); assert.match(config, /\[models\]\ndefault = "daimon-inference-grok"\ndefault_reasoning_effort = "low"\nsession_summary = "daimon-session-title-disabled"\n/u); - assert.match(config, /\[model\.daimon-inference-grok\]\nmodel = "grok-4\.6"\nbase_url = "http:\/\/127\.0\.0\.1:43123\/v1"\nenv_key = "DAIMON_INFERENCE_GRANT"\napi_backend = "chat_completions"\n/u); + assert.match(config, /\[model\.daimon-inference-grok\]\nmodel = "grok-4\.6"\nbase_url = "http:\/\/127\.0\.0\.1:43123\/v1"\nenv_key = "DAIMON_INFERENCE_GRANT"\napi_backend = "chat_completions"\ncontext_window = 131072\nsupports_backend_search = false\nmax_retries = 0\n/u); assert.match(config, /\[\[model\.daimon-inference-grok\.reasoning_efforts\]\]\nvalue = "low"\nlabel = "Low"\ndefault = true\n/u); assert.equal(config.match(/reasoning_efforts\]\]/gu)?.length, 1); assert.doesNotMatch(config, /mcp_servers|auth_provider|access_token|refresh_token/u); diff --git a/src/runtime/grokInferenceClientConfig.ts b/src/runtime/grokInferenceClientConfig.ts index 0841ab7..d9a043c 100644 --- a/src/runtime/grokInferenceClientConfig.ts +++ b/src/runtime/grokInferenceClientConfig.ts @@ -26,7 +26,11 @@ export type GrokInferenceClientConfigInput = Readonly<{ baseUrl: string; model: * disabled, workflows off, the per-call `session_title` request pointed at a * hidden model with a placeholder key the proxy refuses before any credential * read or upstream call, and the declared effort as the model's single - * effort. There is no MCP server. `baseUrl` is the `baseUrl` of the grant (the + * effort. There is no MCP server. `max_retries = 0`: with the default, a + * refused request (HTTP 503) is retried with backoff past a 45 s bound + * (live stub capture), so a gate refusal would hang the judge until its own + * timeout; with it the CLI fails in ~0.35 s and the caller's routed retry + * policy decides. HTTP 401 is never retried either way. `baseUrl` is the `baseUrl` of the grant (the * loopback provider proxy); the manifest pins the sha256 for the production * proxy URL and {@link GROK_INFERENCE_GRANT_ENV}. * @@ -46,7 +50,7 @@ export function renderGrokInferenceClientConfig(input: GrokInferenceClientConfig "[models]", `default = "${GROK_INFERENCE_CLIENT_MODEL_ID}"`, `default_reasoning_effort = "${declared.reasoningEffort}"`, `session_summary = "${GROK_SESSION_TITLE_SINK_MODEL_ID}"`, "", `[model.${GROK_SESSION_TITLE_SINK_MODEL_ID}]`, 'model = "disabled"', `base_url = "${input.baseUrl}"`, `api_key = "${GROK_SESSION_TITLE_SINK_KEY}"`, "max_retries = 0", "hidden = true", "", `[model.${GROK_INFERENCE_CLIENT_MODEL_ID}]`, `model = "${declared.model}"`, `base_url = "${input.baseUrl}"`, `env_key = "${input.envKey}"`, - 'api_backend = "chat_completions"', "context_window = 131072", "supports_backend_search = false", "", + 'api_backend = "chat_completions"', "context_window = 131072", "supports_backend_search = false", "max_retries = 0", "", `[[model.${GROK_INFERENCE_CLIENT_MODEL_ID}.reasoning_efforts]]`, `value = "${declared.reasoningEffort}"`, `label = "${label}"`, "default = true", "" ].join("\n"); } From dd175ef4e9f2269fd800124f87e7bc1e5b528b72 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:20:16 +0200 Subject: [PATCH 056/124] feat: bind the Grok slot preflight receipt to a recycle nonce and slot generation as v2 --- src/contracts/runtimeContractManifest.ts | 2 +- src/runtime/AGENTS.md | 7 +++- .../receipt.extra-canary.json | 4 +- ...t.valid.v1.json => receipt.legacy-v1.json} | 0 .../receipt.missing-canary.json | 4 +- .../receipt.projection-mismatch.json | 4 +- .../receipt.readable-canary.json | 4 +- .../receipt.unknown-member.json | 4 +- .../grok-slot-preflight/receipt.valid.v2.json | 40 ++++++++++++++++++ src/runtime/grokSlotPreflightReceipt.test.ts | 41 ++++++++++++++----- src/runtime/grokSlotPreflightReceipt.ts | 31 ++++++++++++-- src/runtime/index.ts | 2 +- 12 files changed, 121 insertions(+), 22 deletions(-) rename src/runtime/fixtures/grok-slot-preflight/{receipt.valid.v1.json => receipt.legacy-v1.json} (100%) create mode 100644 src/runtime/fixtures/grok-slot-preflight/receipt.valid.v2.json diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index cad2c99..a9eb1be 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -118,7 +118,7 @@ export const GROK_ENGINE_BROKER = { } }, projectionVersion: "noopolis.daimon.grok-broker-projection.v1", - slotPreflightVersion: "noopolis.daimon.grok-slot-preflight.v1", + slotPreflightVersion: "noopolis.daimon.grok-slot-preflight.v2", artifacts: { sourceSha256: "36f60689f0a8af0e3108f5f53d78ed52b7d4b6f934c75b6184606dfa82bc741e", x64Sha256: "36dc76b134eb59cf5a6720b6f94228eb279108e20ea3343fa6efd9ffcb60a4d3", diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 182a3fa..056d2c3 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -87,9 +87,14 @@ must supply canonical non-symlink paths (its fixed tmpfs and workspace roots) and verify that during provisioning. The projection also carries the seccomp profile digest and the `bubblewrap` sandbox runtime a receipt must match. `grokSlotPreflightReceipt.ts` is the zod schema a root slot supervisor's receipt must satisfy -(`noopolis.daimon.grok-slot-preflight.v1`, fixtures under +(`noopolis.daimon.grok-slot-preflight.v2`, fixtures under `fixtures/grok-slot-preflight/`); `verifyGrokSlotPreflightReceipt` binds it to the projection digest and requires a denied canary for exactly every deny path. +The projection digest does not change across recycles, so the receipt also +carries freshness: a supervisor-owned per-slot `generation` (strictly +increasing) and the caller's recycle `nonce` (32 random bytes, hex). The +verifier requires `{expectedNonce, minGeneration}` and refuses another nonce, a +lower generation, and any v1 receipt. `grokBrokerWorkerConfig.ts` is the only source of worker `config.toml` bytes; the manifest pins the sha256 of every model/effort combination and the broker diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json index f694edb..a1f00ca 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json @@ -1,7 +1,9 @@ { - "version": "noopolis.daimon.grok-slot-preflight.v1", + "version": "noopolis.daimon.grok-slot-preflight.v2", "slot": 0, "worker_uid": 2200, + "generation": 3, + "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v1.json b/src/runtime/fixtures/grok-slot-preflight/receipt.legacy-v1.json similarity index 100% rename from src/runtime/fixtures/grok-slot-preflight/receipt.valid.v1.json rename to src/runtime/fixtures/grok-slot-preflight/receipt.legacy-v1.json diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json index 974b321..0b57369 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json @@ -1,7 +1,9 @@ { - "version": "noopolis.daimon.grok-slot-preflight.v1", + "version": "noopolis.daimon.grok-slot-preflight.v2", "slot": 0, "worker_uid": 2200, + "generation": 3, + "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.projection-mismatch.json b/src/runtime/fixtures/grok-slot-preflight/receipt.projection-mismatch.json index c8729e3..fd520ed 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.projection-mismatch.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.projection-mismatch.json @@ -1,7 +1,9 @@ { - "version": "noopolis.daimon.grok-slot-preflight.v1", + "version": "noopolis.daimon.grok-slot-preflight.v2", "slot": 0, "worker_uid": 2200, + "generation": 3, + "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", "projection_sha256": "0000000000000000000000000000000000000000000000000000000000000000", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json index 8f5d407..19cb96f 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json @@ -1,7 +1,9 @@ { - "version": "noopolis.daimon.grok-slot-preflight.v1", + "version": "noopolis.daimon.grok-slot-preflight.v2", "slot": 0, "worker_uid": 2200, + "generation": 3, + "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json b/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json index 2e50e6a..91774b2 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json @@ -1,7 +1,9 @@ { - "version": "noopolis.daimon.grok-slot-preflight.v1", + "version": "noopolis.daimon.grok-slot-preflight.v2", "slot": 0, "worker_uid": 2200, + "generation": 3, + "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v2.json b/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v2.json new file mode 100644 index 0000000..94542eb --- /dev/null +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v2.json @@ -0,0 +1,40 @@ +{ + "version": "noopolis.daimon.grok-slot-preflight.v2", + "slot": 0, + "worker_uid": 2200, + "generation": 3, + "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", + "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", + "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", + "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "sandbox_runtime": "bubblewrap", + "grok_executable_sha256": "39ab87666877d64ef3a40aa60fbe0c3b6a6acd7001b78fe60e2c76bb6cfc4a94", + "canaries": [ + { + "path": "/run/paideia", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/paideia/control", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/run/training/inputs", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-bootstrap-auth", + "method": "sandboxed-read", + "result": "denied" + }, + { + "path": "/var/lib/spawnfile/daimon/grok-subscription-realm", + "method": "sandboxed-read", + "result": "denied" + } + ], + "created_at": "2026-09-17T12:00:00.000Z" +} diff --git a/src/runtime/grokSlotPreflightReceipt.test.ts b/src/runtime/grokSlotPreflightReceipt.test.ts index b141060..2cc85d2 100644 --- a/src/runtime/grokSlotPreflightReceipt.test.ts +++ b/src/runtime/grokSlotPreflightReceipt.test.ts @@ -5,17 +5,19 @@ import test from "node:test"; import { resolveOrganizationGrokBrokerProjection } from "./grokBrokerProjection.js"; import { parseGrokSlotPreflightReceipt, verifyGrokSlotPreflightReceipt } from "./grokSlotPreflightReceipt.js"; +const fresh = { expectedNonce: "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", minGeneration: 3 } as const; + const fixture = async (name: string): Promise> => JSON.parse(await readFile(new URL(`./fixtures/grok-slot-preflight/${name}`, import.meta.url), "utf8")) as Record; const projection = async () => { const input = await fixture("projection-input.json") as { config: unknown; agentId: string; options: Parameters[2] }; return resolveOrganizationGrokBrokerProjection(input.config, input.agentId, input.options); }; test("the committed valid receipt fixture proves the committed projection input", async () => { - const receipt = verifyGrokSlotPreflightReceipt(await fixture("receipt.valid.v1.json"), await projection()); + const receipt = verifyGrokSlotPreflightReceipt(await fixture("receipt.valid.v2.json"), await projection(), fresh); assert.equal(receipt.canaries.length, (await projection()).denyPaths.length); assert.ok(receipt.canaries.every((canary) => canary.method === "sandboxed-read" && canary.result === "denied")); }); test("the schema refuses a readable canary, an unknown member, duplicates and malformed digests or times", async () => { - const valid = await fixture("receipt.valid.v1.json"); + const valid = await fixture("receipt.valid.v2.json"); await assert.rejects(async () => parseGrokSlotPreflightReceipt(await fixture("receipt.readable-canary.json")), /invalid Grok slot preflight receipt/u); await assert.rejects(async () => parseGrokSlotPreflightReceipt(await fixture("receipt.unknown-member.json")), /invalid Grok slot preflight receipt/u); const canaries = valid.canaries as Record[]; @@ -28,23 +30,40 @@ test("the schema refuses a readable canary, an unknown member, duplicates and ma { ...valid, projection_sha256: "A".repeat(64) }, { ...valid, worker_uid: 2_000 }, { ...valid, created_at: "2026-09-17T12:00:00Z" }, - { ...valid, version: "noopolis.daimon.grok-slot-preflight.v2" } + { ...valid, version: "noopolis.daimon.grok-slot-preflight.v3" } ]) assert.throws(() => parseGrokSlotPreflightReceipt(bad), /invalid Grok slot preflight receipt/u); }); test("a receipt for a different projection, slot, profile or deny set is refused", async () => { const projected = await projection(); - const valid = await fixture("receipt.valid.v1.json"); + const valid = await fixture("receipt.valid.v2.json"); // Mutation guard: dropping the digest comparison accepts this fixture. - await assert.rejects(async () => verifyGrokSlotPreflightReceipt(await fixture("receipt.projection-mismatch.json"), projected), /projection_sha256/u); - await assert.rejects(async () => verifyGrokSlotPreflightReceipt(await fixture("receipt.missing-canary.json"), projected), /canaries/u); + await assert.rejects(async () => verifyGrokSlotPreflightReceipt(await fixture("receipt.projection-mismatch.json"), projected, fresh), /projection_sha256/u); + await assert.rejects(async () => verifyGrokSlotPreflightReceipt(await fixture("receipt.missing-canary.json"), projected, fresh), /canaries/u); // Exact match, both halves: a canary for a path the projection does not deny is as wrong as a missing one. - await assert.rejects(async () => verifyGrokSlotPreflightReceipt(await fixture("receipt.extra-canary.json"), projected), /canaries/u); - assert.throws(() => verifyGrokSlotPreflightReceipt(valid, { ...projected, limits: { ...projected.limits, maxTokens: 1 } }), /projection_sha256/u); - assert.throws(() => verifyGrokSlotPreflightReceipt({ ...valid, sandbox_profile_sha256: "1".repeat(64) }, projected), /sandbox_profile_sha256/u); - assert.throws(() => verifyGrokSlotPreflightReceipt({ ...valid, slot: 1 }, projected), /slot/u); + await assert.rejects(async () => verifyGrokSlotPreflightReceipt(await fixture("receipt.extra-canary.json"), projected, fresh), /canaries/u); + assert.throws(() => verifyGrokSlotPreflightReceipt(valid, { ...projected, limits: { ...projected.limits, maxTokens: 1 } }, fresh), /projection_sha256/u); + assert.throws(() => verifyGrokSlotPreflightReceipt({ ...valid, sandbox_profile_sha256: "1".repeat(64) }, projected, fresh), /sandbox_profile_sha256/u); + assert.throws(() => verifyGrokSlotPreflightReceipt({ ...valid, slot: 1 }, projected, fresh), /slot/u); // Mutation guard: never comparing the seccomp digest accepts a receipt taken under another profile. - assert.throws(() => verifyGrokSlotPreflightReceipt({ ...valid, seccomp_profile_sha256: "8".repeat(64) }, projected), /seccomp_profile_sha256/u); + assert.throws(() => verifyGrokSlotPreflightReceipt({ ...valid, seccomp_profile_sha256: "8".repeat(64) }, projected, fresh), /seccomp_profile_sha256/u); assert.throws(() => parseGrokSlotPreflightReceipt({ ...valid, sandbox_runtime: "none" }), /invalid Grok slot preflight receipt/u); assert.throws(() => parseGrokSlotPreflightReceipt((({ sandbox_runtime: _omit, ...rest }) => rest)(valid)), /invalid Grok slot preflight receipt/u); }); + +test("a receipt from an earlier recycle is refused: another nonce, a lower generation, or a v1 receipt without freshness", async () => { + const projected = await projection(); + const valid = await fixture("receipt.valid.v2.json"); + assert.equal(verifyGrokSlotPreflightReceipt(valid, projected, { ...fresh, minGeneration: 1 }).generation, 3); + // Mutation guard: ignoring the nonce accepts a receipt written for another recycle request. + assert.throws(() => verifyGrokSlotPreflightReceipt(valid, projected, { ...fresh, expectedNonce: "a".repeat(64) }), /stale: nonce/u); + // Mutation guard: ignoring the generation accepts a receipt older than the last one accepted. + assert.throws(() => verifyGrokSlotPreflightReceipt(valid, projected, { ...fresh, minGeneration: 4 }), /stale: generation/u); + await assert.rejects(async () => verifyGrokSlotPreflightReceipt(await fixture("receipt.legacy-v1.json"), projected, fresh), /invalid Grok slot preflight receipt/u); + for (const bad of [{ ...valid, nonce: "A".repeat(64) }, { ...valid, nonce: "ab" }, { ...valid, generation: 0 }, { ...valid, generation: 1.5 }, (({ nonce: _omit, ...rest }) => rest)(valid), (({ generation: _omit, ...rest }) => rest)(valid)]) { + assert.throws(() => parseGrokSlotPreflightReceipt(bad), /invalid Grok slot preflight receipt/u); + } + for (const freshness of [{ expectedNonce: "short", minGeneration: 1 }, { expectedNonce: fresh.expectedNonce, minGeneration: 0 }, { expectedNonce: fresh.expectedNonce.toUpperCase(), minGeneration: 1 }]) { + assert.throws(() => verifyGrokSlotPreflightReceipt(valid, projected, freshness), /invalid Grok slot preflight freshness/u); + } +}); diff --git a/src/runtime/grokSlotPreflightReceipt.ts b/src/runtime/grokSlotPreflightReceipt.ts index 7658581..791b9b3 100644 --- a/src/runtime/grokSlotPreflightReceipt.ts +++ b/src/runtime/grokSlotPreflightReceipt.ts @@ -7,6 +7,7 @@ import { grokBrokerProjectionSha256, type OrganizationGrokBrokerProjection } fro export const GROK_SLOT_PREFLIGHT_VERSION = GROK_ENGINE_BROKER.slotPreflightVersion; const sha256 = z.string().regex(/^[a-f0-9]{64}$/u); +const NONCE = /^[a-f0-9]{64}$/u; const canonicalAbsolute = z.string().max(4_096).refine((value) => path.posix.isAbsolute(value) && path.posix.normalize(value) === value && value !== "/" && !value.endsWith("/") && !value.includes("\0"), "canonical absolute path"); /** @@ -22,17 +23,28 @@ export const grokSlotPreflightCanarySchema = z.strictObject({ }); /** - * `noopolis.daimon.grok-slot-preflight.v1`: what the root slot supervisor (P5) + * `noopolis.daimon.grok-slot-preflight.v2`: what the root slot supervisor (P5) * writes after provisioning or recycling one broker slot, and what an * evaluator (Paideia, P4) must hold before it runs a Grok subject turn in that * slot. It binds the slot to one exact projection by digest, so any change to * the model, limits, deny list, profile, worker config, or pinned executable * invalidates it. + * + * The projection digest is identical across recycles of the same slot, so v1 + * could not tell this recycle's receipt from an earlier one. v2 adds + * freshness: `generation` is the supervisor-owned per-slot counter, strictly + * increasing on every provision/recycle, and `nonce` echoes the 32 random + * bytes (hex) the evaluator passed in its recycle request. A v1 receipt is + * refused. */ export const grokSlotPreflightReceiptSchema = z.strictObject({ version: z.literal(GROK_SLOT_PREFLIGHT_VERSION), slot: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER), worker_uid: z.number().int().min(GROK_ENGINE_BROKER.identities.firstWorkerUid).max(4_294_967_294), + /** Supervisor-owned, strictly increasing per slot across provisions and recycles; starts at 1. */ + generation: z.number().int().min(1).max(Number.MAX_SAFE_INTEGER), + /** The caller's recycle nonce: 32 random bytes, lowercase hex. */ + nonce: z.string().regex(NONCE), projection_sha256: sha256, /** The bubblewrap/Landlock `daimon-strict` profile bytes' digest (the projection's `profileSha256`). */ sandbox_profile_sha256: sha256, @@ -57,15 +69,28 @@ export function parseGrokSlotPreflightReceipt(value: unknown): GrokSlotPreflight return result.data; } +/** + * What the evaluator knows about *this* recycle: the nonce it sent, and the + * lowest generation it will accept — one above the last generation it + * accepted for the slot (1 for a slot it has never seen). + */ +export type GrokSlotPreflightFreshness = Readonly<{ expectedNonce: string; minGeneration: number }>; + /** * Parse a receipt and require that it proves *this* projection's slot: same * digest, slot, worker uid, profile and executable, and a denied canary for * exactly every projected deny path (no more, no fewer), under the projected - * seccomp profile and sandbox runtime. + * seccomp profile and sandbox runtime — and that it is *this* recycle's + * receipt: the caller's nonce, at or above the caller's minimum generation. + * A receipt replayed from an earlier recycle fails on the nonce, and one + * whose generation went backwards fails on the generation. */ -export function verifyGrokSlotPreflightReceipt(value: unknown, projection: OrganizationGrokBrokerProjection): GrokSlotPreflightReceipt { +export function verifyGrokSlotPreflightReceipt(value: unknown, projection: OrganizationGrokBrokerProjection, freshness: GrokSlotPreflightFreshness): GrokSlotPreflightReceipt { + if (freshness === null || typeof freshness !== "object" || typeof freshness.expectedNonce !== "string" || !NONCE.test(freshness.expectedNonce) || !Number.isSafeInteger(freshness.minGeneration) || freshness.minGeneration < 1) throw new TypeError("invalid Grok slot preflight freshness"); const receipt = parseGrokSlotPreflightReceipt(value); const mismatch = (member: string): never => { throw new Error(`Grok slot preflight receipt does not match the projection: ${member}`); }; + if (receipt.nonce !== freshness.expectedNonce) throw new Error("Grok slot preflight receipt is stale: nonce"); + if (receipt.generation < freshness.minGeneration) throw new Error("Grok slot preflight receipt is stale: generation"); if (receipt.projection_sha256 !== grokBrokerProjectionSha256(projection)) mismatch("projection_sha256"); if (receipt.slot !== projection.slot) mismatch("slot"); if (receipt.worker_uid !== projection.workerUid) mismatch("worker_uid"); diff --git a/src/runtime/index.ts b/src/runtime/index.ts index 6278626..24195c5 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -7,7 +7,7 @@ export { CODEX_SANDBOX_PROJECTION_VERSION, resolveOrganizationCodexSandboxProjec type OrganizationCodexSandboxProjection } from "./codexSandboxProjection.js"; export { GROK_BROKER_PROJECTION_VERSION, grokBrokerProjectionSha256, grokBrokerServiceRegistrationFor, resolveOrganizationGrokBrokerProjection, verifyGrokBrokerRegistrationMatchesProjection, type OrganizationGrokBrokerProjection, type OrganizationGrokBrokerProjectionOptions } from "./grokBrokerProjection.js"; -export { GROK_SLOT_PREFLIGHT_VERSION, grokSlotPreflightCanarySchema, grokSlotPreflightReceiptSchema, parseGrokSlotPreflightReceipt, +export { GROK_SLOT_PREFLIGHT_VERSION, grokSlotPreflightCanarySchema, grokSlotPreflightReceiptSchema, parseGrokSlotPreflightReceipt, type GrokSlotPreflightFreshness, verifyGrokSlotPreflightReceipt, type GrokSlotPreflightReceipt } from "./grokSlotPreflightReceipt.js"; export { grokBrokerWorkerConfigSha256, renderGrokBrokerWorkerConfig } from "./grokBrokerWorkerConfig.js"; export { GROK_INFERENCE_CLIENT_MODEL_ID, GROK_INFERENCE_GRANT_ENV, grokInferenceClientConfigSha256, renderGrokInferenceClientConfig, renderProductionGrokInferenceClientConfig, type GrokInferenceClientConfigInput } from "./grokInferenceClientConfig.js"; From a9e1f68f56ae834e632cb366a9b0855a9e184141 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:20:25 +0200 Subject: [PATCH 057/124] docs: describe the v2 slot preflight receipt freshness in the engines guide --- docs/engines.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/engines.md b/docs/engines.md index b2517f6..7bf4b93 100644 --- a/docs/engines.md +++ b/docs/engines.md @@ -100,8 +100,9 @@ kills the worker. The broker seals every terminal turn with its usage, request count, declared model and limit reason, and writes one usage row (keyed by `turn`) plus per-request rows for completed and failed turns alike; a replayed turn is never metered twice. `resolveOrganizationGrokBrokerProjection` exposes -a slot's full declared shape, and `noopolis.daimon.grok-slot-preflight.v1` -receipts bind a slot's denied-path canaries to that projection's digest. +a slot's full declared shape, and `noopolis.daimon.grok-slot-preflight.v2` +receipts bind a slot's denied-path canaries to that projection's digest and to +one recycle (the caller's nonce and the slot's increasing generation). AGY uses OS-native secure storage through one private D-Bus and Secret Service realm. Enroll it once with: From 148f3a9af6f4dc6f75daf9a9cb0242f024103a50 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:20:43 +0200 Subject: [PATCH 058/124] docs: document evaluator inference grants, the captured judge request shape and shared stale-realm fate --- docs/engines.md | 9 +++++++++ src/runtime/AGENTS.md | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/docs/engines.md b/docs/engines.md index 7bf4b93..6342210 100644 --- a/docs/engines.md +++ b/docs/engines.md @@ -104,6 +104,15 @@ a slot's full declared shape, and `noopolis.daimon.grok-slot-preflight.v2` receipts bind a slot's denied-path canaries to that projection's digest and to one recycle (the caller's nonce and the slot's increasing generation). +Evaluators (Paideia judges and the optimizer, organization uid only) borrow the +same credential through inference grants: `request_inference_grant` over the +control socket returns a ten-minute token for one declared model and effort, +which the evaluator's Grok CLI presents to the provider proxy through +`env_key` in a config rendered by `renderGrokInferenceClientConfig`. Grant +requests must carry no tools, are metered like a turn, and are written only to +the broker's separate `inferenceLedgerPath` (`kind: "inference"` rows), never +to a subject usage ledger or the wake fuse. + AGY uses OS-native secure storage through one private D-Bus and Secret Service realm. Enroll it once with: diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 056d2c3..1628dc5 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -77,6 +77,46 @@ model (`grok-4.6-build` → `grok-4.6`), otherwise the turn fails as rejected an is still metered. Control protocol v2 is refused-v1 on the wire because both ends ship in this package. +Evaluator inference grants (`grokInferenceGrants.ts`) let Paideia judges and +the DSPy optimizer — uid 2000, the trusted evaluator side — spend the broker's +Grok credential without holding it. `request_inference_grant {model, +reasoningEffort, purpose: judge|optimizer}` is an additive control protocol v2 +verb (`engineBrokerInferenceProtocol.ts`); only the organization uid reaches it, +because the native relay admits only that `SO_PEERCRED` uid on `control.sock` +(the TS backend sees only the relay). The answer is a token +(`inference_` + 32 random bytes), the proxy base URL, an expiry (TTL ten +minutes) and the manifest limits; `release_inference_grant` frees one of the +eight live-grant slots early. Grants are their own kind: their own map keyed by +a random grant id, never the turn capability or turn meter maps, and the proxy +routes a bearer by its prefix to exactly one of the two lookups. A grant has no +worker isolation guard but the same spend gate as a turn (one request in +flight, request ceiling, between-requests token ceiling, estimate on missing +usage), so one grant is one sequential lane — parallel judges each hold one. +`grokInferenceProxyRequest.ts` accepts exactly what Grok 1.0.34 sends for the +Paideia judge argv (live stub capture): `stream: true` with +`stream_options.include_usage`, the declared `model`/`reasoning_effort`, plain +`{role, content}` messages, optional `response_format` json_schema, and **no +`tools` or `tool_choice` member at all** — the CLI's per-call `session_title` +request carries both and is refused locally. Every settled request appends one +`kind: "inference"` row (`purpose`, `grant`, `request`, model, usage, +`usage_source`) to `service.json` v2's optional `inferenceLedgerPath`, which +may never be a subject ledger; readers dedupe on `(grant, request)` +(`dedupeInferenceUsageRows`), and `wakeFuse.ts` skips inference rows. Without +that path every grant request is refused `unavailable`. Grants share the +subject's credential authority, so a stale realm fails both (accepted shared +fate): the grant request is refused `auth_stale`, and a proxied grant request +that meets a stale realm gets HTTP 401 `{"error":"auth_stale"}`, which the CLI +surfaces immediately as `Internal error: "Unauthorized (401) from …: +auth_stale …"`. `grokInferenceClientConfig.ts` renders the evaluator's private +`GROK_HOME` `config.toml` (pinned per model/effort in the manifest): the grant +token through `env_key = "DAIMON_INFERENCE_GRANT"`, the worker's lean settings, +no MCP, and `max_retries = 0` — with the default, Grok retries a refused (503) +request with backoff past 45 s instead of failing in ~0.35 s. Its init frame +reports `apiKeySource: "user"`, `tools: []`, `mcp_servers: []`, and the CLI +must be run with `--model daimon-inference-grok`. The inference ledger +directory must be provisioned setgid to the organization group (e.g. +`2100:2000 2750`) for uid 2000 to read rows the broker creates `0640`. + `grokBrokerProjection.ts` is the public, I/O-free projection of one brokered Grok agent's slot (`noopolis.daimon.grok-broker-projection.v1`): Daimon's own deny collectors plus the caller's evaluator paths, profile/config/prompt From 024e1026eabd2a5b5d835addcbe8dca27cad6d22 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:39:29 +0200 Subject: [PATCH 059/124] feat: give each Grok worker a private TMPDIR under its registered home --- src/contracts/runtimeContractManifest.ts | 11 ++++++++++- src/runtime/native/engineBrokerLauncherCore.inc | 6 +++++- .../engineBrokerLauncherIntegrationLauncher.inc | 2 +- src/runtime/native/fixtureWorker.c | 2 +- src/runtime/native/launcherArgv.test.ts | 9 ++++++++- 5 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index a9eb1be..2235701 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -64,7 +64,16 @@ export const GROK_ENGINE_BROKER = { directory: { uid: 0, group: "worker", mode: 0o1771 }, sessionsDirectory: { relativePath: "sessions", uid: 0, group: "worker", mode: 0o1771 }, readOnlyFiles: { names: ["config.toml", "managed_config.toml", "requirements.toml", "sandbox.toml", "trusted_folders.toml"], uid: 0, gid: 0, mode: 0o444 }, - sandboxEvents: { relativePath: "sessions/sandbox-events.jsonl", owner: "worker", group: "broker", mode: 0o640 } + sandboxEvents: { relativePath: "sessions/sandbox-events.jsonl", owner: "worker", group: "broker", mode: 0o640 }, + // The launcher exports TMPDIR=/tmp; Grok's strict profile grants TMPDIR read-write. + privateTmp: { relativeToWorkerHome: "tmp", owner: "worker", mode: 0o700 }, + // Strict also grants shared /tmp and /var/tmp read-write and refuses to start if either is + // denied, so the deployment keeps them from every worker by mode: root-owned, a non-worker + // group (< 2200), others read-only (Grok needs to open the directory) and no search/write. + sharedTmp: { paths: ["/tmp", "/var/tmp"], uid: 0, maxGroupExclusive: 2_200, otherMode: 0o4, mode: 0o1774 }, + // Spilled tool output the worker reads with read_file: setgid directory in the worker's group, + // files written 0640 by the runtime, never other-readable. + spillDirectory: { relativeToRuntimeHome: "tool-output", owner: "organization", group: "worker", mode: 0o2750, fileMode: 0o640 } } }, bounds: { promptBytes: 65_536, capabilityBytes: 4_096, capabilityBundleBytes: 8_196, outputBytes: 65_536 }, diff --git a/src/runtime/native/engineBrokerLauncherCore.inc b/src/runtime/native/engineBrokerLauncherCore.inc index cb2b9ec..41cc947 100644 --- a/src/runtime/native/engineBrokerLauncherCore.inc +++ b/src/runtime/native/engineBrokerLauncherCore.inc @@ -361,10 +361,13 @@ static pid_t launch(const struct dbl_registration *r, int executable, "--model", "daimon-broker-grok", NULL}; - char home[300], grok[300], mcp_env[DBL_MAX_TOKEN + 24], + char home[300], grok[300], tmp[300], mcp_env[DBL_MAX_TOKEN + 24], provider_env[DBL_MAX_TOKEN + 32]; snprintf(home, sizeof(home), "HOME=%s", r->home); snprintf(grok, sizeof(grok), "GROK_HOME=%s/.grok", r->home); + /* Worker-private temp: strict grants TMPDIR read-write, and shared /tmp is + kept from the worker by the deployment's modes (attested by the broker). */ + snprintf(tmp, sizeof(tmp), "TMPDIR=%s/tmp", r->home); snprintf(mcp_env, sizeof(mcp_env), "DAIMON_MCP_CAPABILITY=%s", mcp); /* Grok 1.0.34 never runs `[auth_provider.*]` helpers for a custom model, so the worker's `[model.daimon-broker-grok] env_key` reads the turn-scoped @@ -375,6 +378,7 @@ static pid_t launch(const struct dbl_registration *r, int executable, erase(mcp, sizeof(mcp)); char *const envp[] = {home, grok, + tmp, mcp_env, provider_env, "DAIMON_CAPABILITY_FD=4", diff --git a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc index ee189d9..1512c47 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc @@ -110,7 +110,7 @@ static void org_cases(void) { char *out = calloc(1, r.output_length + 1); check(read(s, out, r.output_length) == (ssize_t)r.output_length && strstr(out, "uid=2200") && strstr(out, "--always-approve") && - strstr(out, " prompt=prompt fds=0,1,2,3,4 stdin=/dev/null\n") && + strstr(out, " prompt=prompt fds=0,1,2,3,4 stdin=/dev/null tmpdir=/tmp/worker-home/tmp\n") && !strstr(out, "EVIL"), "fixed worker boundary"); check(strstr(out, "=--verbatim\n") && strstr(out, "=--no-plan\n") && diff --git a/src/runtime/native/fixtureWorker.c b/src/runtime/native/fixtureWorker.c index b26a4ae..cb92d4e 100644 --- a/src/runtime/native/fixtureWorker.c +++ b/src/runtime/native/fixtureWorker.c @@ -6,4 +6,4 @@ #include #include #include "engineBrokerLauncher.h" -int main(int argc,char**argv){unsigned char bundle[8200]={0};ssize_t size=pread(4,bundle,sizeof(bundle),0);if(size<6)return 21;size_t offset=0;unsigned provider_length=bundle[0]|(bundle[1]<<8);offset=2;if(offset+provider_length+2>(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output");if(!strcmp(provider,"sleep"))sleep(30);if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;id_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target);for(int i=0;i(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output");if(!strcmp(provider,"sleep"))sleep(30);if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;id_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s tmpdir=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target,getenv("TMPDIR")?getenv("TMPDIR"):"unset");for(int i=0;i { const source = read("engineBrokerLauncherCore.inc"); assert.match(source, new RegExp(`"${GROK_BROKER_PROVIDER_CAPABILITY_ENV}=%s"`, "u")); - assert.match(source, /char \*const envp\[\] = \{home,\s*grok,\s*mcp_env,\s*provider_env,/u); + assert.match(source, /char \*const envp\[\] = \{home,\s*grok,\s*tmp,\s*mcp_env,\s*provider_env,/u); assert.match(renderGrokBrokerWorkerConfig(), new RegExp(`\\nenv_key = "${GROK_BROKER_PROVIDER_CAPABILITY_ENV}"\\n`, "u")); }); + +test("the launcher gives every worker a private TMPDIR under its registered home", () => { + const source = read("engineBrokerLauncherCore.inc"); + assert.match(source, /snprintf\(tmp, sizeof\(tmp\), "TMPDIR=%s\/tmp", r->home\);/u); + assert.match(source, /char \*const envp\[\] = \{home,\s*grok,\s*tmp,/u); + assert.equal(GROK_ENGINE_BROKER.worker.home.privateTmp.relativeToWorkerHome, "tmp"); +}); From aa1aef0f1a1a887501838172cc76e96a2aeb8d5c Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:39:29 +0200 Subject: [PATCH 060/124] feat: refuse Grok turns unless shared temp is closed to workers and the private temp is worker-only --- src/runtime/grokWorkerAttestation.ts | 6 ++- .../grokWorkerAttestationChecks.test.ts | 37 ++++++++++--- src/runtime/grokWorkerTmpAttestation.test.ts | 53 +++++++++++++++++++ src/runtime/grokWorkerTmpAttestation.ts | 48 +++++++++++++++++ 4 files changed, 136 insertions(+), 8 deletions(-) create mode 100644 src/runtime/grokWorkerTmpAttestation.test.ts create mode 100644 src/runtime/grokWorkerTmpAttestation.ts diff --git a/src/runtime/grokWorkerAttestation.ts b/src/runtime/grokWorkerAttestation.ts index adfa3f5..d10d01f 100644 --- a/src/runtime/grokWorkerAttestation.ts +++ b/src/runtime/grokWorkerAttestation.ts @@ -4,6 +4,7 @@ import { lstat,open } from "node:fs/promises"; import path from "node:path"; import { verifyGrokWorkerHome } from "./grokWorkerHomeAttestation.js"; +import { verifyGrokWorkerTmp } from "./grokWorkerTmpAttestation.js"; import { grokWorkerEventsPathFor } from "./grokWorkerSandboxProfile.js"; /** @@ -61,9 +62,12 @@ export function parseGrokWorkerSandboxProfile(bytes:Uint8Array,profileSha256:str * 1.0.34), and a root-owned read-only worker home whose `config.toml` hashes to * the declared renderer output (`grokWorkerHomeAttestation.ts`). */ -export async function prepareGrokWorkerAttestation(input:Readonly<{profilePath:string;eventsPath:string;profileSha256:string;workerUid:number;brokerGid:number;configSha256:string}>,profileOwner:Readonly<{uid:number;gid:number}>={uid:0,gid:0}):Promise{ +export async function prepareGrokWorkerAttestation(input:Readonly<{profilePath:string;eventsPath:string;profileSha256:string;workerUid:number;brokerGid:number;configSha256:string}>,profileOwner:Readonly<{uid:number;gid:number;sharedTmpRoots?:readonly string[]}>={uid:0,gid:0}):Promise{ if(input.eventsPath!==grokWorkerEventsPathFor(input.profilePath))throw new Error("Grok worker sandbox events must be read from $GROK_HOME/sessions/sandbox-events.jsonl"); const profile=await secureOpen(input.profilePath,profileOwner.uid,profileOwner.gid,0o444,65_536);let bytes:Buffer|undefined;let denyPaths:readonly string[]=[];try{bytes=await profile.readFile();denyPaths=parseGrokWorkerSandboxProfile(bytes,input.profileSha256);}catch{throw new Error("Grok worker isolation attestation unavailable");}finally{bytes?.fill(0);await profile.close();} + // The launcher exports TMPDIR=/tmp; the profile lives at /.grok/sandbox.toml. + if(path.basename(path.dirname(input.profilePath))!==".grok")throw new Error("Grok worker isolation attestation unavailable"); + await verifyGrokWorkerTmp(path.dirname(path.dirname(input.profilePath)),input.workerUid,profileOwner.sharedTmpRoots); await verifyGrokWorkerHome(path.dirname(input.profilePath),input.configSha256); const events=await secureOpen(input.eventsPath,input.workerUid,input.brokerGid,0o640,16*1024*1024);try{const stat=await events.stat();return{dev:Number(stat.dev),ino:Number(stat.ino),size:Number(stat.size),denyPaths};}finally{await events.close();} } diff --git a/src/runtime/grokWorkerAttestationChecks.test.ts b/src/runtime/grokWorkerAttestationChecks.test.ts index 694391c..b5568a6 100644 --- a/src/runtime/grokWorkerAttestationChecks.test.ts +++ b/src/runtime/grokWorkerAttestationChecks.test.ts @@ -61,17 +61,40 @@ test("refuses events that change while they are being read", async (t) => { assert.ok(reading.mock.callCount() >= 1); }); -test("prepare refuses a worker home that fails attestation even when profile and events are valid", async (t) => { - // Run as a non-root owner so the profile and events legs pass; the home leg - // (root-owned, read-only config) cannot, and must be what refuses. - const home = await mkdtemp(path.join(os.tmpdir(), "daimon-guard-home-")); - t.after(() => rm(home, { recursive: true, force: true })); +test("prepare refuses a worker home that fails attestation even when profile, temp and events are valid", async (t) => { + // Run as a non-root owner so the profile, temp and events legs pass; the home + // leg (root-owned, read-only config) cannot, and must be what refuses. + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-guard-home-")); + t.after(() => rm(root, { recursive: true, force: true })); + const home = path.join(root, ".grok"); + await mkdir(path.join(home, "sessions"), { recursive: true }); + await mkdir(path.join(root, "tmp"), { mode: 0o700 }); const profile = path.join(home, "sandbox.toml"); const text = '[profiles.daimon-strict]\nextends = "strict"\nrestrict_network = true\ndeny = []\n'; await writeFile(profile, text); await chmod(profile, 0o444); - await mkdir(path.join(home, "sessions")); const events = path.join(home, "sessions", "sandbox-events.jsonl"); await writeFile(events, ""); await chmod(events, 0o640); const input = { profilePath: profile, eventsPath: events, profileSha256: createHash("sha256").update(text).digest("hex"), workerUid: self.uid, brokerGid: self.gid, configSha256: "0".repeat(64) }; - await assert.rejects(prepareGrokWorkerAttestation(input, { uid: self.uid, gid: Number((await stat(profile)).gid) }), /attestation unavailable/u); + await assert.rejects(prepareGrokWorkerAttestation(input, { uid: self.uid, gid: Number((await stat(profile)).gid), sharedTmpRoots: [] }), (error: Error) => error.message === "Grok worker isolation attestation unavailable"); +}); + +test("prepare refuses a worker without a private temp directory before the home check", async (t) => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-guard-tmp-")); + t.after(() => rm(root, { recursive: true, force: true })); + const grokHome = path.join(root, ".grok"); + await mkdir(path.join(grokHome, "sessions"), { recursive: true }); + const profile = path.join(grokHome, "sandbox.toml"); + const text = '[profiles.daimon-strict]\nextends = "strict"\nrestrict_network = true\ndeny = []\n'; + await writeFile(profile, text); await chmod(profile, 0o444); + const events = path.join(grokHome, "sessions", "sandbox-events.jsonl"); + await writeFile(events, ""); await chmod(events, 0o640); + const input = { profilePath: profile, eventsPath: events, profileSha256: createHash("sha256").update(text).digest("hex"), workerUid: self.uid, brokerGid: self.gid, configSha256: "0".repeat(64) }; + const owner = { uid: self.uid, gid: Number((await stat(profile)).gid) }; + // No /tmp: the temp leg refuses (the home leg would refuse too, with a different message). + await assert.rejects(prepareGrokWorkerAttestation(input, owner), /temp isolation attestation unavailable/u); + // A profile outside /.grok cannot name the launcher's TMPDIR home. + await mkdir(path.join(root, "elsewhere", "sessions"), { recursive: true }); + const stray = { ...input, profilePath: path.join(root, "elsewhere", "sandbox.toml"), eventsPath: path.join(root, "elsewhere", "sessions", "sandbox-events.jsonl") }; + await writeFile(stray.profilePath, text); await chmod(stray.profilePath, 0o444); await writeFile(stray.eventsPath, ""); await chmod(stray.eventsPath, 0o640); + await assert.rejects(prepareGrokWorkerAttestation(stray, owner), (error: Error) => /attestation unavailable/u.test(error.message) && !/temp/u.test(error.message)); }); diff --git a/src/runtime/grokWorkerTmpAttestation.test.ts b/src/runtime/grokWorkerTmpAttestation.test.ts new file mode 100644 index 0000000..e05016a --- /dev/null +++ b/src/runtime/grokWorkerTmpAttestation.test.ts @@ -0,0 +1,53 @@ +import assert from "node:assert/strict"; +import { chmod, mkdir, mkdtemp, rm, symlink } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { assertGrokWorkerTmpEntries, verifyGrokWorkerTmp } from "./grokWorkerTmpAttestation.js"; + +const worker = 2200; +const dir = (mode: number, uid: number, gid: number, kind: "dir" | "link" | "file" = "dir") => ({ + uid, gid, mode: (kind === "dir" ? 0o040000 : kind === "link" ? 0o120000 : 0o100000) | mode, + isDirectory: () => kind === "dir", isSymbolicLink: () => kind === "link" +}); +type Entry = ReturnType; +const good = (): { privateTmp: Entry | undefined; shared: (Entry | undefined)[] } => ({ privateTmp: dir(0o700, worker, worker), shared: [dir(0o1774, 0, 2000), dir(0o1774, 0, 2000)] }); + +test("accepts private worker temp and shared temp roots the worker cannot open or write", () => { + assert.doesNotThrow(() => assertGrokWorkerTmpEntries(good(), worker)); + assert.doesNotThrow(() => assertGrokWorkerTmpEntries({ ...good(), shared: [dir(0o1770, 0, 2000), dir(0o700, 0, 0)] }, worker)); +}); + +test("refuses shared /tmp or /var/tmp a worker could traverse, write, or own through its group", () => { + const refusals: Record = { + "shared 1777 (default /tmp)": { ...good(), shared: [dir(0o1777, 0, 0), dir(0o1774, 0, 2000)] }, + "shared other search": { ...good(), shared: [dir(0o1774, 0, 2000), dir(0o1775, 0, 2000)] }, + "shared other write": { ...good(), shared: [dir(0o1776, 0, 2000), dir(0o1774, 0, 2000)] }, + "shared owned by a worker group": { ...good(), shared: [dir(0o1774, 0, worker), dir(0o1774, 0, 2000)] }, + "shared owned by the org user": { ...good(), shared: [dir(0o1774, 2000, 2000), dir(0o1774, 0, 2000)] }, + "shared missing": { ...good(), shared: [undefined, dir(0o1774, 0, 2000)] }, + "shared symlink": { ...good(), shared: [dir(0o777, 0, 0, "link"), dir(0o1774, 0, 2000)] }, + "private missing": { ...good(), privateTmp: undefined }, + "private owned by another worker": { ...good(), privateTmp: dir(0o700, worker + 1, worker + 1) }, + "private group readable": { ...good(), privateTmp: dir(0o750, worker, worker) }, + "private symlink": { ...good(), privateTmp: dir(0o700, worker, worker, "link") }, + "private is a file": { ...good(), privateTmp: dir(0o600, worker, worker, "file") } + }; + for (const [label, entries] of Object.entries(refusals)) assert.throws(() => assertGrokWorkerTmpEntries(entries, worker), /temp isolation attestation unavailable/u, label); +}); + +test("checks the real private temp directory under the worker home and the given shared roots", async (t) => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-grok-tmp-")); + t.after(() => rm(root, { recursive: true, force: true })); + const uid = process.getuid?.() ?? 0; + const home = path.join(root, "home"), shared = path.join(root, "shared"); + await mkdir(path.join(home, "tmp"), { recursive: true }); await mkdir(shared); + await chmod(path.join(home, "tmp"), 0o700); + // A shared root owned by the test user is refused (not root-owned) — as is a symlinked private temp. + await assert.rejects(verifyGrokWorkerTmp(home, uid, [shared]), /temp isolation attestation unavailable/u); + await rm(path.join(home, "tmp"), { recursive: true }); await symlink(shared, path.join(home, "tmp")); + await assert.rejects(verifyGrokWorkerTmp(home, uid, []), /temp isolation attestation unavailable/u); + await rm(path.join(home, "tmp")); await mkdir(path.join(home, "tmp"), { mode: 0o700 }); + await verifyGrokWorkerTmp(home, uid, []); +}); diff --git a/src/runtime/grokWorkerTmpAttestation.ts b/src/runtime/grokWorkerTmpAttestation.ts new file mode 100644 index 0000000..4d2cf7b --- /dev/null +++ b/src/runtime/grokWorkerTmpAttestation.ts @@ -0,0 +1,48 @@ +import type { Stats } from "node:fs"; +import { lstat } from "node:fs/promises"; +import path from "node:path"; + +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; + +type Entry = Pick & Readonly<{ isDirectory(): boolean; isSymbolicLink(): boolean }>; +const HOME = GROK_ENGINE_BROKER.worker.home; + +/** + * Temp-directory isolation for one worker, checked before every turn. + * + * Grok 1.0.34's strict profile grants shared `/tmp` and `/var/tmp` + * read-write, and it refuses to start when either (or any ancestor of a + * granted path) is in `deny` — verified live: `deny = ["/tmp"]`, + * `["/var/tmp"]`, `["/run"]`, `["/etc"]` all fail with "could not apply the + * sandbox profile", while `["/tmp/sub"]` works. So the kernel profile cannot + * keep evaluator temp files away from the worker. Unix modes can, because the + * launcher drops the worker to its own uid/gid with no supplementary groups: + * + * - shared temp roots are root-owned, owned by a group below the worker range, + * and give "other" at most read (Grok opens the directory; without search + * the worker can list names but cannot open, stat, or create anything); a + * deployment that lets workers traverse or write them is refused; + * - the worker's own `/tmp` (the launcher's compiled `TMPDIR`, which + * strict grants read-write) is a real directory owned by the worker with no + * group or other access. + * + * Pure so every refusal is testable without root. + */ +export function assertGrokWorkerTmpEntries(entries: Readonly<{ privateTmp: Entry | undefined; shared: readonly (Entry | undefined)[] }>, workerUid: number): void { + const shared = HOME.sharedTmp; + for (const entry of entries.shared) { + if (entry === undefined || !entry.isDirectory() || entry.isSymbolicLink() || entry.uid !== shared.uid || entry.gid >= shared.maxGroupExclusive || (Number(entry.mode) & 0o007 & ~shared.otherMode) !== 0) throw unavailable(); + } + const own = entries.privateTmp; + if (own === undefined || !own.isDirectory() || own.isSymbolicLink() || own.uid !== workerUid || (Number(own.mode) & 0o077) !== 0) throw unavailable(); +} + +export async function verifyGrokWorkerTmp(workerHome: string, workerUid: number, sharedRoots: readonly string[] = HOME.sharedTmp.paths): Promise { + const inspect = async (file: string): Promise => { try { return await lstat(file); } catch { return undefined; } }; + assertGrokWorkerTmpEntries({ + privateTmp: await inspect(path.join(workerHome, HOME.privateTmp.relativeToWorkerHome)), + shared: await Promise.all(sharedRoots.map(inspect)) + }, workerUid); +} + +const unavailable = (): Error => new Error("Grok worker temp isolation attestation unavailable"); From faae4cdd2a760d25ebdef7e0a17c9171f09e295d Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:39:47 +0200 Subject: [PATCH 061/124] build: rebuild native engine broker artifacts with the worker TMPDIR --- src/contracts/runtimeContractManifest.ts | 6 +++--- .../artifacts/daimon-engine-broker-arm64 | Bin 67456 -> 67456 bytes ...daimon-engine-broker-arm64.provenance.json | 2 +- .../native/artifacts/daimon-engine-broker-x64 | Bin 55392 -> 55392 bytes .../daimon-engine-broker-x64.provenance.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 2235701..e56f6ae 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -129,9 +129,9 @@ export const GROK_ENGINE_BROKER = { projectionVersion: "noopolis.daimon.grok-broker-projection.v1", slotPreflightVersion: "noopolis.daimon.grok-slot-preflight.v2", artifacts: { - sourceSha256: "36f60689f0a8af0e3108f5f53d78ed52b7d4b6f934c75b6184606dfa82bc741e", - x64Sha256: "36dc76b134eb59cf5a6720b6f94228eb279108e20ea3343fa6efd9ffcb60a4d3", - arm64Sha256: "c93216cc6fa4ca50dc404fe41e68da9150a869b14f46eb42484ae77c3aa400a9" + sourceSha256: "27fcdc8ffb6946e391dad039f695968231d1ae435c6a38dd9bb8c7fb00ed18b9", + x64Sha256: "ff83efc37c77c4ee920080d0eedfca497b504a561c876348af0fec25142a5f9c", + arm64Sha256: "25d37be0d294529b3466d73c0d879d18c7850c2d24450a7a9cc11d28b9a4cf1e" } } as const; export const AGY_SUBSCRIPTION_REALM = { diff --git a/src/runtime/native/artifacts/daimon-engine-broker-arm64 b/src/runtime/native/artifacts/daimon-engine-broker-arm64 index daeaf6e3c64540ced18ab7c60f8e232824e7ba86..712890f0a30f784d1246975a870e3a558743b823 100755 GIT binary patch delta 2458 zcmY*b32;;A5&r-88dDy4#C6k^NV-5X2I+0nVSFw<&^_3}#n?#d` z4TOBiZG%h23XyXm7qO8yc%Wy)0on`88^-7v|4SP)MfyGP8Ft}sa!xn>zW>KLIh4Kt zlg33F@E^^6Hx6N2NoFiP4?R^z`bRiiRa5-Fby+5|LyjSH$InJDol%sugqatEN>||r z@h66Y%lnCUStRWGQ`CE7UI$9mjvP`+Dne%|qtxTrv8cG^3R0omR@}EmV$LSy`8O(I zsM=1waIU(9o`hS~zHRzaQbCCvBq()wi>HKCs4%UFBA^&h2$dbrqAyTPD$XQX=fa|3 z4xwyEPS3fxHLj7TErMH*Y!s|Tkoss&s-cin2!iZ5Ez1rqdU-MW0(ktN<&`w0#8nH9XgJ`R{KbQO0n|V=D+`no!piQ|S3^mn4R|R~hrkGB_ zftt*i6C=TJSNU9vPDWWs*@0tR9_Kez0Tv1t@O;5nK+5(}gABJl?2I2|18i&%| zP|Df|OPLz=Sd{#5DT_6fu^7~&Q7*a4*f~!bi$Yz6@_ML@jabXsm4bENwHS4*y`i3c zZPa-0#~G|=v6J=eQl{j+A1hGL&Jhdyhc-FUdq1Gd!eDouuJ*I$Xm;ndXeMG7Y|X1( zaycV`a`t6Rl2<^RqS&mPnL%I2K4})%H(l{;UetQ$#Y`W}))|~e4f`rB-McWb-s``m zxH&(VCkI1fhA)SZF&iOl`v3UU72ZYIRD{?1s5pe{Q&xBnVUr-$>(=Qho4^IesUh5A z<0x3`Q&zS+hi~_Wfl10@slZH_WE09IkHF4_1h!xo*!-}_c=1tW$YIlOW|Q~=^+q{( zr(5m4^>mzf(G>5!oSoqPY9P^DiVYkR(|y@`@nbGwk@$#;Fbg4%P&^RfF2iUxp;sCd z#mW7=(6^!qgRAvA;YUl*ZYh_Hcz8m+*0}>%Z)mS|?nP*`Y%WXYte6&Qua2b^4{xPy ztNI}7Pp#@ZP~QsE7JEiJ#>iHYC-}>#SCMtmL{839k&+Doh_gOU$DzY&s%!p6arY74 ztvYiRPrBbaXB8GJ_e8i=4KO}SS-o*nO z;ZyOUM}nuI%+{DdM7Df4e!A0|zH}4@ZCY9fhirDb2%p+4YTZpNbeDDRXJD|KQbK6y zDVf!`L^DAz2Py8#XRzH~Mf>59-6R$B#LLRVVJr*}vTqbF*wfPkBICg=cr!ozp}R^| z7`tyCmhBe9SwhBTLMqA$(OpEHPgenJ3*#&2yWxiIji!79CF6P(f1-4sp(Tz9Dmm!H zc(RBS4_EO~g~bH_1JjM!)D9OKjr0^;Lp>FwCM}g9uW2*A2pvt9j4uW42ZMrk6dT3d zgfV&rf(=BjC92__rj4q6N}LdCdZ_kZ!*LYEd-tT8zrL|{&-Ak597rP9)YyDChAF$# zt@5rXBMz`OXF1nD5d5bq;_MW)S7LFY77MjK5b)0r1l%I7t-CFxcT-I8U?Fg_IhUS= zkD5)J-p2OjvuE`{Jdq<4HuPMej&s8z9w^+C!QGJ4Vv+h*SW&ggz5qQfFVW9|wsuPI zUsv2=F%>skhX))713ceaLqCG)Rtuekf3;@PcR=0t7`+bPZp)UA<10G;ESaO&hWw;; z`G3-e+xsd8xfdXev;(~DCV!~CIxo?=Wou)bqb{$vr0)m2{VBUI(E;L9l=C?0G{3>! zz%dw@Rg_#DWcs%hr5bT7;)*2P%AmjX?h!S8(;ZgPlDM@$SMH1`TB+9bM(!Y*Qyo#%B>WE?jHAE+ delta 2388 zcmYjT4NP0t6~6a9Lu?=kFUI^6|B*UzC}4g9DL*#0`OCVNF@%=35w_??S5henkSvNc zo(=0VQl;FTv`&+>A?v7SYni&4t?@QiXiF5W-9*{Qx@w#3E^XSS-9RNj49MMiCX77k zyzidx+;`4-_kH);^7AVBdDZZ=X80(HU&HLLZ*G}8xazE9I7H9pJQx*ORMduSoCioE zDKio>t@uWRiWMN|K`!niZ}h;SydnAwtmF+-fB59q9FdwJWbDHG7E?EUGW=bWiPA%G z!F-3x;WPQaNurm+?L|3iIsu0&%=8_2v7)N*w~iH=$gT1Q5;^~N;__RZXQT`rpW&Sa z?04`S+(P!aSWvJa#M9L|VX9!viJ`yb?h zDB3>cJP#t5w9dJevITH!_f|nJfb@q<>E4~BToB~eNm*{yVpM6yV}(Z^@_%h$+ntK2 zm}|ubS|z1G+|RLk(cg%^oUg>Qtv>WMN<8joS(T;|l(<=mXIbmer%D`*Rh2?%89ZK9 zNPRF=l|%jTT2&(*f`3=-#5c~a&Ov@xwI2EQ>YYMqFAP-g?N1_PPJ_0mu9)rc6*D!C zH=wCw#q7Gl#u9OyfOgqqW1}NBro!=H)W$@_6tqr9346|0!jf^^U01{YW!W5D&smE( zYuTsz)ZqW~UZ`c)Cu-Rz8B!2)?_Nz#>}=FyXV*4w3C0V^b|zp#ispMI`C1aEdG1wS(<2e0R(1bH`EEJbZ9m0}_Z z5fx!}TnmaCk98VP=n1`1$9dZi?YUHjJbS6M19w2Q#}OIEexgSlk(1!Hm)bHFR?LWN zq+@wCqUhm?dt*gAd+&|CIBth2yDPf|bLds%Q3Li^t0J2diM%COMT+t+fywa*Jp%_F zmYRqE%LBbckyx6wMk#vC5pf82b{&p$>DEex>qid#avbwSV)GEvrnF;&S%;qPfSZmk z8i0LuV7s`80^Nwm!~>KW)$&XbeK`evtH6w@^313bSNuX%+`j-uXNEMn!~=$8<#0*I zR~(!fhi0eSFd?v&rR0kpj?Cqeo7hAbi4Zk5+NEs7?2rpCI$iWG%scIx?0DP}ISspA z7WxTzU0NE0K34@j1{YnHx-zAW69Giq>3iqwcKGNoQU z#+5A`UE&o=t4UQNYK9P z6SOB#yVhoWrbi&C7Bc^Z8h&5DRozTUn=%zXuisuNh--7xt^ZkApS$rcZ|mPg<~69W zg#pSIyH&v~GT6FS16mq#Tk`Hp{)~#Wb%@#bP_4fGP||3Z4y>}G>K@w* z{>GCu3Q|*tbmdDPh~f4|mEx9_XRd*-sfy0Rc$1xu!b(#P{T*aAe}&$LZ#L_tSMfug zf0{%nDp1+{UEvp+wN!cyldL6(A>9s2X~c7gX>W3#iAQW1VzEd{$m=-x1`=r#I1zDdlJhadY5Z+mMqHTU{0UsWi9cW= z#P7iPA-jewa=sngH9$g-Mfc+{7DGLgx(!=X5em@epsq)&OTpKvRw8&A(Av<-;M+ZB z-IEx>bnz<<@6VwffD1ht!c${V+gqqHAn`Y}pTc~PndXA7*Fp!OrniD#fxg}%S^%TH z$LInW`!eV zlK%3+?A1U0`S1VOMCy{idS*cV(xCcV07q)5BmDl64oauNb~Hox!90FQtf^dXv?lyb udMCW^=nWr4X==D>>9mHP55!coC~5s~Q6M{k?$VH+_)~0~!BP@P*B3}>U~mQlwH&pA)>4kG2hW&k zNyGY9f-M)N3Mys7@C-*Kh@+DrhZEy|`!iBc7nxHAZ_;g#j1B;ND5_A9X_VRM>KdfeRRZTvV)pO%4AEpe&l%r z`Gf{Wcacr>u{-`ieo4Q$BS!MWjd!j~AW@n-ekC~_J~)0NAvM&HxrDfAapn+mSGYRU zVI<>e)1=#pjh>#=LY@l0bdS+MQt6h-ZVwpO7e$NXuHOiFS=#7eUw-B+AXaKsu~xE zC`NAik}~@nb>hJ%P)sSb!k!fo1e2hg`jo?dF?c_4#e9x2m)wQ~Rj7~H_w|LkN;3MEe{6@-YB zXZV4GQmD#q86gNBxlNMW{1Sqq(CdsW_ zF;!U)HFfN^Ta@2nAl1yVTcqG|aZNr%TdR_|njm2+sgcn<{Lgqud?DP$?T!j9H;B8P zhAIF<0W{$we32}syOD(5c|;1d5zZs@Rp;o`Pf#T(RA$fMa~G7q{7cKkMtaeiKCm!B z&%joA;0Y?A>8_!E2VfW6-y(l2f3F-rqw$95s7Xav=%^8>^7IA9avf2}A;)nfG}^4X zI^c%)yv@Q|Qf`uK-jz6F+=VktCvL5ZsA zQZsn{2L%Cn9RrJdpoQi0BisVK0XRI3>Seg(R%)A?;y(cHGS2nd$H=WPW2!+_5)Zl) zyWNROVz*eG0QZh-@;l0h5JRZd*No~avqX5>m6(op0*&ytS%*z)f530l%*=Ve3&EJ#MM>tW|ec`v>Jsx8@MWqm87&^)iQ}Vcd2Vm zc@YLM<1Dr4Qq3-VZ7$53&TD8eT0T&!sj}B*0F?q+*gI9+?GkA5WqUbE5Naj)yc9S< zS`C6U@j&@yj*}6zeA>?kUxgSiG}1q{}}ESTpzQ+N)OR6udVH0>L!eC9z;1N+0+-?x~StdIfjj3g!Mg@U`KI0l5<| zm6>>%V39EZyz-?Y zvA89XGupv;g`Rb%J=*pxLJJxH3@Og&#QWdzjs`QOm`}5KABPpdy8(-%tL*%g;aE3! zU8|{wb@i_yw*&89cc@XOTxNIUv>e7s5-(xS0oaT)Y*L^AdIN5&@Frb@qCsO&smE`?y9c=kj&&47gm} z<_&?RZf>6PT3xu1%kAf}ueNUj#%u4|BVY#2Q;|GwuM|@rvc#`teAzm!);zlvkXIV{ zJmrp?rdWhczK`4UseK@-ZL@>rT~ZCg;EYk3Fw&b7RX*^8+~o+GBR|`ebC-H^i1YC$ z)7EDzcRl%?){T_v57d`z#gCVsO8k#TCcuc^cbRfOqNrJsZO>L4rZP(%VubVv$~1k{ zHB$4Nt*R+wdW6(wF^u}95GkbtJT`M4HYhKEmoLrD^d8SI+~4BLA)g7JT5eRt4mIeE zAj80bfzSks=_ZkWa=%MnWoRYGi7BrWApEG$uTIbx&EzI7-VzNOi$WM zZF$9Mh*`8I6p2fXPXZCGc@_uTUqlOhw+~vN ziP(BY=F^yOsiCVqoH=U>A=|>`vu7p{YdALl#p&Zq?%{bWGvnTM9430sv$-hF4lBQ! z<6a)PU}`hEAgFGEdGwi=?m~lDL4~$(zkI)Hd1SJ8Y!l8qZK3=Hda7KAIt~ry*}83z}jg`%b>yBLJD_`hfi}g zI0}~zS0-;yq;-d<+}87FLC>E9*BVu6LD@yG93E@!+^P)+YecB0jw9(fEoL2QCKhTv znvO|0jxHqw!VO2$`(e=X?zlt?_l3t2{^s2TLO!PC#JI#BOPzG$iR-lVWCkrfX-Vw) alVlTp^5ostZKpKFZHG1L3%8tniTnpMpn$Oe delta 4470 zcmb7I3vg3a8onoOXv%7CVq4nEqXDkkLRVUWwJ4MXg1vzNL19*QKyko|Qn9S9e?K(SbO8I1V4s;iYuZUGEYvHQ%j}H^Oyy ze`_Ee)R)tR>^wW{Wio`G9o9kyh8u?0CXog@Y2+*9V7P7M7()I^4WpkX`LtwofAFhD z=NrjT+VsE>VxeC=5F?Aj?>uNU5EETD?htX&vcko%`&Hpx-b333gCIC&xs>aUnNyVm zvmTKv&t*G#j60-g0;j8__7p)-$|Tbd@NXy$>rsm5BbAG0VS_&(w0=ZlHv-)lSCTp|w)yP(g^;wG5|a zmm-e&ZEoe8{HR*w{WoJ;K>FMS?RXrkwYB>tVU!>=F-)8ala*E(3$>~BP}Am4DMOu) zi5#3MWypaJad|OBTm7kAO^~q!TO@~veIB96y#5bqc3(7z+w2B^G+Xrl0YDQz!r4K^ z#rKL?ehiUL}cGy;)j-%eqhhZ_#UN_LuFE46IO@Sf2XISgr2u&-I13RPr-X| z#WR#evm67x1h6Cc@@>jb%CG8?(>m*19}P5V;LQ}Q9@B9M&a zi_YYW$=AfH4kOQ#@~L_ZvWU1U27t#9;YCoJW!m|bOWXcsGvSg4^7DQHh-bHRAwV_8jek*{^PQqJp{Ey+iq$dOdz5Ht;}+8 zi!=`hQZ;<5qk(Ao9dgjGfwHdvr)4?!Fds^C$S=ik{CaF8S@~A>?IEoOK^~L5r~DEZ zIXsj;IdN#_8EEh_^B@*t6F7XIGP$xP+ez0<9PIt+2X0S~s7Xu0HAK(Daw&V%9T<0e z6SM4SaW~!E@Pa_P_p^*P;oGf;vAg`z)WCP5zl<^F@?B3915+`=iuj>%D2YPTfk+3u1;mEdg2DX(=_9|Um zIqhpOA~wjWTRHE)_jGaf|GC}d;2g=GBZB5mrk@Y^BwLv!FII00y@7N)mWszzeO z9H->roU>e`g^xOsl|AkdaP8FQN4d;AIkZfwk7sgiE%xem0YPH{j;GNlCmDov^sCFxzE?*02#eNub zwER*8y_8=`YU;(CDf*49^g4T8+ke*2vHHnLZ7^2-2m?9f)tkrvXr&BRwC2d8c>RKI zDT1;KYGwBfDG=1>MG39jN)Qd)S{0YrlUVve8A0~3_XIq>%ROgM-c>OSO_!uiQ@gPkn=y6veiPE&m zb4ZZ>dGb24j%G|5K&t5IDTDhL>u(nRJAdWL*dhIlwL~iU3fy$blytI|DpLko0=OXg zJ-+Wcq=Lpm`tg*3{U1fayTe5vyNsxD_wf7bF?wN28u^48-GyWyb-EuS57VG~G#N`{ z?g56ZWAwDUVAeYQe)Z4B%b<+Q%H&40dB^m-+NpuY*g`}j2z{}?#;sj&(;a+Vfn@xx zB^|4z8?iTMKju(c zps@VvPg>DNA)7VRTmu@B$F8uBS=7rNm<2K8b^3;0nda(y5D@pYJeRtr7n5z(VA2&K4;tj zM6?!J^4acU`nqRGN>CTE-4s!1;*94EIfujLGsY95hPOUEEs4wsC;n;m;W}zCVtvLEK@#5?|XWDD5grK(V>*%qy_Y;}kzV6k`*aLb-mwbd9PiZ;Z z$gfOy==U4wrgb&M8!mirS28V4vo|)(C3Ml|mDIlV&hU`N#R=AqO2(GqQ&hp&MlWN} z(=}U9i-@`BL)UCe39s1p8u40c7?X^G;CYKNKWKX$V>Z;(@-}0mKs}%`sBI-54&3Ui( zahMByNk!()Dw41wk!Jlpv+o>;d!T+MNI%^1n6U+U^iu22w{q66V5|^K{-C=H^KfF0 z_Q2)=D+4Cb!#n5al+-eYv}+HSJ_pW1V2DtAa``s|7SOqo5ow8U-YV_ns&q{xCG9qF zx~rT4uml_v-52Ta{T!8jf?L7EJ*)z!09AfmLv*YGm>(E!d+p(JEx8$8VL-b4feYB9|(frtWQbCu*QpT;q?>^n49{5=7fVy6o2Uu+{Yyq%p8jGc5atrC) zF&@6mW#Fvm(%~o4)+W=p_m3aa{X4(=cm0(HOgp#2j98FslzHu_Ggrv$;<^+2J3LBoVTjl4B#1yDinzF~_bZ i-}rZgjyZlm%{!h)>yKNk^`~^j`u#e6G#oqr4*3t5$!GEa diff --git a/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json b/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json index f2bcf17..ff8244c 100644 --- a/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json +++ b/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json @@ -1 +1 @@ -{"version":"daimon.engine-broker-native-build.v1","architecture":"x64","target":"linux/amd64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:36f60689f0a8af0e3108f5f53d78ed52b7d4b6f934c75b6184606dfa82bc741e","binary_sha256":"sha256:36dc76b134eb59cf5a6720b6f94228eb279108e20ea3343fa6efd9ffcb60a4d3","install_path":"/opt/daimon/bin/daimon-engine-broker"} +{"version":"daimon.engine-broker-native-build.v1","architecture":"x64","target":"linux/amd64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:27fcdc8ffb6946e391dad039f695968231d1ae435c6a38dd9bb8c7fb00ed18b9","binary_sha256":"sha256:ff83efc37c77c4ee920080d0eedfca497b504a561c876348af0fec25142a5f9c","install_path":"/opt/daimon/bin/daimon-engine-broker"} From 8d24c50692e4bc0f40adfb486c44023f598312b0 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:40:41 +0200 Subject: [PATCH 062/124] fix: write tool-output spills group-readable so the agent's own Grok worker can read them --- src/runtime/toolResultSpill.test.ts | 18 +++++++++++++++++- src/runtime/toolResultSpill.ts | 18 ++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/runtime/toolResultSpill.test.ts b/src/runtime/toolResultSpill.test.ts index b83a587..a5f6c82 100644 --- a/src/runtime/toolResultSpill.test.ts +++ b/src/runtime/toolResultSpill.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { chmod, chown, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import test from "node:test"; @@ -150,3 +150,19 @@ test("the bound and the exemption list come from the environment, and a nonsense assert.deepEqual([...resolveExemptToolNames({})], []); assert.deepEqual([...resolveExemptToolNames({ [TOOL_RESULT_EXEMPT_ENV]: " mcp_a_b , mcp_c_d ," })], ["mcp_a_b", "mcp_c_d"]); }); + +test("spilled files are readable by the directory's (worker) group and never by other users", async () => { + await withDirectory(async (directory) => { + // What a deployment provisions for a brokered worker: setgid tool-output in the worker's group. + const groups = (process.getgroups?.() ?? []).filter((gid) => gid !== process.getgid?.()); + const workerGroup = groups[0]; + if (workerGroup !== undefined) await chown(directory, process.getuid?.() ?? -1, workerGroup).catch(() => undefined); + await chmod(directory, 0o2750); + const previous = process.umask(0o077); + let capped; + try { capped = await cap({ content: [{ type: "text", text: "x".repeat(200_000) }] }, { spillDirectory: directory }); } finally { process.umask(previous); } + const file = await stat(capped.spillPath!); + assert.equal(file.mode & 0o777, 0o640, "group-readable even under a restrictive umask, never other-readable"); + assert.equal(file.gid, (await stat(directory)).gid, "the file carries the directory's group"); + }); +}); diff --git a/src/runtime/toolResultSpill.ts b/src/runtime/toolResultSpill.ts index 6e993a2..b0a8d55 100644 --- a/src/runtime/toolResultSpill.ts +++ b/src/runtime/toolResultSpill.ts @@ -181,6 +181,19 @@ const notice = (input: Readonly<{ + ` or \`grep -n "" ${input.spillPath}\` — and do NOT repeat this tool call to see it: an identical call returns this same truncation.]`; }; +/** + * Spilled files are group-readable and never other-readable. + * + * A brokered Grok worker runs as its own uid and reads a spill with + * `read_file`, so a 0600 file owned by the runtime (uid 2000) was unreadable to + * the very agent the notice sends there. The group grant reaches exactly that + * agent's worker only when the deployment provisions `tool-output` as + * `: 2750` (setgid, so each file inherits the worker + * group; `GROK_ENGINE_BROKER.worker.home.spillDirectory`). A directory Daimon + * creates itself stays 0700, so for every other engine nothing new is exposed. + */ +export const SPILL_FILE_MODE = 0o640; + /** * Write the full payload where the agent can read it, atomically. * @@ -192,8 +205,9 @@ const writeSpill = async (directory: string, name: string, text: string): Promis await mkdir(directory, { recursive: true, mode: 0o700 }); const file = path.join(directory, name); const temporary = `${file}.${process.pid}.${Date.now()}.tmp`; - const handle = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); - try { await handle.writeFile(text, "utf8"); await handle.sync(); } finally { await handle.close(); } + const handle = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, SPILL_FILE_MODE); + // Explicit, so a restrictive umask cannot strip the group read an agent's sandboxed worker needs. + try { await handle.chmod(SPILL_FILE_MODE); await handle.writeFile(text, "utf8"); await handle.sync(); } finally { await handle.close(); } try { await rename(temporary, file); } catch (error) { await unlink(temporary).catch(() => undefined); throw error; } return file; }; From 45d9e411425930e04e9672b47da57d824d730d14 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:42:47 +0200 Subject: [PATCH 063/124] docs: document the Grok worker temp and spill provisioning contract --- src/runtime/AGENTS.md | 19 +++++++++++++++++++ src/runtime/native/AGENTS.md | 3 +++ 2 files changed, 22 insertions(+) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 1628dc5..a0d4517 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -160,6 +160,25 @@ every profile inside bubblewrap, where a non-empty `deny` list is enforced; `grokWorkerSandboxProfile.ts` renders those profile bytes. A worker-uid process can neither write, rename, nor unlink any of the root-owned files. +Temp and spill isolation (`grokWorkerTmpAttestation.ts`, checked before every +turn; `GROK_ENGINE_BROKER.worker.home.{privateTmp,sharedTmp,spillDirectory}`). +Grok 1.0.34's strict profile grants shared `/tmp` and `/var/tmp` read-write +and refuses to start if either, or any path equal to or above a base grant, is +in `deny` (verified: `/tmp`, `/var/tmp`, `/run`, `/etc`, `sessions` all fail; +`/tmp/sub` works), so the profile cannot hide evaluator temp files. Instead: +- the launcher exports `TMPDIR=/tmp` (strict adds TMPDIR to its + read-write grants; Python, Node and `mktemp` use it); provision it + `: 0700`; +- `/tmp` and `/var/tmp` must be `root: 1774`: + Grok needs to open the directory, but without search or write a worker can + only list names — `cat`/`read_file` get EACCES and it cannot create files. + `1770`/`1771` make Grok refuse the profile; `1775`/`1777` leak. Any non-root + process outside that group that needs temp space must get its own `TMPDIR`; +- spills (`toolResultSpill.ts`) are written `0640`; provision + `/tool-output` as `2000: 2750` (setgid) under a + runtime home the worker can traverse, so each spill carries that agent's + worker group and no other worker can read it. + `agySubscriptionRealm.ts` owns the one host-level private D-Bus/Secret Service realm, durable keyring lease, bounded unlock stdin, and cleanup. `agySubscriptionBootstrap.ts` owns only the interactive first-enrollment AGY diff --git a/src/runtime/native/AGENTS.md b/src/runtime/native/AGENTS.md index 3e7f8ae..8fd8f77 100644 --- a/src/runtime/native/AGENTS.md +++ b/src/runtime/native/AGENTS.md @@ -25,6 +25,9 @@ environment: `DAIMON_MCP_CAPABILITY` and `DAIMON_PROVIDER_CAPABILITY` (Grok config reads the proxy capability through `env_key`). `--auth-provider` mode remains for callers of the older contract. +It also exports `TMPDIR=/tmp`, the worker's private temp +directory, derived only from the root-owned registration. + Received descriptors carry `MSG_CMSG_CLOEXEC` and can already occupy fds 3-5, so `launch()` lifts prompt, capability, output, executable and status fds above 16 before `dup2`-ing them into place; a `dup2` onto itself keeps close-on-exec From 7439edbbaa6a8e7063d6e6bfe2fec90184d0f288 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:56:47 +0200 Subject: [PATCH 064/124] fix: attest every registered Grok worker's private temp and close the temp attestation test gaps --- src/runtime/grokEngineBroker.ts | 4 +- src/runtime/grokWorkerAttestation.ts | 12 +- .../grokWorkerAttestationChecks.test.ts | 26 ++++- src/runtime/grokWorkerTmpAttestation.test.ts | 106 ++++++++++++------ src/runtime/grokWorkerTmpAttestation.ts | 42 +++++-- 5 files changed, 136 insertions(+), 54 deletions(-) diff --git a/src/runtime/grokEngineBroker.ts b/src/runtime/grokEngineBroker.ts index aceb394..35fd7cc 100644 --- a/src/runtime/grokEngineBroker.ts +++ b/src/runtime/grokEngineBroker.ts @@ -9,7 +9,7 @@ import { acquireGrokBrokerRealmLease } from "./grokBrokerRealmLease.js"; import { grokBrokerWorkerConfigSha256 } from "./grokBrokerWorkerConfig.js"; import { parseGrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; import { runGrokEngineBrokerTurn, type GrokEngineBrokerTurnResult } from "./grokEngineBrokerTurn.js"; -import { createGrokWorkerIsolationGuard,prepareGrokWorkerAttestation } from "./grokWorkerAttestation.js"; +import { createGrokWorkerIsolationGuard,grokBrokerAttestationInput,prepareGrokWorkerAttestation } from "./grokWorkerAttestation.js"; import { createLedgeredGrokInferenceGrants, GrokInferenceGrantRefused, type GrokInferenceGrantRequest } from "./grokInferenceGrants.js"; export { EngineBrokerTurnFailure, type GrokEngineBrokerTurnResult } from "./grokEngineBrokerTurn.js"; @@ -30,7 +30,7 @@ export type GrokEngineBroker = Awaited> */ export async function startGrokEngineBroker(options: Readonly<{ grokCommand: string; nativeClient: string; credentialHome: string; turnStore: string; registrations: readonly GrokEngineBrokerRegistration[]; inferenceLedgerPath?: string }>) { const registrations = new Map(options.registrations.map((entry) => [entry.agentId, { ...entry, model: parseGrokBrokerModelPolicy(entry.model) }])); if (registrations.size !== options.registrations.length) throw new Error("engine broker registration conflict"); - const attestationFor = (registration: GrokEngineBrokerRegistration) => ({ ...registration, brokerGid: 2100, configSha256: grokBrokerWorkerConfigSha256(registration.model) }); + const attestationFor = (registration: GrokEngineBrokerRegistration) => grokBrokerAttestationInput(registration, [...registrations.values()], grokBrokerWorkerConfigSha256(registration.model)); const inferenceLedgerPath=options.inferenceLedgerPath;const grants=inferenceLedgerPath===undefined?undefined:createLedgeredGrokInferenceGrants(inferenceLedgerPath); const lease=await acquireGrokBrokerRealmLease(options.credentialHome);const authority = new DurableGrokBrokerCredentialAuthority(options.grokCommand, options.credentialHome);try{await authority.initialize();}catch(error){await lease.close();throw error;} let proxy:Awaited>;try{proxy=await startGrokBrokerProxy(authority,undefined,undefined,undefined,grants);}catch(error){await lease.close();throw error;}let mcp:Awaited>|undefined;try{mcp=await startEngineBrokerMcpFacade();for(const registration of registrations.values())await prepareGrokWorkerAttestation(attestationFor(registration));}catch(error){if(mcp)await mcp.close().catch(()=>undefined);await proxy.close();await lease.close();throw error;}if(!mcp)throw new Error("engine broker unavailable");const turns = new EngineBrokerTurnRegistry(options.turnStore); const active = new Map }>(); let closed = false; const facade = mcp; diff --git a/src/runtime/grokWorkerAttestation.ts b/src/runtime/grokWorkerAttestation.ts index d10d01f..60bc8fc 100644 --- a/src/runtime/grokWorkerAttestation.ts +++ b/src/runtime/grokWorkerAttestation.ts @@ -4,7 +4,7 @@ import { lstat,open } from "node:fs/promises"; import path from "node:path"; import { verifyGrokWorkerHome } from "./grokWorkerHomeAttestation.js"; -import { verifyGrokWorkerTmp } from "./grokWorkerTmpAttestation.js"; +import { grokWorkerHomeForProfile, verifyGrokWorkerTmp, type GrokWorkerTmpOptions, type GrokWorkerTmpWorker } from "./grokWorkerTmpAttestation.js"; import { grokWorkerEventsPathFor } from "./grokWorkerSandboxProfile.js"; /** @@ -62,15 +62,19 @@ export function parseGrokWorkerSandboxProfile(bytes:Uint8Array,profileSha256:str * 1.0.34), and a root-owned read-only worker home whose `config.toml` hashes to * the declared renderer output (`grokWorkerHomeAttestation.ts`). */ -export async function prepareGrokWorkerAttestation(input:Readonly<{profilePath:string;eventsPath:string;profileSha256:string;workerUid:number;brokerGid:number;configSha256:string}>,profileOwner:Readonly<{uid:number;gid:number;sharedTmpRoots?:readonly string[]}>={uid:0,gid:0}):Promise{ +export async function prepareGrokWorkerAttestation(input:Readonly<{profilePath:string;eventsPath:string;profileSha256:string;workerUid:number;brokerGid:number;configSha256:string;registeredWorkers?:readonly Readonly<{profilePath:string;workerUid:number}>[]}>,profileOwner:Readonly<{uid:number;gid:number;tmp?:GrokWorkerTmpOptions}>={uid:0,gid:0}):Promise{ if(input.eventsPath!==grokWorkerEventsPathFor(input.profilePath))throw new Error("Grok worker sandbox events must be read from $GROK_HOME/sessions/sandbox-events.jsonl"); const profile=await secureOpen(input.profilePath,profileOwner.uid,profileOwner.gid,0o444,65_536);let bytes:Buffer|undefined;let denyPaths:readonly string[]=[];try{bytes=await profile.readFile();denyPaths=parseGrokWorkerSandboxProfile(bytes,input.profileSha256);}catch{throw new Error("Grok worker isolation attestation unavailable");}finally{bytes?.fill(0);await profile.close();} // The launcher exports TMPDIR=/tmp; the profile lives at /.grok/sandbox.toml. - if(path.basename(path.dirname(input.profilePath))!==".grok")throw new Error("Grok worker isolation attestation unavailable"); - await verifyGrokWorkerTmp(path.dirname(path.dirname(input.profilePath)),input.workerUid,profileOwner.sharedTmpRoots); + // Every registered worker's private temp is attested, not only this one's (a sibling's open temp is a shared channel). + const workers=[{profilePath:input.profilePath,workerUid:input.workerUid},...(input.registeredWorkers??[])].map((worker)=>({home:grokWorkerHomeForProfile(worker.profilePath),uid:worker.workerUid})); + if(workers.some((worker)=>worker.home===undefined))throw new Error("Grok worker isolation attestation unavailable"); + await verifyGrokWorkerTmp(workers as readonly GrokWorkerTmpWorker[],profileOwner.tmp); await verifyGrokWorkerHome(path.dirname(input.profilePath),input.configSha256); const events=await secureOpen(input.eventsPath,input.workerUid,input.brokerGid,0o640,16*1024*1024);try{const stat=await events.stat();return{dev:Number(stat.dev),ino:Number(stat.ino),size:Number(stat.size),denyPaths};}finally{await events.close();} } +/** The per-turn attestation input for one registration, carrying every registered worker so sibling temp is attested too. */ +export const grokBrokerAttestationInput=>(registration:T,registrations:readonly Readonly<{profilePath:string;workerUid:number}>[],configSha256:string)=>({...registration,brokerGid:2100,configSha256,registeredWorkers:registrations.map((entry)=>({profilePath:entry.profilePath,workerUid:entry.workerUid}))}); /** * The accepted `ProfileApplied` line of one turn, as an absolute byte range of * the events file plus its digest. diff --git a/src/runtime/grokWorkerAttestationChecks.test.ts b/src/runtime/grokWorkerAttestationChecks.test.ts index b5568a6..8bee20e 100644 --- a/src/runtime/grokWorkerAttestationChecks.test.ts +++ b/src/runtime/grokWorkerAttestationChecks.test.ts @@ -7,6 +7,7 @@ import test, { mock } from "node:test"; import { GrokWorkerAttestationFailure, + grokBrokerAttestationInput, prepareGrokWorkerAttestation, verifyGrokWorkerAttestation, type GrokWorkerAttestationSnapshot @@ -75,7 +76,7 @@ test("prepare refuses a worker home that fails attestation even when profile, te const events = path.join(home, "sessions", "sandbox-events.jsonl"); await writeFile(events, ""); await chmod(events, 0o640); const input = { profilePath: profile, eventsPath: events, profileSha256: createHash("sha256").update(text).digest("hex"), workerUid: self.uid, brokerGid: self.gid, configSha256: "0".repeat(64) }; - await assert.rejects(prepareGrokWorkerAttestation(input, { uid: self.uid, gid: Number((await stat(profile)).gid), sharedTmpRoots: [] }), (error: Error) => error.message === "Grok worker isolation attestation unavailable"); + await assert.rejects(prepareGrokWorkerAttestation(input, { uid: self.uid, gid: Number((await stat(profile)).gid), tmp: { sharedRoots: [root], sharedOwnerUid: self.uid, firstWorkerUid: self.uid } }), (error: Error) => error.message === "Grok worker isolation attestation unavailable"); }); test("prepare refuses a worker without a private temp directory before the home check", async (t) => { @@ -98,3 +99,26 @@ test("prepare refuses a worker without a private temp directory before the home await writeFile(stray.profilePath, text); await chmod(stray.profilePath, 0o444); await writeFile(stray.eventsPath, ""); await chmod(stray.eventsPath, 0o640); await assert.rejects(prepareGrokWorkerAttestation(stray, owner), (error: Error) => /attestation unavailable/u.test(error.message) && !/temp/u.test(error.message)); }); + +test("prepare refuses the current turn when a sibling registered worker's private temp is 0777", async (t) => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-guard-sibling-")); + t.after(() => rm(root, { recursive: true, force: true })); + const make = async (name: string) => { + const home = path.join(root, name), grok = path.join(home, ".grok"); + await mkdir(path.join(grok, "sessions"), { recursive: true }); await mkdir(path.join(home, "tmp"), { mode: 0o700 }); + return { home, profile: path.join(grok, "sandbox.toml"), events: path.join(grok, "sessions", "sandbox-events.jsonl") }; + }; + const own = await make("own"), sibling = await make("sibling"); + const text = '[profiles.daimon-strict]\nextends = "strict"\nrestrict_network = true\ndeny = []\n'; + await writeFile(own.profile, text); await chmod(own.profile, 0o444); + await writeFile(own.events, ""); await chmod(own.events, 0o640); + const registration = { profilePath: own.profile, eventsPath: own.events, profileSha256: createHash("sha256").update(text).digest("hex"), workerUid: self.uid, workspace: "/w" }; + const input = grokBrokerAttestationInput(registration, [registration, { profilePath: sibling.profile, workerUid: self.uid }], "0".repeat(64)); + assert.deepEqual(input.registeredWorkers.map((entry) => entry.profilePath), [own.profile, sibling.profile]); + const seams = { uid: self.uid, gid: Number((await stat(own.profile)).gid), tmp: { sharedRoots: [root], sharedOwnerUid: self.uid, firstWorkerUid: self.uid } }; + await chmod(root, 0o700); + // Sibling well provisioned: temp passes and the (root-only) home leg is what refuses. + await assert.rejects(prepareGrokWorkerAttestation({ ...input, brokerGid: self.gid }, seams), (error: Error) => error.message === "Grok worker isolation attestation unavailable"); + await chmod(path.join(sibling.home, "tmp"), 0o777); + await assert.rejects(prepareGrokWorkerAttestation({ ...input, brokerGid: self.gid }, seams), /temp isolation attestation unavailable/u); +}); diff --git a/src/runtime/grokWorkerTmpAttestation.test.ts b/src/runtime/grokWorkerTmpAttestation.test.ts index e05016a..a5a728c 100644 --- a/src/runtime/grokWorkerTmpAttestation.test.ts +++ b/src/runtime/grokWorkerTmpAttestation.test.ts @@ -1,53 +1,87 @@ import assert from "node:assert/strict"; -import { chmod, mkdir, mkdtemp, rm, symlink } from "node:fs/promises"; +import { chmod, mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import test from "node:test"; -import { assertGrokWorkerTmpEntries, verifyGrokWorkerTmp } from "./grokWorkerTmpAttestation.js"; +import { assertGrokWorkerTmpEntries, verifyGrokWorkerTmp, type GrokWorkerTmpOptions } from "./grokWorkerTmpAttestation.js"; -const worker = 2200; -const dir = (mode: number, uid: number, gid: number, kind: "dir" | "link" | "file" = "dir") => ({ - uid, gid, mode: (kind === "dir" ? 0o040000 : kind === "link" ? 0o120000 : 0o100000) | mode, - isDirectory: () => kind === "dir", isSymbolicLink: () => kind === "link" +const worker = 2200, sibling = 2201; +type Entry = Readonly<{ uid: number; gid: number; mode: number; isDirectory(): boolean }>; +const dir = (mode: number, uid: number, gid: number): Entry => ({ uid, gid, mode: 0o040000 | mode, isDirectory: () => true }); +const file = (mode: number, uid: number, gid: number): Entry => ({ uid, gid, mode: 0o100000 | mode, isDirectory: () => false }); +type Entries = { privateTmps: { uid: number; entry: Entry | undefined }[]; shared: (Entry | undefined)[] }; +const good = (): Entries => ({ privateTmps: [{ uid: worker, entry: dir(0o700, worker, worker) }, { uid: sibling, entry: dir(0o700, sibling, sibling) }], shared: [dir(0o1774, 0, 2000), dir(0o1774, 0, 2000)] }); +const withShared = (shared: Entry | undefined): Entries => ({ ...good(), shared: [shared, dir(0o1774, 0, 2000)] }); +const withOwn = (entry: Entry | undefined, uid = worker): Entries => ({ ...good(), privateTmps: [{ uid, entry }, good().privateTmps[1]!] }); +const withSibling = (entry: Entry | undefined): Entries => ({ ...good(), privateTmps: [good().privateTmps[0]!, { uid: sibling, entry }] }); +const refused = /temp isolation attestation unavailable/u; + +test("accepts private worker temps and shared temp roots the workers cannot open or write", () => { + assert.doesNotThrow(() => assertGrokWorkerTmpEntries(good())); + // Per contract the shared group only has to be below the worker range: the broker group 2100 is fine. + assert.doesNotThrow(() => assertGrokWorkerTmpEntries(withShared(dir(0o1774, 0, 2100)))); + assert.doesNotThrow(() => assertGrokWorkerTmpEntries(withShared(dir(0o1770, 0, 2000)))); }); -type Entry = ReturnType; -const good = (): { privateTmp: Entry | undefined; shared: (Entry | undefined)[] } => ({ privateTmp: dir(0o700, worker, worker), shared: [dir(0o1774, 0, 2000), dir(0o1774, 0, 2000)] }); -test("accepts private worker temp and shared temp roots the worker cannot open or write", () => { - assert.doesNotThrow(() => assertGrokWorkerTmpEntries(good(), worker)); - assert.doesNotThrow(() => assertGrokWorkerTmpEntries({ ...good(), shared: [dir(0o1770, 0, 2000), dir(0o700, 0, 0)] }, worker)); +test("refuses shared temp roots that are missing, not directories, not root-owned, worker-grouped, or open to others", () => { + const cases: Record = { + "missing": withShared(undefined), + "regular file": withShared(file(0o1774, 0, 2000)), + "owned by the org user": withShared(dir(0o1774, 2000, 2000)), + "group 2200 (a worker group)": withShared(dir(0o1774, 0, 2200)), + "other search (1775)": withShared(dir(0o1775, 0, 2000)), + "other write (1776)": withShared(dir(0o1776, 0, 2000)), + "default /tmp (1777)": withShared(dir(0o1777, 0, 0)), + "no shared roots at all": { ...good(), shared: [] } + }; + for (const [label, entries] of Object.entries(cases)) assert.throws(() => assertGrokWorkerTmpEntries(entries), refused, label); }); -test("refuses shared /tmp or /var/tmp a worker could traverse, write, or own through its group", () => { - const refusals: Record = { - "shared 1777 (default /tmp)": { ...good(), shared: [dir(0o1777, 0, 0), dir(0o1774, 0, 2000)] }, - "shared other search": { ...good(), shared: [dir(0o1774, 0, 2000), dir(0o1775, 0, 2000)] }, - "shared other write": { ...good(), shared: [dir(0o1776, 0, 2000), dir(0o1774, 0, 2000)] }, - "shared owned by a worker group": { ...good(), shared: [dir(0o1774, 0, worker), dir(0o1774, 0, 2000)] }, - "shared owned by the org user": { ...good(), shared: [dir(0o1774, 2000, 2000), dir(0o1774, 0, 2000)] }, - "shared missing": { ...good(), shared: [undefined, dir(0o1774, 0, 2000)] }, - "shared symlink": { ...good(), shared: [dir(0o777, 0, 0, "link"), dir(0o1774, 0, 2000)] }, - "private missing": { ...good(), privateTmp: undefined }, - "private owned by another worker": { ...good(), privateTmp: dir(0o700, worker + 1, worker + 1) }, - "private group readable": { ...good(), privateTmp: dir(0o750, worker, worker) }, - "private symlink": { ...good(), privateTmp: dir(0o700, worker, worker, "link") }, - "private is a file": { ...good(), privateTmp: dir(0o600, worker, worker, "file") } +test("refuses a private temp that is missing, not a directory, owned by someone else, or has any group or other bit", () => { + const cases: Record = { + "missing": withOwn(undefined), + "regular file": withOwn(file(0o600, worker, worker)), + "owned by another worker": withOwn(dir(0o700, sibling, sibling)), + "group read (0740)": withOwn(dir(0o740, worker, worker)), + "other execute (0701)": withOwn(dir(0o701, worker, worker)), + "other write (0702)": withOwn(dir(0o702, worker, worker)), + "other read (0704)": withOwn(dir(0o704, worker, worker)), + "worker uid below the worker range": withOwn(dir(0o700, 2100, 2100), 2100), + "no workers at all": { ...good(), privateTmps: [] } }; - for (const [label, entries] of Object.entries(refusals)) assert.throws(() => assertGrokWorkerTmpEntries(entries, worker), /temp isolation attestation unavailable/u, label); + for (const [label, entries] of Object.entries(cases)) assert.throws(() => assertGrokWorkerTmpEntries(entries), refused, label); +}); + +test("a misprovisioned sibling worker's temp refuses the current worker's turn", () => { + assert.throws(() => assertGrokWorkerTmpEntries(withSibling(dir(0o777, sibling, sibling))), refused); + assert.throws(() => assertGrokWorkerTmpEntries(withSibling(dir(0o770, sibling, sibling))), refused); + assert.throws(() => assertGrokWorkerTmpEntries(withSibling(undefined)), refused); }); -test("checks the real private temp directory under the worker home and the given shared roots", async (t) => { +test("on a real filesystem: sibling 0777 temp, symlinked temps and symlinked shared roots are refused", async (t) => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-grok-tmp-")); t.after(() => rm(root, { recursive: true, force: true })); const uid = process.getuid?.() ?? 0; - const home = path.join(root, "home"), shared = path.join(root, "shared"); - await mkdir(path.join(home, "tmp"), { recursive: true }); await mkdir(shared); - await chmod(path.join(home, "tmp"), 0o700); - // A shared root owned by the test user is refused (not root-owned) — as is a symlinked private temp. - await assert.rejects(verifyGrokWorkerTmp(home, uid, [shared]), /temp isolation attestation unavailable/u); - await rm(path.join(home, "tmp"), { recursive: true }); await symlink(shared, path.join(home, "tmp")); - await assert.rejects(verifyGrokWorkerTmp(home, uid, []), /temp isolation attestation unavailable/u); - await rm(path.join(home, "tmp")); await mkdir(path.join(home, "tmp"), { mode: 0o700 }); - await verifyGrokWorkerTmp(home, uid, []); + const own = path.join(root, "own"), other = path.join(root, "other"), shared = path.join(root, "shared"), elsewhere = path.join(root, "elsewhere"); + for (const directory of [path.join(own, "tmp"), path.join(other, "tmp"), shared, elsewhere]) { await mkdir(directory, { recursive: true }); await chmod(directory, 0o700); } + // Seams: this test runs unprivileged, so the owner and worker-range floor are the test user. + const options: GrokWorkerTmpOptions = { sharedRoots: [shared], sharedOwnerUid: uid, firstWorkerUid: uid }; + const workers = [{ home: own, uid }, { home: other, uid }]; + await verifyGrokWorkerTmp(workers, options); + + await chmod(path.join(other, "tmp"), 0o777); + await assert.rejects(verifyGrokWorkerTmp(workers, options), refused, "sibling 0777"); + await chmod(path.join(other, "tmp"), 0o700); + + await rm(path.join(own, "tmp"), { recursive: true }); await symlink(elsewhere, path.join(own, "tmp")); + await assert.rejects(verifyGrokWorkerTmp(workers, options), refused, "private temp symlink to a valid directory"); + await rm(path.join(own, "tmp")); await mkdir(path.join(own, "tmp"), { mode: 0o700 }); + + const linkedShared = path.join(root, "linked-shared"); await symlink(elsewhere, linkedShared); + await assert.rejects(verifyGrokWorkerTmp(workers, { ...options, sharedRoots: [linkedShared] }), refused, "shared root symlink to a valid directory"); + const regular = path.join(root, "regular"); await writeFile(regular, ""); await chmod(regular, 0o600); + await assert.rejects(verifyGrokWorkerTmp(workers, { ...options, sharedRoots: [regular] }), refused, "shared root regular file"); + await assert.rejects(verifyGrokWorkerTmp(workers, { ...options, sharedRoots: [] }), refused, "empty shared roots"); + await verifyGrokWorkerTmp(workers, options); }); diff --git a/src/runtime/grokWorkerTmpAttestation.ts b/src/runtime/grokWorkerTmpAttestation.ts index 4d2cf7b..863a9ab 100644 --- a/src/runtime/grokWorkerTmpAttestation.ts +++ b/src/runtime/grokWorkerTmpAttestation.ts @@ -4,11 +4,19 @@ import path from "node:path"; import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; -type Entry = Pick & Readonly<{ isDirectory(): boolean; isSymbolicLink(): boolean }>; +type Entry = Pick & Readonly<{ isDirectory(): boolean }>; const HOME = GROK_ENGINE_BROKER.worker.home; +const FIRST_WORKER_UID = GROK_ENGINE_BROKER.identities.firstWorkerUid; + +export type GrokWorkerTmpWorker = Readonly<{ home: string; uid: number }>; +/** Test seams only; production uses the manifest defaults. */ +export type GrokWorkerTmpOptions = Readonly<{ sharedRoots?: readonly string[]; sharedOwnerUid?: number; firstWorkerUid?: number }>; /** - * Temp-directory isolation for one worker, checked before every turn. + * Temp-directory isolation, checked before every turn for *every* registered + * worker, not only the one about to run: a misprovisioned sibling temp + * directory (group- or world-writable) would be a place this worker could + * write into and that sibling would read from. * * Grok 1.0.34's strict profile grants shared `/tmp` and `/var/tmp` * read-write, and it refuses to start when either (or any ancestor of a @@ -24,25 +32,37 @@ const HOME = GROK_ENGINE_BROKER.worker.home; * deployment that lets workers traverse or write them is refused; * - the worker's own `/tmp` (the launcher's compiled `TMPDIR`, which * strict grants read-write) is a real directory owned by the worker with no - * group or other access. + * group or other access, and every registered worker's is checked. + * + * + * Entries come from `lstat`, so a symlink is never a directory here: a + * symlinked temp root or private temp is refused by the directory check. * * Pure so every refusal is testable without root. */ -export function assertGrokWorkerTmpEntries(entries: Readonly<{ privateTmp: Entry | undefined; shared: readonly (Entry | undefined)[] }>, workerUid: number): void { +export function assertGrokWorkerTmpEntries(entries: Readonly<{ privateTmps: readonly Readonly<{ uid: number; entry: Entry | undefined }>[]; shared: readonly (Entry | undefined)[] }>, options: GrokWorkerTmpOptions = {}): void { const shared = HOME.sharedTmp; + const firstWorkerUid = options.firstWorkerUid ?? FIRST_WORKER_UID; + if (entries.shared.length === 0 || entries.privateTmps.length === 0) throw unavailable(); for (const entry of entries.shared) { - if (entry === undefined || !entry.isDirectory() || entry.isSymbolicLink() || entry.uid !== shared.uid || entry.gid >= shared.maxGroupExclusive || (Number(entry.mode) & 0o007 & ~shared.otherMode) !== 0) throw unavailable(); + if (entry === undefined || !entry.isDirectory() || entry.uid !== (options.sharedOwnerUid ?? shared.uid) || entry.gid >= shared.maxGroupExclusive || (Number(entry.mode) & 0o007 & ~shared.otherMode) !== 0) throw unavailable(); + } + for (const { uid, entry } of entries.privateTmps) { + if (!Number.isSafeInteger(uid) || uid < firstWorkerUid || entry === undefined || !entry.isDirectory() || entry.uid !== uid || (Number(entry.mode) & 0o077) !== 0) throw unavailable(); } - const own = entries.privateTmp; - if (own === undefined || !own.isDirectory() || own.isSymbolicLink() || own.uid !== workerUid || (Number(own.mode) & 0o077) !== 0) throw unavailable(); } -export async function verifyGrokWorkerTmp(workerHome: string, workerUid: number, sharedRoots: readonly string[] = HOME.sharedTmp.paths): Promise { +/** `workers` must list every registered worker (the running one included). */ +export async function verifyGrokWorkerTmp(workers: readonly GrokWorkerTmpWorker[], options: GrokWorkerTmpOptions = {}): Promise { const inspect = async (file: string): Promise => { try { return await lstat(file); } catch { return undefined; } }; assertGrokWorkerTmpEntries({ - privateTmp: await inspect(path.join(workerHome, HOME.privateTmp.relativeToWorkerHome)), - shared: await Promise.all(sharedRoots.map(inspect)) - }, workerUid); + privateTmps: await Promise.all(workers.map(async (worker) => ({ uid: worker.uid, entry: await inspect(path.join(worker.home, HOME.privateTmp.relativeToWorkerHome)) }))), + shared: await Promise.all((options.sharedRoots ?? HOME.sharedTmp.paths).map(inspect)) + }, options); } +/** A registration's worker home is the parent of its `/.grok/sandbox.toml`. */ +export const grokWorkerHomeForProfile = (profilePath: string): string | undefined => + path.basename(path.dirname(profilePath)) === ".grok" ? path.dirname(path.dirname(profilePath)) : undefined; + const unavailable = (): Error => new Error("Grok worker temp isolation attestation unavailable"); From a2ba04bcdcccdcc6f1547b174182a5d3024487f2 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 08:00:14 +0200 Subject: [PATCH 065/124] fix: pin the tool-output spill directory and publish spills by rename into the verified directory --- src/runtime/toolResultSpill.test.ts | 89 ++++++++++++++++++++++++++--- src/runtime/toolResultSpill.ts | 64 +++++++++++++++++++-- 2 files changed, 140 insertions(+), 13 deletions(-) diff --git a/src/runtime/toolResultSpill.test.ts b/src/runtime/toolResultSpill.test.ts index a5f6c82..a4b94c6 100644 --- a/src/runtime/toolResultSpill.test.ts +++ b/src/runtime/toolResultSpill.test.ts @@ -1,11 +1,12 @@ import assert from "node:assert/strict"; -import { chmod, chown, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; +import { chmod, chown, lstat, mkdir, mkdtemp, open, readFile, readdir, rm, stat, symlink, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import test from "node:test"; +import test, { mock } from "node:test"; import { renderMcpToolResult, MCP_TOOL_RESULT_MAX_BYTES, type McpUpstreamResult } from "./mcpToolResult.js"; import { + assertSpillDirectoryStat, capToolResult, DEFAULT_TOOL_RESULT_MAX_BYTES, MIN_TOOL_RESULT_MAX_BYTES, @@ -153,16 +154,88 @@ test("the bound and the exemption list come from the environment, and a nonsense test("spilled files are readable by the directory's (worker) group and never by other users", async () => { await withDirectory(async (directory) => { - // What a deployment provisions for a brokered worker: setgid tool-output in the worker's group. - const groups = (process.getgroups?.() ?? []).filter((gid) => gid !== process.getgid?.()); - const workerGroup = groups[0]; - if (workerGroup !== undefined) await chown(directory, process.getuid?.() ?? -1, workerGroup).catch(() => undefined); - await chmod(directory, 0o2750); + // What a deployment provisions for a brokered worker: setgid tool-output in a group that is not the runtime's own. + const workerGroup = (process.getgroups?.() ?? []).find((gid) => gid !== process.getgid?.()); + if (workerGroup !== undefined) { await chown(directory, process.getuid?.() ?? -1, workerGroup); await chmod(directory, 0o2750); } else await chmod(directory, 0o700); const previous = process.umask(0o077); let capped; try { capped = await cap({ content: [{ type: "text", text: "x".repeat(200_000) }] }, { spillDirectory: directory }); } finally { process.umask(previous); } - const file = await stat(capped.spillPath!); + assert.ok(capped.spillPath, "the spill was written"); + const file = await stat(capped.spillPath); assert.equal(file.mode & 0o777, 0o640, "group-readable even under a restrictive umask, never other-readable"); assert.equal(file.gid, (await stat(directory)).gid, "the file carries the directory's group"); }); }); + +const big = { content: [{ type: "text" as const, text: `HEAD${"y".repeat(100_000)}TAIL` }] }; +const spillName = "daimon-abc123.mcp_desk_archive_dump.log"; + +test("a symlinked, world-open, or group-open-without-setgid spill directory is refused and nothing is written", async () => { + await withDirectory(async (root) => { + const target = path.join(root, "target"); await mkdir(target, { mode: 0o700 }); + const linked = path.join(root, "linked"); await symlink(target, linked); + const capped = await cap(big, { spillDirectory: linked }); + assert.equal(capped.spillPath, undefined); + assert.equal(capped.details.full_output_saved, false); + assert.deepEqual(await readdir(target), [], "nothing written through the symlink"); + const workerGroup = (process.getgroups?.() ?? []).find((gid) => gid !== process.getgid?.()); + if (workerGroup !== undefined) { + // A foreign group without setgid: files would not inherit it, so it is refused. + const noSetgid = path.join(root, "foreign-group-no-setgid"); await mkdir(noSetgid); await chown(noSetgid, process.getuid?.() ?? -1, workerGroup); await chmod(noSetgid, 0o750); + assert.equal((await cap(big, { spillDirectory: noSetgid })).spillPath, undefined, "0750 foreign group without setgid"); + } + // 2750 in the runtime's own group is not a worker grant; 0701/0704 exceed 2750. + for (const mode of [0o777, 0o755, 0o750, 0o2770, 0o2757, 0o2750, 0o701, 0o704]) { + const directory = path.join(root, `mode-${mode.toString(8)}`); await mkdir(directory); await chmod(directory, mode); + const refused = await cap(big, { spillDirectory: directory }); + assert.equal(refused.spillPath, undefined, mode.toString(8)); + assert.deepEqual((await readdir(directory)).filter((name) => !name.startsWith(".")), [], mode.toString(8)); + } + }); +}); + +test("a destination symlink or a pre-existing 0666 file is replaced by a 0640 regular file without touching the target", async () => { + await withDirectory(async (root) => { + const directory = path.join(root, "tool-output"); await mkdir(directory, { mode: 0o700 }); + const victim = path.join(root, "victim.txt"); await writeFile(victim, "VICTIM", { mode: 0o644 }); + await symlink(victim, path.join(directory, spillName)); + const first = await cap(big, { spillDirectory: directory }); + assert.equal(first.spillPath, path.join(directory, spillName)); + const replaced = await lstat(first.spillPath!); + assert.ok(replaced.isFile() && !replaced.isSymbolicLink()); + assert.equal(replaced.mode & 0o777, 0o640); + assert.equal(await readFile(victim, "utf8"), "VICTIM", "the symlink target is untouched"); + + await rm(first.spillPath!); await writeFile(first.spillPath!, "stale", { mode: 0o666 }); await chmod(first.spillPath!, 0o666); + const second = await cap(big, { spillDirectory: directory }); + const rewritten = await lstat(second.spillPath!); + assert.equal(rewritten.mode & 0o777, 0o640); + assert.equal(await readFile(second.spillPath!, "utf8"), big.content[0].text); + }); +}); + +test("the spill directory must be owned by the runtime itself", () => { + const runtime = { uid: 2000, gid: 2000 }; + const entry = (uid: number, gid: number, mode: number) => ({ uid, gid, mode: 0o040000 | mode, isDirectory: () => true }); + assert.doesNotThrow(() => assertSpillDirectoryStat(entry(2000, 2000, 0o700), runtime)); + assert.doesNotThrow(() => assertSpillDirectoryStat(entry(2000, 2200, 0o2750), runtime)); + for (const [label, candidate] of [["owned by a worker", entry(2200, 2200, 0o700)], ["owned by root", entry(0, 2200, 0o2750)], ["not a directory", { ...entry(2000, 2000, 0o700), isDirectory: () => false }]] as const) { + assert.throws(() => assertSpillDirectoryStat(candidate, runtime), /spill directory/u, label); + } +}); + +test("a spill directory whose opened inode differs from the named one is refused", async () => { + await withDirectory(async (directory) => { + const probe = await open(directory, "r"); + const prototype = Object.getPrototypeOf(probe) as { stat: (...args: unknown[]) => Promise<{ ino: number }> }; + await probe.close(); + const original = prototype.stat; let calls = 0; + const swapped = mock.method(prototype, "stat", async function (this: unknown, ...args: unknown[]) { + const real = await original.apply(this, args); + calls += 1; + return calls === 1 ? Object.assign(Object.create(Object.getPrototypeOf(real)), real, { ino: Number(real.ino) + 1 }) : real; + }); + try { assert.equal((await cap(big, { spillDirectory: directory })).spillPath, undefined); } finally { swapped.mock.restore(); } + assert.deepEqual(await readdir(directory), []); + }); +}); diff --git a/src/runtime/toolResultSpill.ts b/src/runtime/toolResultSpill.ts index b0a8d55..f557c38 100644 --- a/src/runtime/toolResultSpill.ts +++ b/src/runtime/toolResultSpill.ts @@ -193,6 +193,7 @@ const notice = (input: Readonly<{ * creates itself stays 0700, so for every other engine nothing new is exposed. */ export const SPILL_FILE_MODE = 0o640; +export const SPILL_DIRECTORY_MAX_MODE = 0o2750; /** * Write the full payload where the agent can read it, atomically. @@ -203,13 +204,66 @@ export const SPILL_FILE_MODE = 0o640; */ const writeSpill = async (directory: string, name: string, text: string): Promise => { await mkdir(directory, { recursive: true, mode: 0o700 }); + const pinned = await pinSpillDirectory(directory); const file = path.join(directory, name); const temporary = `${file}.${process.pid}.${Date.now()}.tmp`; - const handle = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, SPILL_FILE_MODE); - // Explicit, so a restrictive umask cannot strip the group read an agent's sandboxed worker needs. - try { await handle.chmod(SPILL_FILE_MODE); await handle.writeFile(text, "utf8"); await handle.sync(); } finally { await handle.close(); } - try { await rename(temporary, file); } catch (error) { await unlink(temporary).catch(() => undefined); throw error; } - return file; + try { + const handle = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, SPILL_FILE_MODE); + let written: Awaited>; + // Explicit, so a restrictive umask cannot strip the group read an agent's sandboxed worker needs. + try { await handle.chmod(SPILL_FILE_MODE); await handle.writeFile(text, "utf8"); await handle.sync(); written = await handle.stat(); } finally { await handle.close(); } + if (!written.isFile() || written.nlink !== 1 || written.uid !== process.getuid?.()) throw new Error("spill file is not a private regular file"); + // Node has no openat: re-check that the directory entry still names the pinned inode before publishing into it. + await assertSameDirectory(directory, pinned); + // rename replaces a pre-existing destination entry (a symlink included) without following it. + await rename(temporary, file); + const published = await lstat(file); + if (!published.isFile() || published.dev !== written.dev || published.ino !== written.ino) throw new Error("spill file was replaced"); + return file; + } catch (error) { + await unlink(temporary).catch(() => undefined); + throw error; + } finally { + await pinned.handle.close(); + } +}; + +type PinnedDirectory = Readonly<{ handle: Awaited>; dev: number; ino: number }>; + +/** + * The spill directory must be a real directory owned by this runtime and no + * wider than `2750`: either Daimon's own `0700`, or a deployment-provisioned + * setgid directory whose group is not the runtime's own (a worker group). A + * symlinked, foreign-owned, world-accessible, or group-open-without-setgid + * directory is refused and nothing is written. Daimon cannot know *which* + * worker gid belongs to this agent; that mapping is the deployment's + * provisioning contract (`GROK_ENGINE_BROKER.worker.home.spillDirectory`). + */ +const pinSpillDirectory = async (directory: string): Promise => { + const before = await lstat(directory); + if (before.isSymbolicLink() || !before.isDirectory()) throw new Error("spill directory is not a real directory"); + const handle = await open(directory, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); + try { + const opened = await handle.stat(); + if (opened.dev !== before.dev || opened.ino !== before.ino) throw new Error("spill directory was replaced"); + assertSpillDirectoryStat(opened, { uid: process.getuid?.() ?? -1, gid: process.getgid?.() ?? -1 }); + return { handle, dev: Number(opened.dev), ino: Number(opened.ino) }; + } catch (error) { await handle.close(); throw error; } +}; + +/** Pure so a foreign owner is testable without root. */ +export const assertSpillDirectoryStat = (entry: Readonly<{ uid: number; gid: number; mode: number; isDirectory(): boolean }>, runtime: Readonly<{ uid: number; gid: number }>): void => { + const mode = Number(entry.mode) & 0o7777; + const groupOpen = (mode & 0o070) !== 0; + if (!entry.isDirectory() || entry.uid !== runtime.uid || (mode & ~SPILL_DIRECTORY_MAX_MODE) !== 0 + || (groupOpen && ((mode & 0o2000) === 0 || entry.gid === runtime.gid))) { + throw new Error("spill directory is not a private or provisioned worker-group directory"); + } +}; + +const assertSameDirectory = async (directory: string, pinned: PinnedDirectory): Promise => { + const [now, held] = await Promise.all([lstat(directory), pinned.handle.stat()]); + if (now.isSymbolicLink() || Number(now.dev) !== pinned.dev || Number(now.ino) !== pinned.ino || Number(held.ino) !== pinned.ino) throw new Error("spill directory was replaced"); }; /** Newest-first retention, so a busy agent cannot fill its own runtime home. */ From 4aef65c72bc9d12e83299a4cf8a590a18479e9b4 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 08:03:12 +0200 Subject: [PATCH 066/124] fix: refuse non-canonical Grok worker registration paths and truncated worker environment paths --- src/runtime/engineBrokerServiceCli.test.ts | 13 ++ src/runtime/engineBrokerServiceConfig.ts | 7 +- .../native/engineBrokerLauncherCore.inc | 166 ++---------------- .../engineBrokerLauncherIntegrationMain.inc | 60 +++++++ .../native/engineBrokerLauncherServer.inc | 150 ++++++++++++++++ src/runtime/native/launcherArgv.test.ts | 9 +- 6 files changed, 252 insertions(+), 153 deletions(-) diff --git a/src/runtime/engineBrokerServiceCli.test.ts b/src/runtime/engineBrokerServiceCli.test.ts index 20078ae..5522f10 100644 --- a/src/runtime/engineBrokerServiceCli.test.ts +++ b/src/runtime/engineBrokerServiceCli.test.ts @@ -63,3 +63,16 @@ test("v2 may declare an evaluator inference ledger that is never a subject ledge } assert.throws(() => parseEngineBrokerServiceConfig({ ...config("v1", [reg("agent-a", 0)]), inferenceLedgerPath: "/run/paideia-inference/inference.jsonl" }), /invalid engine broker service config/u); }); + +test("registration paths must be canonical, matching the native launcher's registration check", () => { + const good = reg("agent-a", 0); + for (const [field, value] of [ + ["workspace", "/workspace/0/../1"], ["workspace", "/workspace//0"], ["workspace", "/workspace/0/"], ["workspace", "/workspace/./0"], + ["profilePath", "/workers/0/../1/.grok/sandbox.toml"], ["profilePath", "/workers//0/.grok/sandbox.toml"] + ] as const) { + const registration = { ...good, [field]: value, ...(field === "profilePath" ? { eventsPath: value.replace(/sandbox\.toml$/u, "sessions/sandbox-events.jsonl") } : {}) }; + assert.throws(() => parseEngineBrokerServiceConfig(config("v1", [registration])), /invalid engine broker service config/u, `${field}=${value}`); + } + assert.throws(() => parseEngineBrokerServiceConfig({ ...config("v1", [good]), turnStore: "/var/lib/turns/" })); + assert.doesNotThrow(() => parseEngineBrokerServiceConfig(config("v1", [good]))); +}); diff --git a/src/runtime/engineBrokerServiceConfig.ts b/src/runtime/engineBrokerServiceConfig.ts index 742e2a5..af2a159 100644 --- a/src/runtime/engineBrokerServiceConfig.ts +++ b/src/runtime/engineBrokerServiceConfig.ts @@ -31,7 +31,12 @@ const V2_REGISTRATION = [...V1_REGISTRATION, "usageLedgerPath", "limits", "model const invalid = (): TypeError => new TypeError("invalid engine broker service config"); const plain = (value: unknown): value is Record => value !== null && typeof value === "object" && !Array.isArray(value); const exact = (value: Record, fields: readonly string[]): void => { if (Object.keys(value).length !== fields.length || fields.some((field) => !Object.hasOwn(value, field))) throw invalid(); }; -const absolute = (item: unknown): item is string => typeof item === "string" && item.startsWith("/") && !item.includes("/../") && !item.endsWith("/..") && !item.includes("\0"); +/** + * Absolute and canonical: no `.`/`..`/empty components and no trailing slash. + * The native launcher derives HOME, GROK_HOME and TMPDIR from the registered + * home and refuses a non-canonical one, so the broker's view must match it. + */ +const absolute = (item: unknown): item is string => typeof item === "string" && item.length > 1 && item.startsWith("/") && !item.endsWith("/") && path.posix.normalize(item) === item && !item.split("/").slice(1).some((part) => part === "." || part === "..") && !item.includes("\0"); /** The per-request stream written beside a registration's usage ledger. */ export const engineBrokerRequestLedgerPathFor = (usageLedgerPath: string): string => path.posix.join(path.posix.dirname(usageLedgerPath), "requests.jsonl"); diff --git a/src/runtime/native/engineBrokerLauncherCore.inc b/src/runtime/native/engineBrokerLauncherCore.inc index 41cc947..332fb0a 100644 --- a/src/runtime/native/engineBrokerLauncherCore.inc +++ b/src/runtime/native/engineBrokerLauncherCore.inc @@ -25,7 +25,6 @@ #include static __attribute__((noreturn)) void die(void) { _exit(111); } -static int bounded(const char *s, size_t n) { return memchr(s, 0, n) != NULL; } static int safe_component(const char *s, size_t n) { size_t i, l = strnlen(s, n); if (!l || l == n) @@ -157,12 +156,27 @@ static int load_registration(uint32_t slot, struct dbl_registration *out) { close(fd); return -1; } +/* Absolute, NUL-terminated within n, no empty, "." or ".." component and no + trailing slash: the launcher derives HOME, GROK_HOME and TMPDIR from it. */ +static int canonical_path(const char *s, size_t n) { + size_t l = strnlen(s, n), i = 0; + if (l < 2 || l == n || s[0] != '/' || s[l - 1] == '/') + return 0; + while (i < l) { + size_t start = ++i; + while (i < l && s[i] != '/') + i++; + if (i == start || (i - start == 1 && s[start] == '.') || + (i - start == 2 && s[start] == '.' && s[start + 1] == '.')) + return 0; + } + return 1; +} static int valid_registration(const struct dbl_registration *r, const struct dbl_request *q) { return r->uid >= 2200 && r->gid >= 2200 && - bounded(r->workspace, sizeof(r->workspace)) && - bounded(r->home, sizeof(r->home)) && r->workspace[0] == '/' && - r->home[0] == '/' && + canonical_path(r->workspace, sizeof(r->workspace)) && + canonical_path(r->home, sizeof(r->home)) && safe_component(r->agent_id, sizeof(r->agent_id)) && strcmp(r->agent_id, q->agent_id) == 0; } @@ -248,147 +262,3 @@ static uint64_t start_ticks(pid_t pid) { } return 0; } - -static __attribute__((noreturn)) void launch_fail(int status_fd, - uint32_t code) { - (void)full_write(status_fd, &code, sizeof(code)); - _exit(111); -} -static pid_t launch(const struct dbl_registration *r, int executable, - int prompt, int capability, int output, uint32_t *failure, - uint64_t *observed_start_ticks) { - unsigned char provider[DBL_MAX_TOKEN + 1] = {0}, mcp[DBL_MAX_TOKEN + 1] = {0}; - int status_pipe[2]; - if (capability_bundle(capability, provider, mcp) || - pipe2(status_pipe, O_CLOEXEC)) { - erase(provider, sizeof(provider)); - erase(mcp, sizeof(mcp)); - return -1; - } - pid_t p = fork(); - if (p < 0) { - close(status_pipe[0]); - close(status_pipe[1]); - erase(provider, sizeof(provider)); - erase(mcp, sizeof(mcp)); - return -1; - } - if (p != 0) { - uint32_t code = 0; - close(status_pipe[1]); - if (full_read(status_pipe[0], observed_start_ticks, - sizeof(*observed_start_ticks)) || - !*observed_start_ticks) { - close(status_pipe[0]); - erase(provider, sizeof(provider)); - erase(mcp, sizeof(mcp)); - kill(p, SIGKILL); - waitpid(p, NULL, 0); - return -1; - } - ssize_t got = read(status_pipe[0], &code, sizeof(code)); - close(status_pipe[0]); - erase(provider, sizeof(provider)); - erase(mcp, sizeof(mcp)); - if (got == 0) - return p; - if (got == (ssize_t)sizeof(code)) - *failure = code; - kill(p, SIGKILL); - waitpid(p, NULL, 0); - return -1; - } - close(status_pipe[0]); - if (prctl(PR_SET_PDEATHSIG, SIGKILL) || getppid() == 1 || setpgid(0, 0)) - launch_fail(status_pipe[1], 1); - uint64_t identity = start_ticks(getpid()); - if (!identity || full_write(status_pipe[1], &identity, sizeof(identity))) - launch_fail(status_pipe[1], 1); - struct rlimit z = {0, 0}; - if (setrlimit(RLIMIT_CORE, &z) || setgroups(0, NULL) || - prctl(PR_SET_KEEPCAPS, 1, 0, 0, 0) || setresgid(r->gid, r->gid, r->gid) || - setresuid(r->uid, r->uid, r->uid)) - launch_fail(status_pipe[1], 2); - zero_caps(); - if (prctl(PR_SET_KEEPCAPS, 0, 0, 0, 0) || - prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) - launch_fail(status_pipe[1], 3); - /* Received fds carry MSG_CMSG_CLOEXEC and may already sit on 3-5: dup2 onto - itself keeps close-on-exec (the prompt vanished at exec) and an - overlapping order clobbers a source. Lift all of them above the targets - first so every dup2 below changes the fd number and clears CLOEXEC. */ - int status_fd = fcntl(status_pipe[1], F_DUPFD_CLOEXEC, 16); - if (status_fd < 0) - launch_fail(status_pipe[1], 4); - int null_input = open("/dev/null", O_RDONLY | O_CLOEXEC); - int high_prompt = fcntl(prompt, F_DUPFD_CLOEXEC, 16), - high_capability = fcntl(capability, F_DUPFD_CLOEXEC, 16), - high_output = fcntl(output, F_DUPFD_CLOEXEC, 16), - high_executable = fcntl(executable, F_DUPFD_CLOEXEC, 16); - if (null_input < 0 || high_prompt < 0 || high_capability < 0 || - high_output < 0 || high_executable < 0 || - dup2(null_input, STDIN_FILENO) < 0 || dup2(high_prompt, 3) < 0 || - dup2(high_capability, 4) < 0 || dup2(high_output, STDOUT_FILENO) < 0 || - dup2(high_output, STDERR_FILENO) < 0) - launch_fail(status_fd, 4); - /* The worker needs fd 5 only to be executed: close-on-exec keeps the Grok - image descriptor out of the worker and every tool child (execveat with - AT_EMPTY_PATH still works for an ELF; only a #! script would lose it). */ - if (dup2(high_executable, 5) < 0 || fcntl(5, F_SETFD, FD_CLOEXEC) < 0) - launch_fail(status_fd, 5); - close_other_fds(status_fd); - char *const argv[] = {"grok", - "--sandbox", - "daimon-strict", - "--always-approve", - "--no-subagents", - "--prompt-file", - "/proc/self/fd/3", - "--no-memory", - "--disable-web-search", - "--no-plan", - "--verbatim", - "--system-prompt-override", - DBL_GROK_SYSTEM_PROMPT, - "--tools", - DBL_GROK_TOOLS, - "--max-turns", - DBL_GROK_MAX_TURNS, - "--cwd", - (char *)r->workspace, - "--output-format", - "streaming-messages-json", - "--model", - "daimon-broker-grok", - NULL}; - char home[300], grok[300], tmp[300], mcp_env[DBL_MAX_TOKEN + 24], - provider_env[DBL_MAX_TOKEN + 32]; - snprintf(home, sizeof(home), "HOME=%s", r->home); - snprintf(grok, sizeof(grok), "GROK_HOME=%s/.grok", r->home); - /* Worker-private temp: strict grants TMPDIR read-write, and shared /tmp is - kept from the worker by the deployment's modes (attested by the broker). */ - snprintf(tmp, sizeof(tmp), "TMPDIR=%s/tmp", r->home); - snprintf(mcp_env, sizeof(mcp_env), "DAIMON_MCP_CAPABILITY=%s", mcp); - /* Grok 1.0.34 never runs `[auth_provider.*]` helpers for a custom model, so - the worker's `[model.daimon-broker-grok] env_key` reads the turn-scoped - proxy capability from here. It is as exposed as the MCP capability. */ - snprintf(provider_env, sizeof(provider_env), "DAIMON_PROVIDER_CAPABILITY=%s", - provider); - erase(provider, sizeof(provider)); - erase(mcp, sizeof(mcp)); - char *const envp[] = {home, - grok, - tmp, - mcp_env, - provider_env, - "DAIMON_CAPABILITY_FD=4", - "PATH=/usr/local/bin:/usr/bin:/bin", - "LANG=C.UTF-8", - "TZ=UTC", - NULL}; - syscall(SYS_execveat, 5, "", argv, envp, AT_EMPTY_PATH); - erase(provider_env, sizeof(provider_env)); - erase(mcp_env, sizeof(mcp_env)); - launch_fail(status_fd, 6); -} - diff --git a/src/runtime/native/engineBrokerLauncherIntegrationMain.inc b/src/runtime/native/engineBrokerLauncherIntegrationMain.inc index 1191e2d..f773c55 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationMain.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationMain.inc @@ -1,3 +1,61 @@ +/* Non-canonical or unterminated homes/workspaces are refused at registration: + HOME, GROK_HOME and TMPDIR are derived from them. */ +static void noncanonical_registration_cases(struct dbl_registration r) { + const char *homes[] = {"/tmp/worker-home/../other", "/tmp//worker-home", + "/tmp/worker-home/", "/tmp/./worker-home", + "/tmp/worker-home/..", "relative/home", NULL}; + struct dbl_registration bad[8]; + size_t count = 0; + for (; homes[count]; count++) { + bad[count] = r; + bad[count].slot = 100 + (uint32_t)count; + memset(bad[count].home, 0, sizeof(bad[count].home)); + strcpy(bad[count].home, homes[count]); + } + bad[count] = r; + bad[count].slot = 100 + (uint32_t)count; + memset(bad[count].home, 'a', sizeof(bad[count].home)); /* over-long: no NUL */ + bad[count].home[0] = '/'; + count++; + bad[count] = r; + bad[count].slot = 100 + (uint32_t)count; + strcpy(bad[count].workspace, "/tmp/workspace/../etc"); + count++; + int f = open(DBL_REGISTRY, O_TRUNC | O_WRONLY, 0600); + check(f >= 0 && write(f, &r, sizeof(r)) == sizeof(r) && + write(f, bad, sizeof(bad[0]) * count) == + (ssize_t)(sizeof(bad[0]) * count), + "noncanonical registry"); + close(f); + pid_t child = fork(); + if (!child) { + setgid(DBL_BROKER_UID); + setuid(DBL_BROKER_UID); + for (size_t i = 0; i < count; i++) { + int s = connect_socket(), p = sealed("prompt"), + c = sealed_bundle("provider.Path-1", "mcp.Path-2"); + struct dbl_request q = request(); + struct dbl_result result; + q.slot = 100 + (uint32_t)i; + send_request(s, &q, p, c); + if (!read_all(s, &result, sizeof(result)) || + result.status != DBL_STATUS_PRELAUNCH_FAILED || + result.stage != DBL_STAGE_REGISTRATION || result.worker_pid != 0) + _exit(10 + (int)i); + close(s); + close(p); + close(c); + } + _exit(0); + } + int status; + waitpid(child, &status, 0); + check(WIFEXITED(status) && WEXITSTATUS(status) == 0, + "noncanonical registration refused"); + f = open(DBL_REGISTRY, O_TRUNC | O_WRONLY, 0600); + check(f >= 0 && write(f, &r, sizeof(r)) == sizeof(r), "registry restore"); + close(f); +} /* A worker that fails before or at exec must not run anything: the launcher child exits instead. The registered "executable" is a #! script; fd 5 is close-on-exec, so execveat(AT_EMPTY_PATH) cannot run it (a script needs @@ -113,6 +171,8 @@ int main(void) { check(WIFEXITED(status) && WEXITSTATUS(status) == 0, "org cases"); exec_failure_case(r); puts("native-stage exec failure complete"); + noncanonical_registration_cases(r); + puts("native-stage noncanonical registration complete"); kill(broker, SIGKILL); waitpid(broker, 0, 0); check(WIFEXITED(status) && WEXITSTATUS(status) == 0, "org cases"); diff --git a/src/runtime/native/engineBrokerLauncherServer.inc b/src/runtime/native/engineBrokerLauncherServer.inc index a6cb347..75f452b 100644 --- a/src/runtime/native/engineBrokerLauncherServer.inc +++ b/src/runtime/native/engineBrokerLauncherServer.inc @@ -1,3 +1,153 @@ +static __attribute__((noreturn)) void launch_fail(int status_fd, + uint32_t code) { + (void)full_write(status_fd, &code, sizeof(code)); + _exit(111); +} +static pid_t launch(const struct dbl_registration *r, int executable, + int prompt, int capability, int output, uint32_t *failure, + uint64_t *observed_start_ticks) { + unsigned char provider[DBL_MAX_TOKEN + 1] = {0}, mcp[DBL_MAX_TOKEN + 1] = {0}; + int status_pipe[2]; + if (capability_bundle(capability, provider, mcp) || + pipe2(status_pipe, O_CLOEXEC)) { + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); + return -1; + } + pid_t p = fork(); + if (p < 0) { + close(status_pipe[0]); + close(status_pipe[1]); + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); + return -1; + } + if (p != 0) { + uint32_t code = 0; + close(status_pipe[1]); + if (full_read(status_pipe[0], observed_start_ticks, + sizeof(*observed_start_ticks)) || + !*observed_start_ticks) { + close(status_pipe[0]); + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); + kill(p, SIGKILL); + waitpid(p, NULL, 0); + return -1; + } + ssize_t got = read(status_pipe[0], &code, sizeof(code)); + close(status_pipe[0]); + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); + if (got == 0) + return p; + if (got == (ssize_t)sizeof(code)) + *failure = code; + kill(p, SIGKILL); + waitpid(p, NULL, 0); + return -1; + } + close(status_pipe[0]); + if (prctl(PR_SET_PDEATHSIG, SIGKILL) || getppid() == 1 || setpgid(0, 0)) + launch_fail(status_pipe[1], 1); + uint64_t identity = start_ticks(getpid()); + if (!identity || full_write(status_pipe[1], &identity, sizeof(identity))) + launch_fail(status_pipe[1], 1); + struct rlimit z = {0, 0}; + if (setrlimit(RLIMIT_CORE, &z) || setgroups(0, NULL) || + prctl(PR_SET_KEEPCAPS, 1, 0, 0, 0) || setresgid(r->gid, r->gid, r->gid) || + setresuid(r->uid, r->uid, r->uid)) + launch_fail(status_pipe[1], 2); + zero_caps(); + if (prctl(PR_SET_KEEPCAPS, 0, 0, 0, 0) || + prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) + launch_fail(status_pipe[1], 3); + /* Received fds carry MSG_CMSG_CLOEXEC and may already sit on 3-5: dup2 onto + itself keeps close-on-exec (the prompt vanished at exec) and an + overlapping order clobbers a source. Lift all of them above the targets + first so every dup2 below changes the fd number and clears CLOEXEC. */ + int status_fd = fcntl(status_pipe[1], F_DUPFD_CLOEXEC, 16); + if (status_fd < 0) + launch_fail(status_pipe[1], 4); + int null_input = open("/dev/null", O_RDONLY | O_CLOEXEC); + int high_prompt = fcntl(prompt, F_DUPFD_CLOEXEC, 16), + high_capability = fcntl(capability, F_DUPFD_CLOEXEC, 16), + high_output = fcntl(output, F_DUPFD_CLOEXEC, 16), + high_executable = fcntl(executable, F_DUPFD_CLOEXEC, 16); + if (null_input < 0 || high_prompt < 0 || high_capability < 0 || + high_output < 0 || high_executable < 0 || + dup2(null_input, STDIN_FILENO) < 0 || dup2(high_prompt, 3) < 0 || + dup2(high_capability, 4) < 0 || dup2(high_output, STDOUT_FILENO) < 0 || + dup2(high_output, STDERR_FILENO) < 0) + launch_fail(status_fd, 4); + /* The worker needs fd 5 only to be executed: close-on-exec keeps the Grok + image descriptor out of the worker and every tool child (execveat with + AT_EMPTY_PATH still works for an ELF; only a #! script would lose it). */ + if (dup2(high_executable, 5) < 0 || fcntl(5, F_SETFD, FD_CLOEXEC) < 0) + launch_fail(status_fd, 5); + close_other_fds(status_fd); + char *const argv[] = {"grok", + "--sandbox", + "daimon-strict", + "--always-approve", + "--no-subagents", + "--prompt-file", + "/proc/self/fd/3", + "--no-memory", + "--disable-web-search", + "--no-plan", + "--verbatim", + "--system-prompt-override", + DBL_GROK_SYSTEM_PROMPT, + "--tools", + DBL_GROK_TOOLS, + "--max-turns", + DBL_GROK_MAX_TURNS, + "--cwd", + (char *)r->workspace, + "--output-format", + "streaming-messages-json", + "--model", + "daimon-broker-grok", + NULL}; + char home[300], grok[300], tmp[300], mcp_env[DBL_MAX_TOKEN + 24], + provider_env[DBL_MAX_TOKEN + 32]; + /* Worker-private temp: strict grants TMPDIR read-write, and shared /tmp is + kept from the worker by the deployment's modes (attested by the broker). + A truncated path must never name a different directory: fail instead. */ + if ((size_t)snprintf(home, sizeof(home), "HOME=%s", r->home) >= sizeof(home) || + (size_t)snprintf(grok, sizeof(grok), "GROK_HOME=%s/.grok", r->home) >= + sizeof(grok) || + (size_t)snprintf(tmp, sizeof(tmp), "TMPDIR=%s/tmp", r->home) >= + sizeof(tmp)) { + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); + launch_fail(status_fd, 6); + } + snprintf(mcp_env, sizeof(mcp_env), "DAIMON_MCP_CAPABILITY=%s", mcp); + /* Grok 1.0.34 never runs `[auth_provider.*]` helpers for a custom model, so + the worker's `[model.daimon-broker-grok] env_key` reads the turn-scoped + proxy capability from here. It is as exposed as the MCP capability. */ + snprintf(provider_env, sizeof(provider_env), "DAIMON_PROVIDER_CAPABILITY=%s", + provider); + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); + char *const envp[] = {home, + grok, + tmp, + mcp_env, + provider_env, + "DAIMON_CAPABILITY_FD=4", + "PATH=/usr/local/bin:/usr/bin:/bin", + "LANG=C.UTF-8", + "TZ=UTC", + NULL}; + syscall(SYS_execveat, 5, "", argv, envp, AT_EMPTY_PATH); + erase(provider_env, sizeof(provider_env)); + erase(mcp_env, sizeof(mcp_env)); + launch_fail(status_fd, 6); +} + static void supervise(int client, pid_t pid, int output, struct dbl_result *out) { unsigned char bytes[DBL_MAX_OUTPUT + 1] = {0}; diff --git a/src/runtime/native/launcherArgv.test.ts b/src/runtime/native/launcherArgv.test.ts index d371bab..ed73ea9 100644 --- a/src/runtime/native/launcherArgv.test.ts +++ b/src/runtime/native/launcherArgv.test.ts @@ -21,7 +21,7 @@ const defines = (): ReadonlyMap => { /** The compiled worker argv, token by token, exactly as `launch()` passes it to `execveat`. */ const compiledArgv = (): readonly string[] => { - const source = read("engineBrokerLauncherCore.inc"); + const source = `${read("engineBrokerLauncherCore.inc")}${read("engineBrokerLauncherServer.inc")}`; const block = source.match(/char \*const argv\[\] = \{([\s\S]*?)NULL\};/u); assert.ok(block, "launcher argv array not found"); const values = defines(); @@ -55,15 +55,16 @@ test("the compiled system prompt is byte-identical to the contract prompt pinned }); test("the launcher exports the turn provider capability under the env_key the worker config reads", () => { - const source = read("engineBrokerLauncherCore.inc"); + const source = `${read("engineBrokerLauncherCore.inc")}${read("engineBrokerLauncherServer.inc")}`; assert.match(source, new RegExp(`"${GROK_BROKER_PROVIDER_CAPABILITY_ENV}=%s"`, "u")); assert.match(source, /char \*const envp\[\] = \{home,\s*grok,\s*tmp,\s*mcp_env,\s*provider_env,/u); assert.match(renderGrokBrokerWorkerConfig(), new RegExp(`\\nenv_key = "${GROK_BROKER_PROVIDER_CAPABILITY_ENV}"\\n`, "u")); }); test("the launcher gives every worker a private TMPDIR under its registered home", () => { - const source = read("engineBrokerLauncherCore.inc"); - assert.match(source, /snprintf\(tmp, sizeof\(tmp\), "TMPDIR=%s\/tmp", r->home\);/u); + const source = `${read("engineBrokerLauncherCore.inc")}${read("engineBrokerLauncherServer.inc")}`; + assert.match(source, /snprintf\(tmp, sizeof\(tmp\), "TMPDIR=%s\/tmp", r->home\) >=\s*sizeof\(tmp\)/u); + assert.match(source, /canonical_path\(r->home, sizeof\(r->home\)\)/u); assert.match(source, /char \*const envp\[\] = \{home,\s*grok,\s*tmp,/u); assert.equal(GROK_ENGINE_BROKER.worker.home.privateTmp.relativeToWorkerHome, "tmp"); }); From 4c761c84025df6b78ca9cca73d68cd883fea5bf9 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 08:03:26 +0200 Subject: [PATCH 067/124] build: rebuild native engine broker artifacts with canonical registration paths --- src/contracts/runtimeContractManifest.ts | 6 +++--- .../artifacts/daimon-engine-broker-arm64 | Bin 67456 -> 67456 bytes ...daimon-engine-broker-arm64.provenance.json | 2 +- .../native/artifacts/daimon-engine-broker-x64 | Bin 55392 -> 55392 bytes .../daimon-engine-broker-x64.provenance.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index e56f6ae..46a3588 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -129,9 +129,9 @@ export const GROK_ENGINE_BROKER = { projectionVersion: "noopolis.daimon.grok-broker-projection.v1", slotPreflightVersion: "noopolis.daimon.grok-slot-preflight.v2", artifacts: { - sourceSha256: "27fcdc8ffb6946e391dad039f695968231d1ae435c6a38dd9bb8c7fb00ed18b9", - x64Sha256: "ff83efc37c77c4ee920080d0eedfca497b504a561c876348af0fec25142a5f9c", - arm64Sha256: "25d37be0d294529b3466d73c0d879d18c7850c2d24450a7a9cc11d28b9a4cf1e" + sourceSha256: "c0082d4b366ffdb860d8154ee7f402b8f8be1f09d4eda277eda6965198ab6a75", + x64Sha256: "69e2865c722606a71501bc38d8b9c4c2c397e748327a8077e51e040319d1280d", + arm64Sha256: "16a3f89d84b7139d556b070a626c75ece224d5e05c52d48e045b38086c0a0382" } } as const; export const AGY_SUBSCRIPTION_REALM = { diff --git a/src/runtime/native/artifacts/daimon-engine-broker-arm64 b/src/runtime/native/artifacts/daimon-engine-broker-arm64 index 712890f0a30f784d1246975a870e3a558743b823..6c393474c393788fcddb983d037e3d6986f75d10 100755 GIT binary patch delta 8198 zcmc&(e{@vE^`CifcazNz$dX?hLfG9TBxFM%A-_NZ*$p891;vmZTN{Vrzu~lNsPKS6h1&zf#jx44b{BCla()sI=+5tqgn^C4Z2LFnPTdiTUX!F#VyT+Hd|aqG!C-s%MwoMKjL6=c>g{5 zLJMOz3D*+a9BJsx(Q)Pp`&1(2GEK0S!i>`NoG${&Ap*w-D25OH&)95Ln+kqBGp;kHobU#p66grW{fQ|xP z8ih`xqU3a%mYk62i9#pStw5IpT@!`c=@Fn01688XY}yO-b)cP5XfAyP^bAm66q-j_ zDX?z}>>Gtnr6oWY0}Vu>g>*mAjX)1Zp+)o}&|N@-QRoc%9OzM?Ls4i6C8ol@sj#p1 zt$j;rF3>8VNl|Dy-3fFxP3azERKwk&i8HLu-S3u7I^+llzDJvcJO;52nOfn6!92*R(Fh%*x9d z+x@aMlWv(b#k(EYd7S4!%y4B;PVM;=e2^&(FM$pM56a0s=Vb0Yd`0V$5}0LBPU{KE zDbBxxS7cT$`fM_FG;s9k7k#A=#w(PDRPYD_kV$cJNohEMK0z+{WapD;!xAQcA{u4x zYmk^S03`z}(Qpm}EAnu~W0aLu;Eh$qB;3G}ugV-)o8TRuShY$F08HnBJAAQfrBG~O z*VzT0s5ImubpHVZ+ZbzW<*d<%A|_ylTs9^x#$-&H@QCOKnbn%sIG}0iLWTkRC3p)28B=#V61*TTX#6QeaFgYt;>s>bwAHF^jE&!nhY(=KG&P+d0}{E9B?tkS&w%rLSz6 z1z$kIeo8ABzA-3E@FU521ekL2-?0Fd*lqlebiF;3AD}z!r9430_I!RHy>HJfs*D=v z8jSPLF{}*PNjOb0+M+kGk1US#DNl=WgIak4DBPee@K;UW?vYl5pRkLU+l~G#TH?rb z{16ffgVlcK>bJRUZLk0&Q=Cb_87%V)ddT76X8MgIi~oz>cgz_-2Q9j5Ue+p)uP{?` z_7vD@cD9W>X<2q|($2tKvDre2cDpin2R)LV$)6^Fc4k%=CT_sJIBV?)4rU_#8sjv za5r?_PR88F%Hq}RS-k%liS@sRwg6*&x&y+~KN`iNfggh|CMg3KZjt;KGua3qqYrZ@ zUULP_9r_ZIdk9@GZZ@jjVg6C@92Ug>_dJaHU>JW0_2w}C0P0BEoh^YGR61pL$sXt% z#FCF8Y6y=@75z}hbOr=((fL%JcY)tQPffWoqub234I)N4PSlUs8S6hCp6nh<%=7a1 z>5;sq)J4#|A0-GmcCvoNg+&I4=fA?IQ-A&q{46C-t>$}Z$CEqN#;@dN5KldoUV5krmB~5r1Q>Qj(gE&i(w@m-Fd9ecJO~SMff!5kH|Rf225$$v?hKR^-sHoZzf|= zb4H0O4(%`w-Z8k}AdWv4+R1z+s$JF0T*((oj*OuvQyj_#bkfgh$*(^pF_H{CenZ~SGNTYQ}FB1cIk-$8Rr z=JQ7CD0!3bC-=-uek9Pa$}0!`Z}q)>z$IF$=+=#{t(V0`AFUeLU7Ro5eDf=drRCn2?MV z$tSsHbxfgaXHWJDLkM+))odhqudCmY%lZb7b5-KXKpc*oFNc*Qr0Xd}^Tzu3UW5;n zvU03XWYPg)2N!sQt~!+b9H=v#t#x7wGg9kVYRaWnhSijgNaTtX}P_{xsX-EK83rrl~;O(yd$OtD|S--w( z@Uau)mlViMcD68s^LT-{T}^U|*=M1zN9Qx0&EPY@2eGTd1F_4-_-S^qJh2tYwULcy*V|%Q+dG?X^4{mdbw~^>1hztv5W+pOF&+*ZhtldUQ?KqQ zQ_ZNyq7)34sj-f76))mLcxxQ`(o?R!>nm4{sN=0{Xh*PI-QucHPZy5&7b4_2QCi(x zz1l6Qr*e|~1>mQm6m&>xY`0fZzs$7w@q*ZidtB$20ZDzA$?D&dC-|p;$ID{p4v(yE zQDjw|AiM(%)dWjb2P;!A*O}De=S(V`8H9;A(P4%D3N%LQ{rTf1~xB5?bGyEyS8oLZ_ zD<#%=u>~m?9fHVpHOv5InLQoo8y}Wg2(wUL@d)iG*MeLdFmH6oneS6ZJ`>?0~66p{wueFr-Pli$v zlSGIR=k~uZ;Yfh@;KE~G;q67{W&d|x8Pc%8p}=UH?HF|sqY3&>WTT->dhf{F|th!HTFvVhGWlB z28Nr={!2H;`A72O{U>b+{x7_Vew#2PbW?ENq-i1;zZ|yyn^MN<;8)^s{5v*?d)pnD zFxjh%Et^ph+i<@yAGT%#HCLB4hm#C!MiwqTIj$wV zj@n%L^G@O}vX+Ucnc_T!9>EQ+ewQ>{u};U*oF@e41qOZ)b?uv5Te|b}3v}2uS^DGx zeeL?;#Q9>pV$)ei5_0`>b^Q#fw3@M1-Q#JU`>s{{;*tKdnd(5+UW7qi@pwPU<2t`U zT#w?wJ0;bTqzt%4yWpoAbbhXAH-pC{MTkC+eDecjJUg&oUhz-|+XBx{b1nnsi=}I8 zT&rW$vCHu0q+y9hOfE%c|yvBYP+k=VZ@x@cK6pHdEtLC zL~QO<@M}2BXMVut%&otHjtw%~iuiK&E19m;%=G^LycovSzd#K4 z_WXX^^6WO@7{T9%>L;<$BdxXm%k2yNi?K&y-hy9*x}aa~H!`Z!fGC8@-BeUpEgd>fch%+bpV4D=4_kHx z=8D9*SyuGFAUbx_%=&6eUkH_P-4O?-<5k*J|8u^LEDM_~8_$XRkN!UmVxlr|k~$X7 z;YVoi!fL*Z#ujGswa8S{3Ln(6ykhS2XQ;&@Fp44R6t2_B<-H zklF2TXqpFf3+P(V+<>ON4(i&cX$DyBws$n`HqiI>V<6DL0Zls!>iJmH){862A2n?a zt|_ihumI3b(8ocC4r|&O(DR`BH-$f8Vp!mA{5|H184rPBO}hy|(g+lgxjkoI0{$_> z-#LuAV+As#ylf+E}g60K^_p1vPVn4JEk;P+vaPl1W9 z%snpYDd}l0hIk(3;7c@b?esW1sz*^$dg)*7+0vF?N?V&Jo#{nmg;d-}>()+}w)N4k z*WNdA`EOx7h(eS7DZ|rpE~>dO>TSQJtM185dlk*MiDo>tC>}J$zDDcrsgpdf(;FaV zd&#u!2d3dSAs9=m)@9vX)74{n+;6FUVyETF7cBFC^=r%a9TvHr$Nl-}kRN859#jp*G~Tw)zOc> z-8<3c46rl6a3|Iu+-ddCZ6`jWf2_~Tng-`udFb0NqE&2pt#Cpr`El~Z<)}T5oR=$_ z@Q6JTu4?f2{@}-NDb10uCXtUM3ApKq2ggW%#1|UzM0oK-xSxWxPHtwPKFIV&T$UJJ}_)a%E!?Hf|eIq2sJaosX&9YPX(a`rd(XfV|7`=`s z(JAy>Pv$YN*B&14JEC6p9Ss~F*NfBsU0ML+f2R|eBD~`Nv%pH)qgCR=Uhm(vuQ{*k z8syw74Vk)DNPN2GJ-l*?Cqj33bxR!nI4%T7=(eswz~&?bY4Id=WyrkJye7=9=~@Q% z8+VPkt*5J7MyJS`i0wKer2ccdo-T)h&zPWOhEzvr-_ouXz|G+URJOY7NdtxqvrAWZ zeFiqEJ=7`ksXV32VZ`jMYeR5^zS`9c*c|ypy0q&FBUD8`fO@))2=<$A@Ld@uOfzyK Q*N=sCJ@`tqiQD-90=M`?2><{9 delta 7830 zcmc&(dw5jUwcq=k$>a?)BxLfQnFR8fKpv1u$c2!T1V}(ph^b=Ll9;wq16E!ZL7fTW zEnZM_y3j!d2|g+lrG+L4sc;?CQratusaO!bOn_FXkDFWs5=c1rx6jPL+W+t*{)8xFT<68F5vBgB&VmN;5It-N z;5*37E#{`6R#Q-`i7I%LZCOBt{S%D~SYmP1!*8|!UmffVcCe4~rQ7&T^r)0;u7yD* zXmkqbY}z9gL|39V4rLCVmx}Ud1m%-KXQC9LB%qX_M55SHrlU-wYB?dg9yReyrF-Nw z^Q-{FN@O9rZAMH!9V| z5gAh@#!Q&74aK8AhX9FS^MQ$nGnthH<$~|h_A#m2KB7;GVYBkdktC$V(8GWz8dMOO*@tRD$PoCM1&!x*KUkAkut_u=w_hFVQ3P~1^Nikyf8GG9tHX)&?#YPD(wXN zIneqrG@ZTx8kZ0kzaR|Fr1%62O#`|p47E`u(8WMkg`rur1n3r^>%-7-v=!(FKsSe> zIdlZ*Fwh-gXdaCKosbCshM@&CF%kX+dN>T7NcRBU2=sUuT0~C)-4FC!7+OLHfPM{h zFbthc-vCV?3;$|2{OhEQvG6a@`0Ftn1s z02-GB|AwJe6rTkD0$mh_)=(wT#Xwhup)+U+&@DjMhoQ4*E6@*sZVp51=?KtapgY3Q zIW&@F$@+)PHjS(~xWgwe{4}&F`q#YU?Pu($`Q2GVCW#6yme@n6WtFKm$@*!RVo?mF z%EjbPyLt4$HO=FWW=(((ByCO4cNtVMh!Ws9$4c#(tl;fU z26cky0GQ4Lw|5xSTw&P2ZnX(K-rbOcr2cz!Y-Ma|D`$-zC}IFZz_KuC(I;c>F>Rt9 zbQWt`W3Q%Jgib$nK1u{*=k5j@lovtCcSUpd(w!M;DQCodptd}UZ4&epc$jKvrl&IQ zaJGW;4Qft15KOLt^NwZ0Mm%hY3HmQ<ZhBWtFtPJ;i|n04DVf=>3}{@`eV^1h z9aaKrwX&Jx7aBNtD zGjd=V4cHDZqX>OBHD{HlJO&Lv8!YRPFdP#Gel0NYO|&~}p?L+Q7vX?DIR&mvcdR0N zwv{iU>g@9PABvX1t6}R7dLsMT(nvLH3h&t_v7X;T=3}f|4?slvJHwbX@YArxBzNz{ zyCm=BG&VFaOt*|nz3E$s+w>_Ubc!Kgemp|m9^@Yf&*4F=f2SMuz99Z6>dirXHR{=n zwYT^RsB2tB$#&T5!<0`WYlw)85baRMaC!yLb$+bQE8w4{jPZA*%#LQ8`jDgSXX}P+ zjP-mK9PD;_YP^g0P<+n9u@$hqAH@$ncCKzn!6bK4Th3mdNfo)b@jiMgw~BA4Bd8xI zdtMiRhK}Xs@+W8{@8{CcReB)*UOtaL&Mz3-b`@KS5AGF#eHs(7x49b_nH^SyVu|Bc z^G3);n#*DxB11K-C)H8FtLcE_b^CpgStD6VlO8V||;gl(&HIqyD-)QILFL@+ATRJtme^A8NVA*OqQCeqShZd(K*6N2h ze8XDVkEz6&hC$4Amh;Kz+4i1s_+6}g+^+S#TFZq0h~2TacVrFoj>t^Ca2xl2cP3H2&{D8Lgz!Z z#L1-R(LveRIfNa5MV1{yvcVyC{Z(Y3EVj0{?>^Qema!5;7rdy8n63ctI*p204Zob~ zxL+iKG9-xixXO7)Lo*cIPUh4uV`E@-#A)$H8Eb4yLgHAi^az8YgW@fro5q|rP?3gg zX{=Fn#8B=6){)Ab98`Wpp$PA~UfiVnYnqB&>cy7v8+eqvR|MQebR?+g+z4gSsV{l3 z-05uN>+Uqm&@it_HnFm1({Pv9i&6@KU_Vorl@I&k>eQD@j&_lqGEGUvOaMAQe4g?oV6 z1Qw%XE+ja~F$17`B6I#3TOr(#?nD?`0J#FWFdQ!T;k=s8YVvxqHy)qc);&OUQ$I5IRec zXvJD;EzX`NArtW`!bhxWK3uH33s^G_L@p~s2MuhG>#`Q@alDQl)8g1b?rDip`8(Eu z$=^V>%2d0>ks}PDgV5n}3A_bYsgX5`7dYKjIWcbqDdiksdaqp&i-S%J^68 zXxj@90~1SRWht$1BKO21perJoxk^rL#oKJ7Z8V#2jbcmBuDx@$g8Q9tZ2@dthCS%^ z$Dp)U7prsHiq&Y;qZo7b=l2z>QT7rQe>L>u^|=4brV{l~M~NDNI$l5fU-Or!>y%0A zR|TWJH;h!ObE>DS=d(@TprQW!w&`lr_UY=EX=d+rTfXV)Ay%dSEn$rJy1k|NVT(K_2ew0I`oQB#WqC_X8kShNhyqXX zZG^7BQBK8y$J%QcQ+I|7aC+-_6H{N;@q|t;uhsLFj(0G1yN)Mza)ZE=1Nt_uzDO0* zXWq`$C@!h^OQ0QB^Y$-0B=wMAQqR^(>Y2x7efAM-Z9|Q1vVQZhWxIQaTBE&}?uhXY z=Ei!@S>wEacg1^)v3$Zd-XV^OLLW>|&k_}pF!valitJqRj+Fcr?)6x2i{l_-Wc4uJ zUNy;_jb#@b;NLJ#u_@Nm52_}Yju9vn+n{)-xLH6_jr#4uy1;J>)|*kEhdTKbC2bDO zh&K+gHq#Akv=Lh%-N1@6a0sVW&*huwXVtmWXI;@eE3iw&?XX;QyhiC!q`|U>bS%j+ zNnkEu;EPb#uHPaoj)F_{VRfdIeTlxU{%NXu5&ab#2ihdo*!Rk;9){Fj#n}D0i#|~E zfU7MY$59qj?KnkV$AQe^abM2iI{ylIcN}39LIp)kdkO|2U_@g=wEDs_8jhYoxns3+iQcCy>d2IRh^tCW4=23S|8{i`k4qd=WmA^&NKEFv>Yf^8S;i#nNbYDd(w}Qn zM-9Tn8kudtiViG^p#QF&)lyP^WhHv9|QG2iA`T8sgg~9sci5= z&&FA|to%Gf`zl3Bd|Y-n*swL)9;7k#Rpu1|RK}64 zqpRrN`ltElG*G|L{Qd=T=;;3?AV0ui)I57C|CqMU7Ps~L z{bWuyH@}QV;)*94^kJVjbUlAl9sk{2gO#76h{Xl`+pdzue=)km|I#+NmUU{{VdPiS z?~nwLdwt!SHcrU)XxbFed7w9gZUk)t^@FYhWp8QPW>6F8E+Gd!2Ws1^X>k(XI6$!o zY{EWGTMX(0-2!TON7IsJOoEG2KC`$i`xb}`v*j8*ueIx^D)G6BU8tXN=p1g4-21Dgga9vey{%D8S?wj!&r5pKia^d}L& z8&R5oiKeW2lxeH<5*J;(in5@aX00rw{uNQumTtPTB1?+fLzydcq!oK0D3kW>p~qGh zO6hOWZ&t2OJ^Qw%+0Y7BY+DU4$>UHR2SyvGo*5`j5Vh~8#~zv` zojX8>K=%JTjrzrp?sUViXEAM7Gs?Yz*KFNoe(AU78v2cS(@t~kvu~Q8d&4}#(`nx9 zHP<}9!~DYQ=Gqs3ZQk;lS-KQ+^79j){q^{di-QEQe_FKCo%PaBI<@Jp3L9R(g~&SLb9H5Fh(KzR^VJh*eAn?xW__38|&19Uj2e z5F$Ln)%C!-JG-7<{jR&qzdDaMhyKJ5{b#>1SJ!3O7WyMT7>Fmtiw_Lqi3<>>SOS&! zAYa-VsEDsG;;Ecp+D9GYHSrP9k!Yei;3eAYc8I4 z+CF?MFQx&s+f3#Wm#Zn*-}gja?0f1s*slw_<@@vi`u|=BFi}Ls|Jwt!TPw%+rmlZz zmpE5EeVn_bs7TMf5-&9Sak&>yi0<`lmw1tR^QZtEqR)By07n;(4#35p6q%Ptw+GpF z&z)eehixc)S9!L}Xru&JO7WZ!TK_!k2R(KJFB+qNtQ6M8M@u}Vw|edc9v$3BrLCS9 z4Cpe*&TI922G+DJpd9*i8|Se{VDzTt0XRh4JKSCY4vo)N|jp&!BUAp6@)*W}WxlYp=cc z+H0@9XNH!#ke0fTns`IYz!eFb!|kz>c!l*G)R8Bk4?sqz~lp!q>AEJnQLo!{n6z&s0IcQ~`yDEb@%f zn@BduI|J|C19C8A50rrcoDDewWq1Iyp?)YmQ5-0lD9I>8Q3j!;q1aLQv53Etlm*!f zg?*GGN!=v_UPhvXp$tG_2MwdB(5Gp6XsRa?h@TXc5ELr|JaH&}PaWV2f&bkH;w(^_VvgKIkm@0^F-BUPFv^k`4rLy}(B4ivX?>I8^43y-IcRz$`{eH;ko4y|1JJTi`& zBEzCS{XoKrm!9-|4`9bryFdc6jNLe^$)e{HtDr(G{qDrx6?e+<2^d!glpQ) z>Giz7vgjgoawy3hM~UVxT`gzzy3P;=ng})ztUZV|(qgcsV4Xp17<~wKJ=pvpHiEta zdj@Pl5Zi^iM#WK=s4iWLg4jsP0h;umLN8fs=+P?+ZM$3qU~U}ft3ROWS?2+ zBG@0nnuFNhloSpBM#H~BY+srPHV>>li0wyC`m_{*8fugIF8ofXxP57Q{MeKG;IAH9_oPdK>IpU~7Zep>zoBL9lf}>~OjP_BvQ! z5Icg>W8vRe_&12nuAdeAP6QRjX0NnFd&X5tl2ZOj*lrFvZRpqA@we{<{52&z@y2*A zSNU5sJbh0w=AGtm61bT$=PSQY-~)^~C;ICI-o=Fk-)tf^AX`M5IBJ`pMd^+fx9tIVC)n)f^kpA zc7Y9y6B%0szP1fGiLv=6`q%pY!bC47B%$yF<7CEdH`oJTGqx~p5%?726vj;gH#6ol z&+ilX0AoG}{dEHGV$8YEUn_6}V?Gc4H8;?|)^{Tle9rsJgu(}m`3(0L3H&bORK^7Y zzrmOfSbx62D;e`K?RN_N5@Ws}`0WC}z?jp&-y-mHjQMc(n>~V9$ONCteo5fR81v=B z-*%k`_%LI>H27Ntet3{l4YZ_%-Qg-l;RL5he-aVB* z(=yJGdEW2UFf`u5n@K&4c`4p=F7F<-9AuvNiuY1=euz2;D_-dJ-Icy)7pf({ zYg&0`XmN|$ijgQ)Gs2YWq34(5)q75jM{5kY;s`8*J{q9q0DVxW4+iKTV3oD^OH|r7 z$un4t>ls^d7A%cLojwjsh260O-q2KD-%6jS)jt+UuZMCEDEo(iarhcJm+1bh*UeXs zT%Db?xU zz6}3WZ?ZP=EMUTTP1Ck;Si`A`_orXq)0d_Zw!rYxEP@6v%{gA0`vSxH5G^pLCiWhz zXi!Hdm9K#oOT=WtbcwrSv=w%p;n? zF54r!vo4mLc3gpXlCIJDeu?rsbhBSi`EE)`?J0jx!%|b^F*GeT$;DqNoK|IANUa0LMxVyeZae8q zOy_-a4}8d61-;fUt4gh5SUz>4l{O4akY~}sfqhIlsA}XRnwQZ2FS(##!lvFbCUZp(eVSZJ--xhaKs&eK4sNopAIve|Gukj(elr~acNdwcf1M^EA@{v55&JO7#e@2F(^W-H|IJBoc zpOy{%N?uz(XV{Cfe1v`)Zk0cv=*(>SZOYGFefMW7)-BYcuiH(L>r53X2rb^k7GBKv zf6(`gsWO7;Dz`P?u^;0^ol>1It64N8YoYudt<9PnwHVe~dnkyIc$^z*q)L_%PsrP; zW<-B^5jBkHZ7%q4j$TYh!JN|7DfHur^_{KXVfMyGdS~P;LznN#kgfFp1yt6a;_e8; zH`Fl~^z2@<&uvZP_~#D9PE<$GqU_=F!}Ry;vGU*Pmuy$}`@hk&D{ku~xg_DX+H4w^ zlPbSS3v&kHL*|;C!}9A?JZgcV_tSJ?R7#}xgzG++_wW|}Nbb6qqDN=T7RnzT@42c@ z4eav6IJlH*?Bhm0fz&%NB;Nf02U;BkfXoWK>=dx6(vQn0I?q(yU;iQfe*=QsCF4X; zk7KAJX+!~uWcD5yu9mpE-+2$Gh}hyHYSqk5%}zrF@Q1(+d?#;>6$iCxbFQqE?~}Q1g^~W@ z>^^WA+QL#ccWxI(778Q9R(k{7CaG&L>a$M@qf5^22`8Y>&iELaQ+VgQ7M)m2$*x$> z+(5M#)h^iW;=Sg*sdfwKEDEq;g57pbZ}|{d^_3qrt>YlN;_&SdeJ2F0&4$Jfz9t4V zME-ZF}o+g{)VN5)v!iLN&RPly$R&-i9=MJ4m(7x9H?yp688j`K9>98W072ytR}@Y zc^Bqg-X8M2RRol)dhCtzcEeL$$7KDBQ|$PwY6I-J%J<4+vTl}~$C)yGK6=~#tLb@T z?J7P`H(3iX_&JJc0rrj2f!an+A9>!Zb4-OhnQsE0HJ=W+d#24q6QIG+;+ht`D|pBL zJHnKya2L}m=&|!DYD{$BRK3C7fd=iU8LeyX?&xyVe3~*Qx<~t8v01- zfOh9Z>CBi!PbnH= zocq`qPAFllNauDkXAW?)b#51P7I13)8BNO>h_|Qj=Uh>PESTp#FMP#&HHL!t{)Tw94e0Pk!^m|c}iHlS>r3%9vgKo+*B~#@xG*y$(4Ud{_i$H6fMFaC9pO9+5Nk zYwT%#2;;n;sHLar>$@MBz&SBh@qXDDkA3a%Vcx#SFq+y^oJEl?6|Y~gyTLYEprclD zRbZe&PQWKF-`K%PdzGr`XjN@|0Sz3@rta`NhtWw;OUaxY(`{czSe8J8>r-i1Zcf7Q zbM=+JAGbfO{wKVuW$c(Ts?N=jQ)ydnpAi4Z8)Q=)(Hn4k#^^HDqAzuCtB+EKGRRX1 zKCt^QshuzBbz;Y=zx{+NY*evGKLW6~bVr@-Kq8Ygbs*ugT><1D3e)&ZQ~?W;^ruD+ zOvU@Vy7440k5b+)KlQBQ$zi(q$+D5QlESjm5Y1NnUk*5HvaDq7Tr@4nTZ;EUYfLaM z-vymri#nBMN83t%R6-++g(`iX!bTe zh^wX>N1IT&1eJwnMY?MJwybodA*19oM|10`+a1ylWwD{=aT+x)+4Sh+Je3M-9>%V2 zJVAxy?sT8R(d#`|dHNZnYC9o(Wv|o#dIKmsf42ncqV>8VqOKEwt6URTMSR>jPRGZM zGWZ^&?&BXcJbr=}jepN@?|JGmp{F79JPn&*8PZ)pt11$1azt&j{*DdXxCHK1uj{8@ z1%Jna+dW@Yy$CDyo>qEkLQ>SwR*}~$_G3+O?;bUeKAw;`avCD2V; zQO98h=Lw$$&+DOM#z#l>HSl~8K1N;hlKSMKW{g8?*o0vqILs5Oi*uyB{@2vC$IuP* zUVfA&=H+;tfJ?SnmmymIiZj==^z?N>EWbq@0KpTALI|lY@yXCs@lgkfAXL)=B!9)h z>8A#5#nd_Dc=WTZg1>xFs*0^)OpZXWJk|x7f;F~2iWZU-fmD@q@R8dy$8@DH_#EHI zvvsk3d`RA>KKKRApEx>p06IbjiuY&2J$c@raEPe)(awo+hEYf9)WpPDW(*H4D=uLj zID14Pu{Br#a4}Q+9MuQW46G+5O?{h&SYH2E_$11!phm*J|05ti(Pakf0B(n} zMA{^$;n3&gn!La;v57WJo+Ni&nQt>3Izo>8vR+$fOVYmF!bFVD6xDf-=?UW&Fs7(n zvqRfGhhE9=9rLEHV!5sI`fM^zSs?eQFP`F*4JQxRZ=O1_vtdeQz43v!#}D}jenZ3O zyhwZt&H=2^2IOo&f&!P|4)EGRb`q6)r!#CE|7^7ZTqG{PES3R2cAWEP5)gOH$ zd)CA15E!nVt7+|cw-;)f1##b2;0sDLEfahZWFBM_c7_tr0 z{w0Uti^_34(WVR)6PH^tkASH z48XDy9YQ)+Y1(?orq>ZA2qp6ynl>FWAF>G2@)kOPv_qR94LxeRH?CB6n9Us@oWAd3*O<_Js}(wd5wI=o~;`XKWln;;7y+aQY} z&25^tj8eArlWS<)7Mrnl4h+)bEq&xLTD>JM*0M;`_^Wiu9cj*kC_+i3Lt7@xX_VB^ zPcEi04FlzqU~(hUP&XUPsV<^j4H>?c!-Z0MZR>bc zoCk#;t+q|JO~*RLVKmZRk>*znxsmZNhm4Q3REORbnY!HQjrBa8d!+ML$;YCnE!VV<@GiL`&E*EeiipUtvHWYnbQBJfgQ$<84cnYaxuKEECgw%f zOq>SkiY%Lm6ab|rC@o(}k=w@_6FElc{_PXe3-MM)nl=gw{9waKW#HBaaRuOB1$UA@ z-##;=9dGWxXd1#uJlvuS#}+uA8b9nI3%KmR(2T|bo_xHXdF?-&%m#N79DiTO51V-g z+y!tZ!RWs(F9+8Gj(;?ja>GNyLe9y%1N!Sww+SIVE1=Idwt+Jvyl39#JW>i|-VV-4WVE_xWBk zZo=y-`q?+K(`CGcU7$fbM|k)8xIllqVi1{fyRaf#t_s18mBn$#N4tc_>-Vrxk}DHArXh zYkl{6^I@M{fAYv&1HR96HG814v)L9E#+w39hs-7Us4XhLtIl@FQgR)&g*OG*CVKYh z@Nj3qZ%*2Bboi}dbPUHy@-Yvx$C6{V@VY*_wL02zY&g{&GaJs9li`buh>n@puB8!Q zEX7xIJHN0++41`VKOM4-)MfjR70!h`#*-bVq+b7@t0Kcy5ruX7gLjl( z#gq=&&v@@<$PCEtD9HxSfgFL7X<&Aggwg|L5K0z`6J;pMV3ZUTHwr(!bYU6}nSjDE z8fh|hGnw&{jS`A75Ty&sFp3O%RQqCWX-# z_#01=!Q-TtXtE`S<^>nh`>;`hOQpXF#vkI4w$Q4OH0dGQ6Ou;uU>CX}4y}K|zW1X40&z_B_gqRK)+>5JT#<~kO9W!vf zz!uXv1J?>1WctRy+XU`p`mcc(#Lf=mK;8!Cr(@yrbc!d%L?f&zRAvqBUS}-V4&Asq z-DC%?v&KaFf2FOUe_CUx#Tpvf{3}fpeF2T#7TUe#SDKC6bdL(TZ81_C-D{iEy<@?! zrtN~R=dCuJj@V)%I;NU=J6#3i_*}OMfiTl;TMQ-JJ4Yl1uogtPr})Si$_Kk5fbBvPz%B;6I)II$ zC15v!-59`jqs?H?f!z_nI_Nyu&Rsf3ydS{EQbd;+$_9HnfQ_RZu*G1N0JbMR1@?8Y z?E$Ql-T-?TY^V`!j%05-0`@1cjsVt0x4|ZNMSKI;ew5M`@dcY1!1kwUU{`{b1K5F7 z19m6asR3*X?E-rlY*7H4MpwYbMIpWctefJa5MQt>0@%Sc0qkP1s{_~}v;^!Xup0x| zVYC_SIj}nd*ep5^wsSP%8^C5$M0AXlQ}2%cTlfpku3lfI$s||o58cO$P8;&sF8obA zhrhOjT)Z(ZVwKvW;YmEsn75kREbxbn`506E0`Fza$D~>(a3fJHCNzej5(L9=>k8;m@g5kQ{ZP9^W{Lbdj;_%6P&ZeuCIA~g?SEUahR67o!@ z2EJFhkGtVnv7-&&zAFEsXOto6Lq%p#NsDp`J&~&)3zchztXqj!-vvd19Xecj6rMqo zK_4^d!#cgvpc@VP0~2K@#(O)7ewC(`WWiG~bb3E96b{F3L_pVbBU9s1WN7e z@b*Ow9*+G5PI}TDoA%nRVh1I=Ng}E|`yL8somA^aGS@ zN?ZlI`HWUd>HEI!QNhBdbV$jGF>b&615BSk&-dMKK7y2Q(VHrNNrSo$BjdSl00F;U zAk?$qlpQpqU#}h;gq^MGF%H=QDKG2Fg0t=$h|c+&w60&A^fWc~>mhZb&+zvhy3wzf z6itq#c=Hc8X;@O4bdY8x^_(-S9lo)67j^YiWV?b&zC=SOz^Shz45DRpB@Ufv0_na{ zaa0qttk^{3-Yb{32F&Ech&0Ej3)h_h3o(;HK=8VR+LC$?YJ?lC;qdasu1pSXi50%b zuy0MV{=kvi=^Bl7)WD}}eofBBYRc=MhWm6${|sqAy^HrJ=uH1a>0{FRr*vIgjkxo& zv-PiZ<>E!@Nuvhzy6>m&^`5*4m1hTUpnRX&$SJ~iLD(yw35x{_?`k<=bZB?#+Nsr+ zLG+ISW4rzXUAr5kGZ@Zn3K}@gejn^EovB{z3dQs(6Z+8nfw59AS~;+f?e=v|YvQ!c zQ`mN$ng))V_)5LWbbV&3Euj*;p8w^VoCdIRjb$r~toD*XZ6}`TMKnOU;!A(}$_OrChp_>WH3o zRm>ZP+49IV&!hYDa6sKpv1!xo`=Dv$&sXjnyd}1gFD*+7r+sN}_Y8u{Q}MG_f@(h~ zH`*B87N}KG+Gu(Dcxf>mOP^!*e@7$TPnoZ_(q{KNQN>Hp>HD#2svmIW%f8PsdSo@G8?PQJm9Ni#?p{4X35 zZ)Wt7s%dY=Y{^5(LwZOlBo8?+JzL**=$|C%FZ92|T+(xNVpz5`jqI80?x|KVZ$VCd z-hLKmovpGLmKJYfCr{?0Z}mN6s|shjI?q+${s8@iPOd4ClrXxU`HZxkW@QycPJp*d z-DRv02l9sU`8U*(^|Z8v3WoPbe4iWM+wS}$uU-sC#+Xu+NZK@fQ^;5U!RReZY5IuA z&F8+Ry(8rQd%xDStOF(8ungBImap{Fz4lO^E1B0nHz0PR5>A7&he?BIPWD)74sFf$ zbnE`5rrpSMO_UbI-Zjghu$&|q49XcKIcP@C$M_VRF>LOs%f~e zqVY^6=;E)A|9TKyuaWC*fD>_OYyuL8S&8BHHCBJq;^iYEI%6KTeJ-!_-@h-{RJ!cA z>fGx)B=bK+((PA9e!)k&Cb|Zwp%J4$Rp<%VNyp`&Q3J^HAw?hLiVo!!R1-MQWmg^s zkg*2sa@BF$^Alituu_iVdYE*MH{Xwx|Fpmg1E@?9j!Vt9?YZVk;CSFh6w{?>k*ICk z$4PRbL zCr*UMJN6iUGWZ76!Y29!ZwJ2)JX%V+tMK>^g7@issA#j%?nU!R=TOXNMZPQ9cMzUK zu|e%&@8^ZJ<%KC>ZMGF5=n<|mV1*kM5 zdvI{k&uwCe6+nT=Y!+OHz_ z)zV}+hTa>KFk4fugXc>lw^fRz;6Y}W%lA)(&pcl{AMu`Pp2s}X_qqMt4Y5G=#2JpB z$+%vL{lno=LXDo?>)Wp!#$Yf}=4e$0z^MmUoH1G7FZkk`ZK);l70loV$eC}<;akEc zS0Q3IQ?|{;X<%6gu89xDd|&HKTk#&|gPWmvcMc7|r$_Q3T;YKJ1~XQ*;C%<)ajc9g z24ySW%d`S|`y492r>nPCuTXDPXoqGC-uaH{$AC~{Uq{;;!1GnakGL?`%)On8`uAcE zR)Lg#IC;=F_$Y(zz2*B?ao26d`&pAIdi@2Ie=r<1o3O0dNgQ_qCVa_(z8HGcI>3~y zQ;!=}TF?yF`0q=4jRL6Mei$F*p0yd;bI^i2#2k{hsL`x*dziBu-2RuatYNs9Nhe69 z$60-fYjITN`_S3wPH5RUdqYL!u?4^Xj1S;1jEVRGR&7s;&3_IGK0cA}&ba;HzXspr zz*(T|F;ouOrwY|qtm^4x-3Hbf3wdVX6-q3zAI8STACRjbbvY25x45C>wr<$^JO`&i zZJ_jVQC$u)Yyo@;7OZ#DZ^z}t_L!{?e?D%D80mf)N_1-Z5);>&woD8%3G%?e|+D(TS&aV3sq-JEsE{5 zXgs}2126-~dwz5px@(^bU53zo4dAKJ#KlZgPJc?<$B#6xokKUqKVp95G~IvSdh>#d zbo0I*=BXE{TfQ@6sGhPbV}Is3+3ETbE1{{J2Nh>%_*)L#rTJRu!=xxBm+8s;_{ay5 zXK>(FesG)DfTib?v^qa7D--eW7za+U>lhA>?IGy@kpbpe)m?rQwt{5bS^!0_E52^tx)%kO&DbA<6rEz(QVgDisy-w-XpF_vx9bK@k zH9T^kN>{05qLC9_sZMkOX~CiDr^(pIEB_7fMD8uuw8bdH$~5gYucrOJe%-&ml)U`A-e5RiTcl}ic+V}?GzYd- z8Dulo*4h$H^MhZzK-1E}*Fok%+81iti);tElkv0otOq#{G82>C2FYI*7ofQvkY$jE znTKqL%tco^2O-C_Ez`8A*a3+LOLMJAd?|$A#)l3Rnr~~L5^v|%RCSTYtay- zA96LMa~&K*)e<5WErFanfo3hf<0c=h=@SuexPY9SmMMx3^@<$j=#-z;wrrn zvhF+7gKUBP39=2+?hJ=xoV6}UJ7gwgI%Ga%0c0WdZ|EnTrpXOymW?wtZ7o$b^y%(= zQq$1Fod8Mx z**Gb>cCn`QVlR1CXA(p^CVCT1`}e?ZZP1Km4IUXK6H}YKgx>u3c&V1ue-D%%py-{G z9$b&f+lsN6MqBNFG3Qww&vzPQb=CxpwI}`YIZ_ zYpmrbyl$lOUH7HrW4S0VpeFYg4~L@=Tom2q3c;1|vZ9;2rl+39+ol&a4eLrgT;nph zjo^4j_(|qJFj)U0EpHm&Nyls6%fGCW2QC*}$M3%EW-7RRaQrg^KddVTC)3rY81D+a z#jVz~$AwPs>g(W2!A%d~>cH&>hYOB)*z^T1_n)wErd&Fm-+!?jIT4 zj@RO&wABBys7o@q=v(3k?rSoJ(j+BrqWH+wsXsPc+mV{haQ=PpMw`2 zNs#8!t4C&;@joX1Y3^3<_}DK=+v|@VecCLoCeLxNq|lz@X^~E>cYZqLM3PRVMb3@T z*$%msJSWn^+6)%%PYeqyGy+#h4JU@(=}1S1rjm5hi(! Date: Thu, 17 Sep 2026 08:03:38 +0200 Subject: [PATCH 068/124] docs: record sibling temp attestation, spill directory pinning and canonical registration paths --- src/runtime/AGENTS.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index a0d4517..3551638 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -168,7 +168,8 @@ in `deny` (verified: `/tmp`, `/var/tmp`, `/run`, `/etc`, `sessions` all fail; `/tmp/sub` works), so the profile cannot hide evaluator temp files. Instead: - the launcher exports `TMPDIR=/tmp` (strict adds TMPDIR to its read-write grants; Python, Node and `mktemp` use it); provision it - `: 0700`; + `: 0700`. Every registered worker's private temp is attested + before any turn, so one misprovisioned sibling refuses all turns; - `/tmp` and `/var/tmp` must be `root: 1774`: Grok needs to open the directory, but without search or write a worker can only list names — `cat`/`read_file` get EACCES and it cannot create files. @@ -177,7 +178,16 @@ in `deny` (verified: `/tmp`, `/var/tmp`, `/run`, `/etc`, `sessions` all fail; - spills (`toolResultSpill.ts`) are written `0640`; provision `/tool-output` as `2000: 2750` (setgid) under a runtime home the worker can traverse, so each spill carries that agent's - worker group and no other worker can read it. + worker group and no other worker can read it. The writer pins the directory + (`O_DIRECTORY|O_NOFOLLOW`, dev/ino re-checked before publishing) and refuses + one that is a symlink, not owned by the runtime, wider than `2750`, or + group-open without setgid or in the runtime's own group; it cannot tell + *which* worker gid belongs to the agent, so that mapping stays the + deployment's. A spill is published by rename, replacing any existing entry + (a planted symlink included) without following it. +- registered workspace and home paths must be canonical (no `.`, `..`, empty + components or trailing slash) in both `service.json` and `registrations.bin`; + the launcher refuses the slot otherwise. `agySubscriptionRealm.ts` owns the one host-level private D-Bus/Secret Service realm, durable keyring lease, bounded unlock stdin, and cleanup. From 92fb1ca5060fb9c11aaf235f1a245278e573111d Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 15:52:07 +0200 Subject: [PATCH 069/124] fix: refuse Grok worker deny paths bubblewrap cannot materialize as the worker uid --- src/pi/grokSandbox.ts | 6 + src/runtime/AGENTS.md | 28 ++++ src/runtime/grokBrokerProjection.test.ts | 18 +++ src/runtime/grokBrokerProjection.ts | 28 +++- src/runtime/grokWorkerAttestation.ts | 14 +- .../grokWorkerAttestationChecks.test.ts | 34 +++- src/runtime/grokWorkerDenyPlacement.test.ts | 118 ++++++++++++++ src/runtime/grokWorkerDenyPlacement.ts | 147 ++++++++++++++++++ src/runtime/grokWorkerSandboxProfile.ts | 10 +- src/runtime/index.ts | 2 + 10 files changed, 397 insertions(+), 8 deletions(-) create mode 100644 src/runtime/grokWorkerDenyPlacement.test.ts create mode 100644 src/runtime/grokWorkerDenyPlacement.ts diff --git a/src/pi/grokSandbox.ts b/src/pi/grokSandbox.ts index ddc9ee4..1c448ed 100644 --- a/src/pi/grokSandbox.ts +++ b/src/pi/grokSandbox.ts @@ -8,6 +8,7 @@ import { readChild } from "./cliChildOutput.js"; import { cliChildEnvironment } from "./cliEnvironment.js"; import { terminateChild, trackCliChild } from "./cliProcess.js"; import { renderGrokSandboxArgs } from "./cliEngineSpawn.js"; +import { assertGrokWorkerDenyPathsPlaceable } from "../runtime/grokWorkerDenyPlacement.js"; import { GROK_WORKER_SANDBOX_EVENTS_RELATIVE_PATH, GROK_WORKER_SANDBOX_PROFILE, renderGrokWorkerSandboxProfile } from "../runtime/grokWorkerSandboxProfile.js"; export const GROK_DAIMON_SANDBOX_PROFILE = GROK_WORKER_SANDBOX_PROFILE; @@ -36,6 +37,11 @@ export async function prepareAndVerifyGrokSandbox( if (denied.some((entry) => overlaps(entry, cwd) || overlaps(entry, engineHome))) { throw unavailable(); } + // Grok 1.0.34 materializes every deny target inside bubblewrap as the uid it + // runs Grok under — here, this process's own — and refuses the whole profile + // if one cannot be resolved. Refusing now names the path; letting it through + // would kill every turn with a bare `bwrap: Can't create file at …`. + await assertGrokWorkerDenyPathsPlaceable(denied, { uid: process.getuid?.() ?? 0, gid: process.getgid?.() ?? 0 }); await writeProfile(engineHome, denied); const beforeBytes = await eventFileSize(path.join(engineHome, SANDBOX_EVENTS)); const child = trackCliChild(spawn(authority.command, [ diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 3551638..d9544a7 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -160,6 +160,34 @@ every profile inside bubblewrap, where a non-empty `deny` list is enforced; `grokWorkerSandboxProfile.ts` renders those profile bytes. A worker-uid process can neither write, rename, nor unlink any of the root-owned files. +Deny-path placement (`grokWorkerDenyPlacement.ts`). Grok 1.0.34 materializes +every `deny` entry inside bubblewrap **as the worker uid**, bind-mounting +`$GROK_HOME/sandbox-blocked-{file,dir}` over the target, so an entry is +placeable only when every ancestor directory is searchable by that uid and the +target already exists and is not a symlink. One unplaceable entry makes Grok +refuse the *whole* profile (`bwrap: Can't create file at …: Permission +denied`), so every turn of that worker fails, not just that path. Matrix: +`.runtime/grok-deny-placement/EVIDENCE.md` in the ecosystem folder. The rule +therefore has two halves: +- shape, decidable without a filesystem and asserted by the renderer: canonical, + and strictly below every base-profile grant (`GROK_WORKER_BASE_PROFILE_GRANTS`); +- placement, asserted by whoever provisions the paths — root provisioning and + every slot recycle on the Spawnfile side, `prepareGrokWorkerAttestation` + before every brokered turn, and `prepareAndVerifyGrokSandbox` on the direct + path, which runs as the worker uid itself. The broker (uid 2100) cannot + descend into a `2000: 0710` runtime home, so an `EACCES` below an + ancestor the worker *can* search is left undecided there; root, which holds + `CAP_DAC_READ_SEARCH`, decides every entry. + +When a protected path is not placeable, the deny entry is **lifted** to the +nearest ancestor that is — never adding `o+x` to a private directory, because a +lift masks a superset and never widens the worker's reach. The durable +wake-acceptance store is exactly that case: it lives under the organization's +`state` directory, which the ownership guard secures `2000:2000 0700`, so the +mask goes on that directory (`acceptanceStoreDenyPath` in +`grokBrokerProjection.ts`, which refuses a mask that does not contain the +store). + Temp and spill isolation (`grokWorkerTmpAttestation.ts`, checked before every turn; `GROK_ENGINE_BROKER.worker.home.{privateTmp,sharedTmp,spillDirectory}`). Grok 1.0.34's strict profile grants shared `/tmp` and `/var/tmp` read-write diff --git a/src/runtime/grokBrokerProjection.test.ts b/src/runtime/grokBrokerProjection.test.ts index 10794bb..592ca7a 100644 --- a/src/runtime/grokBrokerProjection.test.ts +++ b/src/runtime/grokBrokerProjection.test.ts @@ -52,3 +52,21 @@ test("a provisioned registration must describe its projection exactly", () => { assert.throws(() => verifyGrokBrokerRegistrationMatchesProjection(parse({ ...registration, ...drift }), projection), /does not match/u, JSON.stringify(drift)); } }); + +test("the acceptance store mask may be a covering directory, and must actually cover the store", () => { + // Grok 1.0.34 cannot materialize a deny target under a directory the worker cannot search, so a + // deployment that secures `/state` to `2000:2000 0700` masks that directory instead. + const store = "/var/lib/spawnfile/instance/state/wake-acceptance"; + const lifted = resolveOrganizationGrokBrokerProjection(config, "foreman", + { ...options, acceptanceStorePath: store, acceptanceStoreDenyPath: "/var/lib/spawnfile/instance/state" }); + assert.equal(lifted.denyPaths.includes("/var/lib/spawnfile/instance/state"), true); + assert.equal(lifted.denyPaths.includes(store), false); + // Default: the store itself, exactly as before. + assert.equal(resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, acceptanceStorePath: store }).denyPaths.includes(store), true); + for (const acceptanceStoreDenyPath of ["/var/lib/spawnfile/instance/other", "/var/lib/spawnfile/instance/state/wake-acceptance/inner"]) { + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, acceptanceStorePath: store, acceptanceStoreDenyPath }), + /acceptance store deny path must contain the acceptance store/u, acceptanceStoreDenyPath); + } + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, acceptanceStorePath: store, acceptanceStoreDenyPath: "/var" }), + /canonical absolute acceptanceStoreDenyPath|base profile grant/u); +}); diff --git a/src/runtime/grokBrokerProjection.ts b/src/runtime/grokBrokerProjection.ts index 4052e41..34414d4 100644 --- a/src/runtime/grokBrokerProjection.ts +++ b/src/runtime/grokBrokerProjection.ts @@ -55,6 +55,19 @@ export type OrganizationGrokBrokerProjectionOptions = Readonly<{ limits: EngineBrokerTurnLimits; /** The wake-acceptance store, always denied like the Codex projection's. */ acceptanceStorePath: string; + /** + * The deny entry that covers the acceptance store, when the store itself + * cannot be one. + * + * Grok 1.0.34 materializes every deny target inside bubblewrap as the worker + * uid, so a target whose parent directory the worker cannot search is + * unplaceable and makes Grok refuse the whole profile. The durable store sits + * under the organization's private `state` directory (`2000:2000 0700`), so a + * deployment that secures it that way declares the mask on that directory + * instead — strictly stronger, since nothing else lives there. Must contain + * the store; defaults to the store itself. + */ + acceptanceStoreDenyPath?: string; /** Evaluator and host-bind paths the deployment must keep from the worker (R4). */ denyPaths?: readonly string[]; /** sha256 of the seccomp profile bytes the deployment runs the worker under. */ @@ -79,21 +92,26 @@ export type OrganizationGrokBrokerProjectionOptions = Readonly<{ * * The agent must be a Grok agent that *declares* its model and reasoning * effort; nothing is defaulted. The deny list is Daimon's own protected set for - * this agent (realm, bootstrap, peers, acceptance store) plus the caller's - * evaluator paths, sorted and deduplicated exactly as the profile renderer - * does. A supplied `profileSha256` that differs is refused. + * this agent (realm, bootstrap, peers, and the mask covering the acceptance + * store) plus the caller's evaluator paths, sorted and deduplicated exactly as + * the profile renderer does. A supplied `profileSha256` that differs is + * refused. */ export function resolveOrganizationGrokBrokerProjection(config: unknown, agentId: string, options: OrganizationGrokBrokerProjectionOptions): OrganizationGrokBrokerProjection { const parsed = parseOrganizationRuntimeConfig(config); const agent = parsed.agents.find((entry) => entry.id === agentId); if (agent === undefined || agent.engine.kind !== "grok") throw new Error("Grok broker projection requires a known Grok agent"); if (agent.engine.model === undefined || agent.engine.reasoningEffort === undefined) throw new Error("Grok broker projection requires a declared model and reasoning effort"); - for (const [label, value] of [["workerHomePath", options.workerHomePath], ["acceptanceStorePath", options.acceptanceStorePath]] as const) { + const acceptanceStoreDenyPath = options.acceptanceStoreDenyPath ?? options.acceptanceStorePath; + for (const [label, value] of [["workerHomePath", options.workerHomePath], ["acceptanceStorePath", options.acceptanceStorePath], ["acceptanceStoreDenyPath", acceptanceStoreDenyPath]] as const) { if (!path.posix.isAbsolute(value) || path.posix.normalize(value) !== value || value === "/" || value.endsWith("/")) throw new Error(`Grok broker projection requires a canonical absolute ${label}`); } + if (options.acceptanceStorePath !== acceptanceStoreDenyPath && !options.acceptanceStorePath.startsWith(`${acceptanceStoreDenyPath}/`)) { + throw new Error("Grok broker projection acceptance store deny path must contain the acceptance store"); + } if (!/^[a-f0-9]{64}$/u.test(options.seccompProfileSha256)) throw new Error("Grok broker projection requires a seccomp profile sha256"); const model = agent.engine.model as GrokBrokerModel, reasoningEffort = agent.engine.reasoningEffort as GrokBrokerReasoningEffort; - const denyPaths = [...new Set([...grokSandboxProtectedPaths(agent.id, parsed.agents, [options.acceptanceStorePath]), ...(options.denyPaths ?? [])])].sort(); + const denyPaths = [...new Set([...grokSandboxProtectedPaths(agent.id, parsed.agents, [acceptanceStoreDenyPath]), ...(options.denyPaths ?? [])])].sort(); renderGrokWorkerSandboxProfile(denyPaths); const profileSha256 = grokWorkerSandboxProfileSha256(denyPaths); if (options.profileSha256 !== undefined && options.profileSha256 !== profileSha256) throw new Error("Grok broker projection profile digest mismatch"); diff --git a/src/runtime/grokWorkerAttestation.ts b/src/runtime/grokWorkerAttestation.ts index 60bc8fc..c441411 100644 --- a/src/runtime/grokWorkerAttestation.ts +++ b/src/runtime/grokWorkerAttestation.ts @@ -3,6 +3,7 @@ import { constants } from "node:fs"; import { lstat,open } from "node:fs/promises"; import path from "node:path"; +import { assertGrokWorkerDenyPathsPlaceable } from "./grokWorkerDenyPlacement.js"; import { verifyGrokWorkerHome } from "./grokWorkerHomeAttestation.js"; import { grokWorkerHomeForProfile, verifyGrokWorkerTmp, type GrokWorkerTmpOptions, type GrokWorkerTmpWorker } from "./grokWorkerTmpAttestation.js"; import { grokWorkerEventsPathFor } from "./grokWorkerSandboxProfile.js"; @@ -61,12 +62,23 @@ export function parseGrokWorkerSandboxProfile(bytes:Uint8Array,profileSha256:str * `$GROK_HOME/sessions/` (the root `sandbox-events.jsonl` stays empty on * 1.0.34), and a root-owned read-only worker home whose `config.toml` hashes to * the declared renderer output (`grokWorkerHomeAttestation.ts`). + * + * It also refuses a deny list bubblewrap could not materialize + * (`grokWorkerDenyPlacement.ts`), naming the entry and the ancestor that stops + * it. Without this the worker dies with a bare `bwrap: Can't create file at + * …: Permission denied` on *every* turn, because one unplaceable entry makes + * Grok refuse the whole profile. The broker cannot descend into a directory + * opened to the worker's group alone (`/tool-state` under a + * `2000: 0710` home), so an `EACCES` there is left undecided; root + * provisioning, which holds `CAP_DAC_READ_SEARCH`, is the authority that + * decides every entry. */ -export async function prepareGrokWorkerAttestation(input:Readonly<{profilePath:string;eventsPath:string;profileSha256:string;workerUid:number;brokerGid:number;configSha256:string;registeredWorkers?:readonly Readonly<{profilePath:string;workerUid:number}>[]}>,profileOwner:Readonly<{uid:number;gid:number;tmp?:GrokWorkerTmpOptions}>={uid:0,gid:0}):Promise{ +export async function prepareGrokWorkerAttestation(input:Readonly<{profilePath:string;eventsPath:string;profileSha256:string;workerUid:number;workerGid?:number;brokerGid:number;configSha256:string;registeredWorkers?:readonly Readonly<{profilePath:string;workerUid:number}>[]}>,profileOwner:Readonly<{uid:number;gid:number;tmp?:GrokWorkerTmpOptions}>={uid:0,gid:0}):Promise{ if(input.eventsPath!==grokWorkerEventsPathFor(input.profilePath))throw new Error("Grok worker sandbox events must be read from $GROK_HOME/sessions/sandbox-events.jsonl"); const profile=await secureOpen(input.profilePath,profileOwner.uid,profileOwner.gid,0o444,65_536);let bytes:Buffer|undefined;let denyPaths:readonly string[]=[];try{bytes=await profile.readFile();denyPaths=parseGrokWorkerSandboxProfile(bytes,input.profileSha256);}catch{throw new Error("Grok worker isolation attestation unavailable");}finally{bytes?.fill(0);await profile.close();} // The launcher exports TMPDIR=/tmp; the profile lives at /.grok/sandbox.toml. // Every registered worker's private temp is attested, not only this one's (a sibling's open temp is a shared channel). + await assertGrokWorkerDenyPathsPlaceable(denyPaths,{uid:input.workerUid,gid:input.workerGid??input.workerUid}); const workers=[{profilePath:input.profilePath,workerUid:input.workerUid},...(input.registeredWorkers??[])].map((worker)=>({home:grokWorkerHomeForProfile(worker.profilePath),uid:worker.workerUid})); if(workers.some((worker)=>worker.home===undefined))throw new Error("Grok worker isolation attestation unavailable"); await verifyGrokWorkerTmp(workers as readonly GrokWorkerTmpWorker[],profileOwner.tmp); diff --git a/src/runtime/grokWorkerAttestationChecks.test.ts b/src/runtime/grokWorkerAttestationChecks.test.ts index 8bee20e..45480ed 100644 --- a/src/runtime/grokWorkerAttestationChecks.test.ts +++ b/src/runtime/grokWorkerAttestationChecks.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; -import { appendFile, chmod, link, mkdir, mkdtemp, open, rm, stat, writeFile } from "node:fs/promises"; +import { appendFile, chmod, link, mkdir, mkdtemp, open, realpath, rm, stat, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import test, { mock } from "node:test"; @@ -122,3 +122,35 @@ test("prepare refuses the current turn when a sibling registered worker's privat await chmod(path.join(sibling.home, "tmp"), 0o777); await assert.rejects(prepareGrokWorkerAttestation({ ...input, brokerGid: self.gid }, seams), /temp isolation attestation unavailable/u); }); + +test("prepare refuses a deny entry bubblewrap could not materialize, before any other leg", async (t) => { + // The production defect: the wake-acceptance store sits under a `0700` organization state directory, + // so bubblewrap — which materializes every deny target as the worker uid — could not create it and + // Grok refused the whole profile, failing every turn with `bwrap: Can't create file at …`. + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "daimon-guard-deny-"))); + t.after(async () => { await chmod(path.join(root, "state"), 0o700); await rm(root, { recursive: true, force: true }); }); + const grokHome = path.join(root, ".grok"); + await mkdir(path.join(grokHome, "sessions"), { recursive: true }); + await mkdir(path.join(root, "tmp"), { mode: 0o700 }); + await mkdir(path.join(root, "state", "wake-acceptance"), { recursive: true }); + const profile = path.join(grokHome, "sandbox.toml"); + const events = path.join(grokHome, "sessions", "sandbox-events.jsonl"); + await writeFile(events, ""); await chmod(events, 0o640); + const owner = { uid: self.uid, gid: Number((await stat(path.join(root, "tmp"))).gid) }; + const withDeny = async (denied: string) => { + const text = `[profiles.daimon-strict]\nextends = "strict"\nrestrict_network = true\ndeny = ["${denied}"]\n`; + await chmod(profile, 0o644).catch(() => undefined); + await writeFile(profile, text); await chmod(profile, 0o444); + return { profilePath: profile, eventsPath: events, profileSha256: createHash("sha256").update(text).digest("hex"), workerUid: self.uid, brokerGid: self.gid, configSha256: "0".repeat(64) }; + }; + await chmod(path.join(root, "state"), 0o600); + await assert.rejects( + prepareGrokWorkerAttestation(await withDeny(path.join(root, "state", "wake-acceptance")), owner), + (error: Error) => /is not placeable/u.test(error.message) && error.message.includes(`cannot search ${path.join(root, "state")}`) + ); + // The lift: the private directory itself is placeable, so this leg passes and a later one refuses. + await assert.rejects( + prepareGrokWorkerAttestation(await withDeny(path.join(root, "state")), owner), + (error: Error) => !/is not placeable/u.test(error.message) + ); +}); diff --git a/src/runtime/grokWorkerDenyPlacement.test.ts b/src/runtime/grokWorkerDenyPlacement.test.ts new file mode 100644 index 0000000..f5013a0 --- /dev/null +++ b/src/runtime/grokWorkerDenyPlacement.test.ts @@ -0,0 +1,118 @@ +import assert from "node:assert/strict"; +import { chmodSync, mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { + assertGrokWorkerDenyPathPlacement, + assertGrokWorkerDenyPathShape, + assertGrokWorkerDenyPathsPlaceable, + GROK_WORKER_BASE_PROFILE_GRANTS, + grokWorkerCanSearch, + grokWorkerDenyPathChain, + GrokWorkerDenyPlacementError, + type GrokWorkerDenyPathEntry, + type GrokWorkerDenyPathStep +} from "./grokWorkerDenyPlacement.js"; +import { renderGrokWorkerSandboxProfile } from "./grokWorkerSandboxProfile.js"; + +const entry = (uid: number, gid: number, mode: number, kind: "dir" | "file" | "link" = "dir"): GrokWorkerDenyPathEntry => + ({ uid, gid, mode, isDirectory: () => kind === "dir", isSymbolicLink: () => kind === "link" }); +const worker = { uid: 2200, gid: 2200 }; +const steps = (denyPath: string, entries: readonly (GrokWorkerDenyPathEntry | string)[]): GrokWorkerDenyPathStep[] => + grokWorkerDenyPathChain(denyPath).map((target, index) => { + const value = entries[index]; + return typeof value === "string" ? { path: target, code: value } : { path: target, entry: value }; + }); + +test("search permission follows owner, then group, then other — as the worker's cleared-group process does", () => { + assert.equal(grokWorkerCanSearch(entry(2200, 2200, 0o700), worker), true); + assert.equal(grokWorkerCanSearch(entry(2200, 2200, 0o677), worker), false, "owner bits win even when group and other would allow"); + assert.equal(grokWorkerCanSearch(entry(2000, 2200, 0o710), worker), true); + assert.equal(grokWorkerCanSearch(entry(2000, 2000, 0o700), worker), false); + assert.equal(grokWorkerCanSearch(entry(0, 0, 0o711), worker), true); + assert.equal(grokWorkerCanSearch(entry(0, 0, 0o755), worker), true); + assert.equal(grokWorkerCanSearch(entry(0, 2000, 0o750), worker), false); +}); + +test("refuses a deny entry at or above any Grok 1.0.34 base-profile grant", () => { + for (const grant of GROK_WORKER_BASE_PROFILE_GRANTS) { + assert.throws(() => assertGrokWorkerDenyPathShape(grant), GrokWorkerDenyPlacementError, grant); + } + assert.throws(() => assertGrokWorkerDenyPathShape("/var"), /base profile grant \/var/u); + assert.throws(() => renderGrokWorkerSandboxProfile(["/run"]), /base profile grant \/run/u); + assert.throws(() => renderGrokWorkerSandboxProfile(["/var/lib/spawnfile/daimon/usage", "/tmp"]), /base profile grant \/tmp/u); + // Strictly below every grant is exactly what Grok accepts. + assertGrokWorkerDenyPathShape("/tmp/sub"); + assertGrokWorkerDenyPathShape("/var/lib/spawnfile/daimon/usage"); +}); + +test("refuses the wake-acceptance shape: a deny entry under a parent the worker cannot search", () => { + // `/state` is `2000:2000 0700`; the store beneath it is what production used to deny. + const denyPath = "/var/lib/spawnfile/instances/daimon/org/state/wake-acceptance"; + assert.throws( + () => assertGrokWorkerDenyPathPlacement(denyPath, steps(denyPath, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o711), entry(0, 0, 0o711), + entry(0, 0, 0o711), entry(2000, 2000, 0o711), entry(2000, 2000, 0o700), entry(2000, 2000, 0o700) + ]), worker), + (error: Error) => error instanceof GrokWorkerDenyPlacementError + && /cannot search \/var\/lib\/spawnfile\/instances\/daimon\/org\/state \(700 2000:2000\); deny that directory itself instead/u.test(error.message) + ); + // The lift production now emits: the private directory itself, whose own parent is traversable. + const lifted = "/var/lib/spawnfile/instances/daimon/org/state"; + assertGrokWorkerDenyPathPlacement(lifted, steps(lifted, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o711), entry(0, 0, 0o711), + entry(0, 0, 0o711), entry(2000, 2000, 0o711), entry(2000, 2000, 0o700) + ]), worker); +}); + +test("refuses a missing target, a symlink and a non-directory ancestor; leaves an undecidable EACCES alone", () => { + const denyPath = "/run/training/slot/state"; + assert.throws(() => assertGrokWorkerDenyPathPlacement(denyPath, steps(denyPath, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), "ENOENT" + ]), worker), /does not exist; bubblewrap would have to create it as the worker uid/u); + assert.throws(() => assertGrokWorkerDenyPathPlacement(denyPath, steps(denyPath, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o777, "link") + ]), worker), /is a symlink; bubblewrap refuses to bind over one/u); + assert.throws(() => assertGrokWorkerDenyPathPlacement(denyPath, steps(denyPath, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o644, "file"), entry(0, 0, 0o755), entry(0, 0, 0o700) + ]), worker), /is not a directory/u); + assert.throws(() => assertGrokWorkerDenyPathPlacement(denyPath, steps(denyPath, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), "EPERM" + ]), worker), /could not be read \(EPERM\)/u); + // The broker (uid 2100) cannot descend into a `2000: 0710` runtime home the worker itself can + // search, so an EACCES *below a worker-searchable ancestor* is undecided here, never a refusal. + assertGrokWorkerDenyPathPlacement(denyPath, steps(denyPath, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(2000, 2200, 0o710), "EACCES" + ]), worker); +}); + +test("walks the real filesystem and names the ancestor that stops a deny entry", async () => { + // realpath: macOS `/var` is a symlink, and a symlinked ancestor is refused on purpose — Daimon's + // registered paths are canonical, and bubblewrap must bind over the inode the deny entry names. + const root = realpathSync(mkdtempSync(path.join(tmpdir(), "grok-deny-"))); + const state = path.join(root, "state"); + // This process stands in for root provisioning: it can stat every component, and judges searchability + // for the worker from the modes it reads. Here the worker is this uid, so only `state` blocks it. + const self = { uid: process.getuid?.() ?? 0, gid: process.getgid?.() ?? 0 }; + try { + mkdirSync(path.join(state, "wake-acceptance"), { recursive: true }); + writeFileSync(path.join(state, "wake-acceptance", "store.jsonl"), "{}\n"); + chmodSync(state, 0o600); + await assert.rejects( + assertGrokWorkerDenyPathsPlaceable([path.join(state, "wake-acceptance")], self), + (error: Error) => error instanceof GrokWorkerDenyPlacementError + && error.message.includes(`cannot search ${state}`) && error.message.includes("deny that directory itself instead") + ); + // The lift: the unsearchable directory itself is placeable, and masks strictly more. + await assertGrokWorkerDenyPathsPlaceable([state], self); + await assert.rejects(assertGrokWorkerDenyPathsPlaceable([path.join(root, "absent")], self), /does not exist/u); + symlinkSync(state, path.join(root, "link")); + await assert.rejects(assertGrokWorkerDenyPathsPlaceable([path.join(root, "link")], self), /is a symlink/u); + await assert.rejects(assertGrokWorkerDenyPathsPlaceable(["relative/path"], self), /not an absolute path/u); + } finally { + chmodSync(state, 0o700); + rmSync(root, { force: true, recursive: true }); + } +}); diff --git a/src/runtime/grokWorkerDenyPlacement.ts b/src/runtime/grokWorkerDenyPlacement.ts new file mode 100644 index 0000000..f2995d2 --- /dev/null +++ b/src/runtime/grokWorkerDenyPlacement.ts @@ -0,0 +1,147 @@ +import { lstat } from "node:fs/promises"; +import path from "node:path"; + +/** + * Paths Grok 1.0.34's strict base profile grants read or read-write. A `deny` + * entry that equals or contains one of them makes Grok refuse the profile + * outright (verified for `/tmp`, `/var/tmp`, `/run`, `/etc`, `/var` and + * `sessions`; `/tmp/sub` is accepted), so every entry must sit strictly below + * each grant it touches. + */ +export const GROK_WORKER_BASE_PROFILE_GRANTS = Object.freeze([ + "/bin", "/dev", "/etc", "/lib", "/proc", "/run", "/sbin", "/sys", "/tmp", "/usr", "/var", "/var/tmp" +] as const); + +const within = (candidate: string, root: string): boolean => candidate === root || candidate.startsWith(`${root}/`); + +export class GrokWorkerDenyPlacementError extends Error { + constructor(readonly denyPath: string, readonly reason: string) { + super(`Grok worker sandbox deny path ${JSON.stringify(denyPath)} is not placeable: ${reason}`); + this.name = "GrokWorkerDenyPlacementError"; + } +} + +/** + * The shape half of the deny-placement policy: everything decidable without + * touching a filesystem, so the profile renderer can refuse a bad entry before + * its bytes are ever pinned or written. + * + * `renderGrokWorkerSandboxProfile` already refuses entries that are relative, + * non-canonical, `/`, trailing-slashed, or carrying a character TOML or Grok + * would reinterpret; this adds the base-profile grant rule. + */ +export function assertGrokWorkerDenyPathShape(denyPath: string): void { + const grant = GROK_WORKER_BASE_PROFILE_GRANTS.find((candidate) => within(candidate, denyPath)); + if (grant !== undefined) { + throw new GrokWorkerDenyPlacementError(denyPath, `it equals or contains the base profile grant ${grant}, which Grok refuses`); + } +} + +/** The subset of `lstat` the placement rules read; pure so every refusal is testable without root. */ +export type GrokWorkerDenyPathEntry = Readonly<{ + uid: number; + gid: number; + mode: number; + isDirectory: () => boolean; + isSymbolicLink: () => boolean; +}>; + +/** One resolved path component: the entry, or the errno that stopped the walk. */ +export type GrokWorkerDenyPathStep = Readonly<{ path: string; entry?: GrokWorkerDenyPathEntry; code?: string }>; + +export type GrokWorkerDenyPathWorker = Readonly<{ uid: number; gid: number }>; + +/** + * POSIX search permission: owner bits win, then group, then other. The worker + * runs with its supplementary groups cleared, so its primary gid is the only + * group that can apply. + */ +export const grokWorkerCanSearch = (entry: GrokWorkerDenyPathEntry, worker: GrokWorkerDenyPathWorker): boolean => + entry.uid === worker.uid ? (entry.mode & 0o100) !== 0 + : entry.gid === worker.gid ? (entry.mode & 0o010) !== 0 + : (entry.mode & 0o001) !== 0; + +/** The `/`-rooted ancestor chain of `denyPath`, deepest last, followed by the entry itself. */ +export const grokWorkerDenyPathChain = (denyPath: string): readonly string[] => { + const components = denyPath.split("/").slice(1); + return ["/", ...components.map((_, index) => `/${components.slice(0, index + 1).join("/")}`)]; +}; + +/** + * The placement half of the policy, as a pure function of an already-walked + * chain. + * + * Grok 1.0.34 materializes every `deny` entry inside bubblewrap **as the worker + * uid**, by bind-mounting `$GROK_HOME/sandbox-blocked-{file,dir}` over the + * target. So bwrap must be able to *resolve* the target as that uid: every + * ancestor directory needs the search bit for it, and the target must already + * exist — otherwise bwrap tries to create it and needs write on the parent, + * which a private parent never grants. A single unplaceable entry makes Grok + * refuse the whole profile, so every turn of that worker fails, not just that + * path. Verified matrix: `.runtime/grok-deny-placement/EVIDENCE.md`. + * + * The walk may legitimately stop early: a caller that is neither root nor the + * worker (the broker, uid 2100) cannot descend into a directory the worker's + * own group opens to it alone — `/tool-state` under a + * `2000: 0710` runtime home is exactly that. An `EACCES` below an + * ancestor the *worker* can search is therefore "not decidable from here", not + * a refusal; every decidable failure still refuses. + */ +export function assertGrokWorkerDenyPathPlacement( + denyPath: string, + steps: readonly GrokWorkerDenyPathStep[], + worker: GrokWorkerDenyPathWorker +): void { + assertGrokWorkerDenyPathShape(denyPath); + const chain = grokWorkerDenyPathChain(denyPath); + if (steps.length !== chain.length || steps.some((step, index) => step.path !== chain[index])) { + throw new GrokWorkerDenyPlacementError(denyPath, "its resolved path chain does not match the entry"); + } + for (const [index, step] of steps.entries()) { + const ancestor = index < steps.length - 1; + if (step.entry === undefined) { + if (step.code === "ENOENT") throw new GrokWorkerDenyPlacementError(denyPath, `${step.path} does not exist; bubblewrap would have to create it as the worker uid`); + if (step.code !== "EACCES") throw new GrokWorkerDenyPlacementError(denyPath, `${step.path} could not be read (${step.code ?? "unknown error"})`); + // Undecidable from here, and only after every shallower ancestor passed. + return; + } + if (step.entry.isSymbolicLink()) throw new GrokWorkerDenyPlacementError(denyPath, `${step.path} is a symlink; bubblewrap refuses to bind over one`); + if (!ancestor) return; + if (!step.entry.isDirectory()) throw new GrokWorkerDenyPlacementError(denyPath, `${step.path} is not a directory`); + if (!grokWorkerCanSearch(step.entry, worker)) { + throw new GrokWorkerDenyPlacementError(denyPath, `worker uid ${worker.uid} cannot search ${step.path} (${(step.entry.mode & 0o7777).toString(8)} ${step.entry.uid}:${step.entry.gid}); deny that directory itself instead`); + } + } +} + +/** Walks one deny path on the real filesystem, recording what stopped it rather than throwing. */ +export async function readGrokWorkerDenyPathChain(denyPath: string): Promise { + const steps: GrokWorkerDenyPathStep[] = []; + for (const target of grokWorkerDenyPathChain(denyPath)) { + try { + steps.push({ path: target, entry: await lstat(target) }); + } catch (error) { + steps.push({ path: target, code: (error as NodeJS.ErrnoException).code }); + break; + } + } + const chain = grokWorkerDenyPathChain(denyPath); + while (steps.length < chain.length) steps.push({ path: chain[steps.length]!, code: steps.at(-1)?.code ?? "EACCES" }); + return steps; +} + +/** + * Fails closed before a worker is ever launched with a profile Grok would + * refuse. Callers with the widest view run it: root provisioning at container + * start and on every slot recycle, and the direct (non-broker) path, which runs + * as the worker uid itself. + */ +export async function assertGrokWorkerDenyPathsPlaceable( + denyPaths: readonly string[], + worker: GrokWorkerDenyPathWorker +): Promise { + for (const denyPath of denyPaths) { + if (!path.posix.isAbsolute(denyPath)) throw new GrokWorkerDenyPlacementError(denyPath, "it is not an absolute path"); + assertGrokWorkerDenyPathPlacement(denyPath, await readGrokWorkerDenyPathChain(denyPath), worker); + } +} diff --git a/src/runtime/grokWorkerSandboxProfile.ts b/src/runtime/grokWorkerSandboxProfile.ts index b6ff126..881ffb1 100644 --- a/src/runtime/grokWorkerSandboxProfile.ts +++ b/src/runtime/grokWorkerSandboxProfile.ts @@ -1,6 +1,8 @@ import { createHash } from "node:crypto"; import path from "node:path"; +import { assertGrokWorkerDenyPathShape } from "./grokWorkerDenyPlacement.js"; + export const GROK_WORKER_SANDBOX_PROFILE = "daimon-strict" as const; export const GROK_WORKER_SANDBOX_EVENTS_RELATIVE_PATH = "sessions/sandbox-events.jsonl" as const; @@ -17,7 +19,12 @@ export const GROK_WORKER_SANDBOX_EVENTS_RELATIVE_PATH = "sessions/sandbox-events * * Entries are sorted and deduplicated so equal sets render equal bytes (and the * same `profileSha256`). A path that is not absolute and canonical, or that - * carries a character TOML or Grok would reinterpret, is refused. + * carries a character TOML or Grok would reinterpret, is refused — and so is + * one that equals or contains a base-profile grant, the first half of the + * deny-placement policy (`grokWorkerDenyPlacement.ts`). The other half — + * the entry exists and every ancestor is searchable by the worker uid — needs + * a filesystem, so it is asserted by whoever provisions the paths and, on the + * direct path, before every turn. */ export function renderGrokWorkerSandboxProfile(denyPaths: readonly string[] = []): string { const denied = [...new Set(denyPaths)].sort(); @@ -25,6 +32,7 @@ export function renderGrokWorkerSandboxProfile(denyPaths: readonly string[] = [] if (!path.posix.isAbsolute(entry) || path.posix.normalize(entry) !== entry || entry === "/" || entry.endsWith("/") || /["\\\u0000-\u001f\u007f*?[\]]/u.test(entry)) { throw new TypeError("invalid Grok worker sandbox deny path"); } + assertGrokWorkerDenyPathShape(entry); } return [ `[profiles.${GROK_WORKER_SANDBOX_PROFILE}]`, diff --git a/src/runtime/index.ts b/src/runtime/index.ts index 24195c5..b3f2ed1 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -14,6 +14,8 @@ export { GROK_INFERENCE_CLIENT_MODEL_ID, GROK_INFERENCE_GRANT_ENV, grokInference export { GROK_INFERENCE_PROXY_BASE_URL, ENGINE_BROKER_INFERENCE_FAILURE_CODES, type EngineBrokerInferenceFailureCode } from "./engineBrokerInferenceProtocol.js"; export { dedupeInferenceUsageRows, INFERENCE_USAGE_LEDGER_VERSION, GROK_INFERENCE_PURPOSES, type GrokInferencePurpose } from "./inferenceUsageLedger.js"; export { GROK_INFERENCE_AUTH_STALE_BODY } from "./grokInferenceProxy.js"; +export { assertGrokWorkerDenyPathPlacement, assertGrokWorkerDenyPathShape, assertGrokWorkerDenyPathsPlaceable, GROK_WORKER_BASE_PROFILE_GRANTS, grokWorkerCanSearch, grokWorkerDenyPathChain, GrokWorkerDenyPlacementError, readGrokWorkerDenyPathChain } from "./grokWorkerDenyPlacement.js"; +export type { GrokWorkerDenyPathEntry, GrokWorkerDenyPathStep, GrokWorkerDenyPathWorker } from "./grokWorkerDenyPlacement.js"; export { grokWorkerSandboxProfileSha256, renderGrokWorkerSandboxProfile } from "./grokWorkerSandboxProfile.js"; export { engineBrokerRequestLedgerPathFor, parseEngineBrokerServiceConfig, type EngineBrokerServiceConfig, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; export { DEFAULT_GROK_BROKER_TURN_LIMITS, ENGINE_BROKER_LIMIT_REASONS, type EngineBrokerLimitReason, type EngineBrokerTurnAccounting, From 8c7e309faf836837118b6d2619480e0676cde9ff Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 16:02:47 +0200 Subject: [PATCH 070/124] fix: let the Grok broker projection mask the wake-acceptance store through a covering directory --- src/runtime/grokBrokerProjection.test.ts | 25 +++++++++--------- src/runtime/grokBrokerProjection.ts | 33 +++++++++++------------- 2 files changed, 27 insertions(+), 31 deletions(-) diff --git a/src/runtime/grokBrokerProjection.test.ts b/src/runtime/grokBrokerProjection.test.ts index 592ca7a..960e917 100644 --- a/src/runtime/grokBrokerProjection.test.ts +++ b/src/runtime/grokBrokerProjection.test.ts @@ -53,20 +53,19 @@ test("a provisioned registration must describe its projection exactly", () => { } }); -test("the acceptance store mask may be a covering directory, and must actually cover the store", () => { +test("the acceptance store mask may be the directory that covers the store", () => { // Grok 1.0.34 cannot materialize a deny target under a directory the worker cannot search, so a - // deployment that secures `/state` to `2000:2000 0700` masks that directory instead. + // deployment that secures `/state` to `2000:2000 0700` declares that directory here. const store = "/var/lib/spawnfile/instance/state/wake-acceptance"; - const lifted = resolveOrganizationGrokBrokerProjection(config, "foreman", - { ...options, acceptanceStorePath: store, acceptanceStoreDenyPath: "/var/lib/spawnfile/instance/state" }); - assert.equal(lifted.denyPaths.includes("/var/lib/spawnfile/instance/state"), true); + const state = "/var/lib/spawnfile/instance/state"; + const lifted = resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, acceptanceStorePath: state }); + assert.equal(lifted.denyPaths.includes(state), true); assert.equal(lifted.denyPaths.includes(store), false); - // Default: the store itself, exactly as before. - assert.equal(resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, acceptanceStorePath: store }).denyPaths.includes(store), true); - for (const acceptanceStoreDenyPath of ["/var/lib/spawnfile/instance/other", "/var/lib/spawnfile/instance/state/wake-acceptance/inner"]) { - assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, acceptanceStorePath: store, acceptanceStoreDenyPath }), - /acceptance store deny path must contain the acceptance store/u, acceptanceStoreDenyPath); - } - assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, acceptanceStorePath: store, acceptanceStoreDenyPath: "/var" }), - /canonical absolute acceptanceStoreDenyPath|base profile grant/u); + // The store itself still works where its parent is traversable, and produces a different digest — + // a recomputation that disagrees with the deployment fails closed rather than certifying the slot. + const leaf = resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, acceptanceStorePath: store }); + assert.equal(leaf.denyPaths.includes(store), true); + assert.notEqual(grokBrokerProjectionSha256(leaf), grokBrokerProjectionSha256(lifted)); + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, acceptanceStorePath: "/var" }), + /base profile grant/u); }); diff --git a/src/runtime/grokBrokerProjection.ts b/src/runtime/grokBrokerProjection.ts index 34414d4..298b433 100644 --- a/src/runtime/grokBrokerProjection.ts +++ b/src/runtime/grokBrokerProjection.ts @@ -53,21 +53,22 @@ export type OrganizationGrokBrokerProjectionOptions = Readonly<{ architecture: "arm64" | "x64"; usageLedgerPath: string; limits: EngineBrokerTurnLimits; - /** The wake-acceptance store, always denied like the Codex projection's. */ - acceptanceStorePath: string; /** - * The deny entry that covers the acceptance store, when the store itself - * cannot be one. + * The deny entry that protects the durable wake-acceptance store: the store + * itself, or a directory containing it. * - * Grok 1.0.34 materializes every deny target inside bubblewrap as the worker - * uid, so a target whose parent directory the worker cannot search is - * unplaceable and makes Grok refuse the whole profile. The durable store sits - * under the organization's private `state` directory (`2000:2000 0700`), so a - * deployment that secures it that way declares the mask on that directory - * instead — strictly stronger, since nothing else lives there. Must contain - * the store; defaults to the store itself. + * Grok 1.0.34 materializes every deny target inside bubblewrap **as the + * worker uid**, so a target whose parent directory the worker cannot search + * is unplaceable and makes Grok refuse the whole profile — every turn of that + * worker then fails, not just that path. Deployments that keep the store + * under a private `state` directory (`2000:2000 0700`) therefore declare that + * directory here: it covers the store, nothing else lives there, and lifting + * the mask adds the worker no reach, where opening the parent with `o+x` + * would. The caller is the one party that knows both the layout and the + * modes; whoever recomputes this projection must pass the same value or the + * digests will not match, which is the intended fail-closed outcome. */ - acceptanceStoreDenyPath?: string; + acceptanceStorePath: string; /** Evaluator and host-bind paths the deployment must keep from the worker (R4). */ denyPaths?: readonly string[]; /** sha256 of the seccomp profile bytes the deployment runs the worker under. */ @@ -102,16 +103,12 @@ export function resolveOrganizationGrokBrokerProjection(config: unknown, agentId const agent = parsed.agents.find((entry) => entry.id === agentId); if (agent === undefined || agent.engine.kind !== "grok") throw new Error("Grok broker projection requires a known Grok agent"); if (agent.engine.model === undefined || agent.engine.reasoningEffort === undefined) throw new Error("Grok broker projection requires a declared model and reasoning effort"); - const acceptanceStoreDenyPath = options.acceptanceStoreDenyPath ?? options.acceptanceStorePath; - for (const [label, value] of [["workerHomePath", options.workerHomePath], ["acceptanceStorePath", options.acceptanceStorePath], ["acceptanceStoreDenyPath", acceptanceStoreDenyPath]] as const) { + for (const [label, value] of [["workerHomePath", options.workerHomePath], ["acceptanceStorePath", options.acceptanceStorePath]] as const) { if (!path.posix.isAbsolute(value) || path.posix.normalize(value) !== value || value === "/" || value.endsWith("/")) throw new Error(`Grok broker projection requires a canonical absolute ${label}`); } - if (options.acceptanceStorePath !== acceptanceStoreDenyPath && !options.acceptanceStorePath.startsWith(`${acceptanceStoreDenyPath}/`)) { - throw new Error("Grok broker projection acceptance store deny path must contain the acceptance store"); - } if (!/^[a-f0-9]{64}$/u.test(options.seccompProfileSha256)) throw new Error("Grok broker projection requires a seccomp profile sha256"); const model = agent.engine.model as GrokBrokerModel, reasoningEffort = agent.engine.reasoningEffort as GrokBrokerReasoningEffort; - const denyPaths = [...new Set([...grokSandboxProtectedPaths(agent.id, parsed.agents, [acceptanceStoreDenyPath]), ...(options.denyPaths ?? [])])].sort(); + const denyPaths = [...new Set([...grokSandboxProtectedPaths(agent.id, parsed.agents, [options.acceptanceStorePath]), ...(options.denyPaths ?? [])])].sort(); renderGrokWorkerSandboxProfile(denyPaths); const profileSha256 = grokWorkerSandboxProfileSha256(denyPaths); if (options.profileSha256 !== undefined && options.profileSha256 !== profileSha256) throw new Error("Grok broker projection profile digest mismatch"); From 089c3f86b99cbe17a2025e9d6be5b77ea71cfadb Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 20:06:44 +0200 Subject: [PATCH 071/124] fix: accept the contracted traverse-only runtime home for brokered Grok agents --- src/contracts/runtimeContractManifest.ts | 4 ++ src/runtime/physicalReadiness.test.ts | 67 +++++++++++++++++++++++- src/runtime/physicalReadiness.ts | 60 +++++++++++++++++---- 3 files changed, 119 insertions(+), 12 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 46a3588..95a6673 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -71,6 +71,10 @@ export const GROK_ENGINE_BROKER = { // denied, so the deployment keeps them from every worker by mode: root-owned, a non-worker // group (< 2200), others read-only (Grok needs to open the directory) and no search/write. sharedTmp: { paths: ["/tmp", "/var/tmp"], uid: 0, maxGroupExclusive: 2_200, otherMode: 0o4, mode: 0o1774 }, + // The organization runtime home of a brokered Grok agent: traverse-only for the + // worker group so the worker can reach `tool-output/` and nothing else (no group + // read, no group write, no world bits; `physicalReadiness.ts` refuses anything else). + organizationRuntimeHome: { owner: "organization", group: "worker", mode: 0o710 }, // Spilled tool output the worker reads with read_file: setgid directory in the worker's group, // files written 0640 by the runtime, never other-readable. spillDirectory: { relativeToRuntimeHome: "tool-output", owner: "organization", group: "worker", mode: 0o2750, fileMode: 0o640 } diff --git a/src/runtime/physicalReadiness.test.ts b/src/runtime/physicalReadiness.test.ts index bdbb248..b645cce 100644 --- a/src/runtime/physicalReadiness.test.ts +++ b/src/runtime/physicalReadiness.test.ts @@ -4,7 +4,7 @@ import os from "node:os"; import path from "node:path"; import test from "node:test"; -import { prepareOrganizationRuntimePaths } from "./physicalReadiness.js"; +import { assertRuntimeDirectory, prepareOrganizationRuntimePaths } from "./physicalReadiness.js"; import type { OrganizationRuntimeAgentConfig } from "./organizationRuntime.js"; const agent = (workspacePath: string, runtimeHomePath: string): OrganizationRuntimeAgentConfig => ({ @@ -46,3 +46,68 @@ test("preflight requires safe workspace and private runtime roots, and proves ph await assert.rejects(prepareOrganizationRuntimePaths([agent(workspace, workspace)]), /overlap/); } finally { await rm(root, { recursive: true, force: true }); } }); + +const withRoots = async (body: (root: string) => Promise): Promise => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-physical-")); + try { await body(root); } finally { await rm(root, { force: true, recursive: true }); } +}; + +const runtime = { uid: 2000, gid: 2000, firstWorkerUid: 2200 }; +const entry = (mode: number, uid = 2000, gid = 2000, kind: "dir" | "link" = "dir") => ({ + uid, gid, mode: (kind === "dir" ? 0o040000 : 0o120000) | mode, + isDirectory: () => kind === "dir", isSymbolicLink: () => kind === "link" +}); + +test("a brokered Grok runtime home is accepted at exactly 2000: 0710 and nothing wider", () => { + assert.doesNotThrow(() => assertRuntimeDirectory(entry(0o710, 2000, 2200), "runtimeHomePath", "worker-traversable", runtime)); + assert.doesNotThrow(() => assertRuntimeDirectory(entry(0o710, 2000, 2201), "runtimeHomePath", "worker-traversable", runtime)); + const refusals: Record> = { + "0700 (no worker traversal, pre-P1b layout)": entry(0o700, 2000, 2200), + "0711 (world traverse)": entry(0o711, 2000, 2200), + "0712": entry(0o712, 2000, 2200), + "0714": entry(0o714, 2000, 2200), + "0730 (group write)": entry(0o730, 2000, 2200), + "0750 (group read)": entry(0o750, 2000, 2200), + "0770": entry(0o770, 2000, 2200), + "0777": entry(0o777, 2000, 2200), + "2710 (setgid)": entry(0o2710, 2000, 2200), + "owned by a worker": entry(0o710, 2200, 2200), + "owned by root": entry(0o710, 0, 2200), + "group is the runtime's own": entry(0o710, 2000, 2000), + "group below the worker range": entry(0o710, 2000, 2100), + "a symlink": entry(0o710, 2000, 2200, "link") + }; + for (const [label, candidate] of Object.entries(refusals)) { + assert.throws(() => assertRuntimeDirectory(candidate, "runtimeHomePath", "worker-traversable", runtime), /runtimeHomePath/u, label); + } +}); + +test("every other engine's runtime home stays exactly 0700, and a workspace stays group/other-write free", () => { + assert.doesNotThrow(() => assertRuntimeDirectory(entry(0o700), "runtimeHomePath", "private", runtime)); + for (const mode of [0o710, 0o701, 0o750, 0o770, 0o711, 0o755, 0o2700]) { + assert.throws(() => assertRuntimeDirectory(entry(mode, 2000, 2200), "runtimeHomePath", "private", runtime), /must have mode 0700/u, mode.toString(8)); + } + // The brokered Grok workspace contract (2000: 0750) passes the workspace shape. + assert.doesNotThrow(() => assertRuntimeDirectory(entry(0o750, 2000, 2200), "workspacePath", "safe", runtime)); + assert.doesNotThrow(() => assertRuntimeDirectory(entry(0o700), "workspacePath", "safe", runtime)); + for (const mode of [0o770, 0o720, 0o702, 0o777]) { + assert.throws(() => assertRuntimeDirectory(entry(mode, 2000, 2200), "workspacePath", "safe", runtime), /must not grant group or other write/u, mode.toString(8)); + } +}); + +test("the engine kind decides the runtime home shape on a real filesystem", async () => { + await withRoots(async (root) => { + const workspace = path.join(root, "workspace"), home = path.join(root, "home"); + await mkdir(workspace, { mode: 0o700 }); + await mkdir(home, { mode: 0o710 }); + const grok = { ...agent(workspace, home), engine: { kind: "grok" as const, model: "grok-4.6" as const, reasoningEffort: "low" as const } }; + // 0710 reaches the Grok branch: only the worker-group requirement is left to refuse it here. + await assert.rejects(prepareOrganizationRuntimePaths([grok]), /group-owned by the agent's Grok worker group/u); + // The same home refuses a Codex agent for being wider than 0700. + await assert.rejects(prepareOrganizationRuntimePaths([agent(workspace, home)]), /must have mode 0700/u); + await chmod(home, 0o700); + const authority = await prepareOrganizationRuntimePaths([agent(workspace, home)]); + await authority.close(); + await assert.rejects(prepareOrganizationRuntimePaths([grok]), /must have mode 0710 for a brokered Grok agent/u); + }); +}); diff --git a/src/runtime/physicalReadiness.ts b/src/runtime/physicalReadiness.ts index 075ea47..50ce64a 100644 --- a/src/runtime/physicalReadiness.ts +++ b/src/runtime/physicalReadiness.ts @@ -2,9 +2,26 @@ import { constants, type Stats } from "node:fs"; import { lstat, open, realpath } from "node:fs/promises"; import path from "node:path"; +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; import type { OrganizationRuntimeAgentConfig } from "./organizationRuntime.js"; type Identity = Readonly<{ dev: number; ino: number; uid: number; mode: number }>; +/** + * `private` is every engine's runtime home: 0700, nothing but the runtime user. + * + * `worker-traversable` is the brokered Grok shape, and only that shape: the + * agent's own sandboxed worker runs as another uid and must be able to *walk + * into* this home to read the setgid `tool-output/` spill directory the + * truncation notice sends it to (`GROK_ENGINE_BROKER.worker.home.organizationRuntimeHome`). + * Traverse-only means `0710`: no group read (the worker cannot list the home or + * see the acceptance store, telemetry, memory or credential names) and no group + * write. Anything wider — `0711`, `0750`, `0770`, any world bit — is refused, + * as is a group that is not a worker group. Daimon cannot tell *which* worker + * gid belongs to this agent; the per-slot mapping is the deployment's + * provisioning contract, re-checked by the slot preflight receipt's worker-uid + * canaries. + */ +type DirectoryShape = "safe" | "private" | "worker-traversable"; type Directory = { readonly configured: string; readonly real: string; readonly fd: Awaited>; readonly identity: Identity; closed: boolean }; /** @@ -20,6 +37,11 @@ export type OrganizationRuntimePathAuthority = Readonly<{ close(): Promise; }>; +/** Only a brokered Grok agent's home is worker-traversable; every other engine keeps 0700. */ +function runtimeHomeShape(agent: OrganizationRuntimeAgentConfig): DirectoryShape { + return agent.engine.kind === "grok" ? "worker-traversable" : "private"; +} + export async function prepareOrganizationRuntimePaths( agents: readonly OrganizationRuntimeAgentConfig[] ): Promise { @@ -28,7 +50,7 @@ export async function prepareOrganizationRuntimePaths( try { for (const agent of agents) { workspaces.set(agent.id, await verifyDirectory(agent.workspacePath, "workspacePath", "safe")); - homes.set(agent.id, await verifyDirectory(agent.runtimeHomePath, "runtimeHomePath", "private")); + homes.set(agent.id, await verifyDirectory(agent.runtimeHomePath, "runtimeHomePath", runtimeHomeShape(agent))); } const roots = [...workspaces.values(), ...homes.values()]; for (let left = 0; left < roots.length; left += 1) for (let right = left + 1; right < roots.length; right += 1) { @@ -51,7 +73,7 @@ export async function prepareOrganizationRuntimePaths( if (workspace === undefined || home === undefined) throw new Error(`no runtime path authority for ${agent.id}`); await Promise.all([ verifyIdentity(workspace, "workspacePath", "safe"), - verifyIdentity(home, "runtimeHomePath", "private") + verifyIdentity(home, "runtimeHomePath", runtimeHomeShape(agent)) ]); }; return { @@ -70,10 +92,10 @@ export async function prepareOrganizationRuntimePaths( }; } -async function verifyDirectory(configured: string, label: string, mode: "safe" | "private"): Promise { +async function verifyDirectory(configured: string, label: string, shape: DirectoryShape): Promise { await assertNoSymlinkComponents(configured); const before = await lstat(configured); - assertDirectory(before, label, mode); + assertDirectory(before, label, shape); const fd = await open(configured, constants.O_RDONLY | directoryFlag() | noFollow()); try { const opened = await fd.stat(); @@ -89,7 +111,7 @@ async function verifyDirectory(configured: string, label: string, mode: "safe" | } } -async function verifyIdentity(directory: Directory, label: string, mode: "safe" | "private"): Promise { +async function verifyIdentity(directory: Directory, label: string, shape: DirectoryShape): Promise { if (directory.closed) throw new Error(`${label} authority is closed`); await assertNoSymlinkComponents(directory.configured); const entry = await lstat(directory.configured); @@ -97,7 +119,7 @@ async function verifyIdentity(directory: Directory, label: string, mode: "safe" if (!sameIdentity(identity(entry), directory.identity) || !sameIdentity(identity(opened), directory.identity)) { throw new Error(`${label} changed after readiness validation`); } - assertDirectory(entry, label, mode); + assertDirectory(entry, label, shape); if (await realpath(directory.configured) !== directory.real) throw new Error(`${label} changed after readiness validation`); } @@ -112,12 +134,28 @@ async function assertNoSymlinkComponents(target: string): Promise { } } -function assertDirectory(entry: Stats, label: string, mode: "safe" | "private"): void { +/** Identity of the process Daimon runs as; a seam so every refusal is testable unprivileged. */ +export type RuntimeIdentity = Readonly<{ uid: number; gid: number; firstWorkerUid?: number }>; +type DirectoryEntry = Readonly<{ uid: number; gid: number; mode: number; isDirectory(): boolean; isSymbolicLink(): boolean }>; + +/** Pure shape check for a caller-prepared runtime root. */ +export function assertRuntimeDirectory(entry: DirectoryEntry, label: string, shape: DirectoryShape, runtime: RuntimeIdentity): void { if (!entry.isDirectory() || entry.isSymbolicLink()) throw new Error(`${label} must be an existing real directory`); - if (entry.uid !== process.getuid?.()) throw new Error(`${label} must be owned by the runtime user`); - const permissions = entry.mode & 0o777; - if (mode === "private" && permissions !== 0o700) throw new Error(`${label} must have mode 0700`); - if (mode === "safe" && (permissions & 0o022) !== 0) throw new Error(`${label} must not grant group or other write access`); + if (entry.uid !== runtime.uid) throw new Error(`${label} must be owned by the runtime user`); + const permissions = Number(entry.mode) & 0o7777; + if (shape === "private" && permissions !== 0o700) throw new Error(`${label} must have mode 0700`); + if (shape === "worker-traversable") { + const home = GROK_ENGINE_BROKER.worker.home.organizationRuntimeHome; + if (permissions !== home.mode) throw new Error(`${label} must have mode 0710 for a brokered Grok agent`); + if (entry.gid < (runtime.firstWorkerUid ?? GROK_ENGINE_BROKER.identities.firstWorkerUid) || entry.gid === runtime.gid) { + throw new Error(`${label} must be group-owned by the agent's Grok worker group`); + } + } + if (shape === "safe" && (permissions & 0o022) !== 0) throw new Error(`${label} must not grant group or other write access`); +} + +function assertDirectory(entry: Stats, label: string, shape: DirectoryShape): void { + assertRuntimeDirectory(entry, label, shape, { uid: process.getuid?.() ?? -1, gid: process.getgid?.() ?? -1 }); } function identity(entry: Stats): Identity { return { dev: entry.dev, ino: entry.ino, uid: entry.uid, mode: entry.mode & 0o7777 }; } From 277ea2686fb94ab68f9bf2e17abd8a59b8ea339b Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 20:09:05 +0200 Subject: [PATCH 072/124] fix: create every runtime-home directory private so a traversable Grok home exposes only tool-output --- src/observability/causalEvents.ts | 7 ++- src/observability/orgObserver.ts | 3 +- src/pi/cliSession.ts | 3 +- src/pi/piHarness.ts | 5 +- src/pi/turnTrace.ts | 3 +- src/pi/worldTrajectory.ts | 3 +- src/runtime/runtimeHomeLayout.test.ts | 90 +++++++++++++++++++++++++++ src/runtime/runtimeHomeLayout.ts | 13 ++++ 8 files changed, 118 insertions(+), 9 deletions(-) create mode 100644 src/runtime/runtimeHomeLayout.test.ts create mode 100644 src/runtime/runtimeHomeLayout.ts diff --git a/src/observability/causalEvents.ts b/src/observability/causalEvents.ts index 044b4ca..9a29dd5 100644 --- a/src/observability/causalEvents.ts +++ b/src/observability/causalEvents.ts @@ -2,6 +2,7 @@ import { createHash, randomUUID } from "node:crypto"; import { constants } from "node:fs"; import { appendFile, mkdir, open, readFile, rename, stat, unlink } from "node:fs/promises"; import path from "node:path"; +import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "../runtime/runtimeHomeLayout.js"; /** * Daimon's own copy of the `noopolis.causal-event.v1` wire envelope. Field- @@ -107,7 +108,7 @@ const readSeqStore = async (runtimeHomePath: string): Promise => const writeSeqStore = async (runtimeHomePath: string, store: CausalSeqStore): Promise => { const directory = telemetryDir(runtimeHomePath); - await mkdir(directory, { recursive: true }); + await mkdir(directory, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); const file = seqFilePath(runtimeHomePath); const temporary = `${file}.${randomUUID()}.tmp`; const handle = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); @@ -182,7 +183,7 @@ export const nextCausalSeq = async (input: { const lockPath = path.resolve(telemetryDir(input.runtimeHomePath), "causal.seq.lock"); const previous = seqAllocationQueues.get(lockPath) ?? Promise.resolve(); const allocation = previous.catch(() => undefined).then(async () => { - await mkdir(telemetryDir(input.runtimeHomePath), { recursive: true }); + await mkdir(telemetryDir(input.runtimeHomePath), { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); await acquireSeqLock(lockPath); try { const store = await readSeqStore(input.runtimeHomePath); @@ -207,7 +208,7 @@ export const nextCausalSeq = async (input: { /** Appends one CausalEvent record as a line of `runtimeHome/telemetry/causal.jsonl`. */ export const appendCausalEvent = async (runtimeHomePath: string, event: CausalEvent): Promise => { - await mkdir(telemetryDir(runtimeHomePath), { recursive: true }); + await mkdir(telemetryDir(runtimeHomePath), { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); await appendFile(jsonlFilePath(runtimeHomePath), `${JSON.stringify(event)}\n`, "utf8"); }; diff --git a/src/observability/orgObserver.ts b/src/observability/orgObserver.ts index c6dbd7f..2d21ef8 100644 --- a/src/observability/orgObserver.ts +++ b/src/observability/orgObserver.ts @@ -2,6 +2,7 @@ import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; import type { MemoryRecallAudit } from "@noopolis/mneme"; +import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "../runtime/runtimeHomeLayout.js"; export interface WakeBenchRow { agent: string; @@ -207,7 +208,7 @@ export class OrgObserver { async write(runtimeRoot: string): Promise { const telemetryDir = path.join(runtimeRoot, "telemetry"); - await mkdir(telemetryDir, { recursive: true }); + await mkdir(telemetryDir, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); const summaryRecord = { assertions: this.assertions, behavior: this.behaviorSummary(), diff --git a/src/pi/cliSession.ts b/src/pi/cliSession.ts index 49f8af8..ec4de9e 100644 --- a/src/pi/cliSession.ts +++ b/src/pi/cliSession.ts @@ -35,6 +35,7 @@ import { decodeGrokHeadlessResult } from "./grokHeadlessResult.js"; import { terminateChild, trackCliChild } from "./cliProcess.js"; import type { PiSessionLike } from "./piAgentHandle.js"; import type { PiSessionFactoryInput } from "./piHarness.js"; +import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "../runtime/runtimeHomeLayout.js"; export type CliEngineKind = "agy" | "codex" | "grok"; @@ -145,7 +146,7 @@ export const prepareCliRuntimeHome = async (runtimeHomePath: string | undefined) `${runtimeHomePath}/.local/state`, `${runtimeHomePath}/.cache`, `${runtimeHomePath}/.tmp` - ].map((directory) => mkdir(directory, { recursive: true }))); + ].map((directory) => mkdir(directory, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }))); }; const childSecretValues = (redactedNames: readonly string[]): readonly string[] => diff --git a/src/pi/piHarness.ts b/src/pi/piHarness.ts index 19a1ace..69096d9 100644 --- a/src/pi/piHarness.ts +++ b/src/pi/piHarness.ts @@ -23,6 +23,7 @@ import { type PiWakeEnvironmentContextRef } from "./piAgentWakeSupport.js"; import { DAIMON_WAKE_ID_ENV } from "./cliEnvironment.js"; import { createPiWorldTools, piWorldToolNames, type PiWorldBinding } from "./worldTools.js"; import type { PiWorldToolContextRef } from "./worldNudge.js"; +import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "../runtime/runtimeHomeLayout.js"; import { bindPiRawTrainingCapture, validatePiRawTrainingCaptureOptions, @@ -105,10 +106,10 @@ export class PiHarnessAdapter implements AgentHarnessAdapter { `${input.runtimeHomePath}/.cache`, `${input.runtimeHomePath}/.tmp`, `${input.runtimeHomePath}/tool-state` - ].map((directory) => mkdir(directory, { recursive: true }))); + ].map((directory) => mkdir(directory, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }))); await mkdir(input.workspacePath, { recursive: true }); const memoryRuntimeHomePath = this.options.memory?.runtimeHomePath ?? input.runtimeHomePath; - await mkdir(memoryRuntimeHomePath, { recursive: true }); + await mkdir(memoryRuntimeHomePath, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); const modelSpec = this.options.model ?? { auth: { method: "codex" as const }, provider: "openai", diff --git a/src/pi/turnTrace.ts b/src/pi/turnTrace.ts index 0f0f45c..5ac198b 100644 --- a/src/pi/turnTrace.ts +++ b/src/pi/turnTrace.ts @@ -6,6 +6,7 @@ import type { MemoryPrepareTurnResult, MemoryWakeMode } from "@noopolis/mneme"; import type { HarnessModelSpec, WakeEvent } from "../core/types.js"; import { redactCredentialText } from "../core/credentialRedaction.js"; +import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "../runtime/runtimeHomeLayout.js"; export interface PiTurnTraceModel { authMethod: NonNullable["method"]; @@ -269,7 +270,7 @@ export const writeTurnTraceRecord = async ( ): Promise => { const telemetryPath = path.join(runtimeHomePath, "telemetry"); const turnsPath = path.join(telemetryPath, "turns"); - await mkdir(turnsPath, { recursive: true }); + await mkdir(turnsPath, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); const body = `${JSON.stringify(record, null, 2)}\n`; await writeFile(path.join(turnsPath, `${sanitizeTraceFileId(record.turn_id)}.json`), body, "utf8"); await appendFile(path.join(telemetryPath, "turns.ndjson"), `${JSON.stringify(record)}\n`, "utf8"); diff --git a/src/pi/worldTrajectory.ts b/src/pi/worldTrajectory.ts index 93d59df..6883ed0 100644 --- a/src/pi/worldTrajectory.ts +++ b/src/pi/worldTrajectory.ts @@ -5,6 +5,7 @@ import path from "node:path"; import type { PiTurnTraceModel } from "./turnTrace.js"; import { redactTraceText, sanitizeTraceFileId } from "./turnTrace.js"; import type { PiWorldTurnContext } from "./worldNudge.js"; +import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "../runtime/runtimeHomeLayout.js"; export const WORLD_TRAJECTORY_SCHEMA = "daimon.world_trajectory.v1" as const; @@ -189,7 +190,7 @@ export const persistPiWorldTrajectory = async ( }; const telemetryPath = path.join(input.runtimeHomePath, "telemetry"); const trajectoriesPath = path.join(telemetryPath, "world-trajectories"); - await mkdir(trajectoriesPath, { recursive: true }); + await mkdir(trajectoriesPath, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); const bytes = `${JSON.stringify(record, null, 2)}\n`; await writeFile( path.join(trajectoriesPath, `${sanitizeTraceFileId(input.turnId)}.json`), diff --git a/src/runtime/runtimeHomeLayout.test.ts b/src/runtime/runtimeHomeLayout.test.ts new file mode 100644 index 0000000..6c4363d --- /dev/null +++ b/src/runtime/runtimeHomeLayout.test.ts @@ -0,0 +1,90 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, readdir, rm, stat } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { appendCausalEvent, CAUSAL_EVENT_VERSION, nextCausalSeq } from "../observability/causalEvents.js"; +import { summarizePrompt, writeTurnTraceRecord } from "../pi/turnTrace.js"; +import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "./runtimeHomeLayout.js"; + +/** + * A brokered Grok runtime home is traversable by its worker uid (0710), so + * anything Daimon creates inside it must stay 0700 — a default `mkdir` would + * make telemetry (prompts, replies, trajectories) readable by the model's own + * sandboxed worker. + */ +const withTraversableHome = async (body: (home: string) => Promise): Promise => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-home-layout-")); + const home = path.join(root, "runtime-home"); + await mkdir(home, { mode: 0o710 }); + try { await body(home); } finally { await rm(root, { force: true, recursive: true }); } +}; +const mode = async (target: string): Promise => (await stat(target)).mode & 0o7777; + +test("the runtime-home subdirectory mode grants nobody but the runtime user", () => { + assert.equal(RUNTIME_HOME_SUBDIRECTORY_MODE, 0o700); +}); + +test("telemetry directories Daimon creates in a traversable runtime home are private", async () => { + await withTraversableHome(async (home) => { + await appendCausalEvent(home, { + version: CAUSAL_EVENT_VERSION, id: "daimon:t1:turn.output.completed", type: "turn.output.completed", + occurred_at: "2026-01-01T00:00:00.000Z", actor: { kind: "agent", id: "a" }, subject: { kind: "turn", id: "t1" }, + causes: [], run_id: "run", seq: 1, payload: {} + } as never); + assert.equal(await mode(path.join(home, "telemetry")), RUNTIME_HOME_SUBDIRECTORY_MODE); + await rm(path.join(home, "telemetry"), { recursive: true }); + + await nextCausalSeq({ runtimeHomePath: home, agentId: "a", turnId: "t1", count: 1 } as never); + assert.equal(await mode(path.join(home, "telemetry")), RUNTIME_HOME_SUBDIRECTORY_MODE); + + await writeTurnTraceRecord(home, { + agent_id: "mapper", completed_at: "2026-01-01T00:00:01.000Z", + engine: { auth_method: "none", kind: "pi", model: "llama3.2", provider: "local" }, + memory: { enabled: false }, prompt: summarizePrompt("hi"), reply: { output_chars: 2, reply_given: true }, + schema: "daimon.turn_trace.v1", session: { dispose_after_wake: false, mode: "awake", thread_id: "t" }, + started_at: "2026-01-01T00:00:00.000Z", status: "completed", timings_ms: { total: 1 }, tools: [], + turn_id: "turn-1", wake: { event_id: "w", kind: "message" } + }); + assert.equal(await mode(path.join(home, "telemetry", "turns")), RUNTIME_HOME_SUBDIRECTORY_MODE); + }); +}); + +// Creations that are not inside an agent's runtime home. +const OUTSIDE_RUNTIME_HOME = [ + "src/runtime/native/copyArtifact.ts", "src/observability/emitCausalFixture.ts", "src/pi/auth.ts", "src/runtime/cli.ts" +]; + +const mkdirCalls = (source: string): string[] => { + const calls: string[] = []; + for (let index = source.indexOf("mkdir("); index !== -1; index = source.indexOf("mkdir(", index + 1)) { + let depth = 0; + for (let cursor = index + "mkdir".length; cursor < source.length; cursor += 1) { + if (source[cursor] === "(") depth += 1; + else if (source[cursor] === ")") { depth -= 1; if (depth === 0) { calls.push(source.slice(index, cursor + 1)); break; } } + } + } + return calls; +}; + +test("no runtime-home directory is created without an explicit private mode", async () => { + // Source policy: a default `mkdir` under an agent's runtime home would be 0755, + // and the home of a brokered Grok agent is traversable by its worker uid. + const offenders: string[] = []; + const walk = async (directory: string): Promise => { + for (const entry of await readdir(directory, { withFileTypes: true })) { + const target = path.join(directory, entry.name); + if (entry.isDirectory()) { if (entry.name !== "fixtures" && entry.name !== "artifacts") await walk(target); continue; } + if (!entry.name.endsWith(".ts") || entry.name.endsWith(".test.ts") || OUTSIDE_RUNTIME_HOME.includes(target)) continue; + const source = await readFile(target, "utf8"); + for (const call of mkdirCalls(source)) { + // The workspace is a caller-prepared root with its own contract (group-readable for Grok). + if (call.includes("mode:") || call.includes("{ mode }") || call.includes("workspacePath")) continue; + offenders.push(`${target}: ${call.replace(/\s+/gu, " ").slice(0, 90)}`); + } + } + }; + await Promise.all(["src/pi", "src/observability", "src/runtime", "src/mcp", "src/core"].map(walk)); + assert.deepEqual(offenders, []); +}); diff --git a/src/runtime/runtimeHomeLayout.ts b/src/runtime/runtimeHomeLayout.ts new file mode 100644 index 0000000..f8a280d --- /dev/null +++ b/src/runtime/runtimeHomeLayout.ts @@ -0,0 +1,13 @@ +/** + * Mode for every directory Daimon creates inside an agent's runtime home. + * + * A brokered Grok agent's runtime home is `0710` + * (`GROK_ENGINE_BROKER.worker.home.organizationRuntimeHome`) so its sandboxed + * worker can traverse into the setgid `tool-output/` spill directory. Traverse + * is all it may have: anything Daimon creates in that home — telemetry traces + * (prompts, replies, world trajectories), tool state, receipts, the engine's + * XDG directories and the private `.tmp` — stays `0700`, so a default + * `mkdir` (0755 under the usual umask) never turns a traversable home into a + * readable one. + */ +export const RUNTIME_HOME_SUBDIRECTORY_MODE = 0o700; From bcd2f8ec6856226eb9f23b61b9dc3c140dafb818 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 20:09:54 +0200 Subject: [PATCH 073/124] docs: state the traverse-only Grok runtime home rule and the private subdirectory rule --- src/runtime/AGENTS.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index d9544a7..4d83329 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -203,6 +203,18 @@ in `deny` (verified: `/tmp`, `/var/tmp`, `/run`, `/etc`, `sessions` all fail; only list names — `cat`/`read_file` get EACCES and it cannot create files. `1770`/`1771` make Grok refuse the profile; `1775`/`1777` leak. Any non-root process outside that group that needs temp space must get its own `TMPDIR`; +- the organization runtime home of a brokered Grok agent is `2000: + 0710` — traverse-only, so the worker can reach `tool-output/` and nothing + else. `physicalReadiness.ts` accepts exactly that shape for a `grok` agent + (owner the runtime user, mode `0710`, group a worker group that is not the + runtime's own) and keeps the plain `0700` rule for every other engine; wider + (`0711`, `0730`, `0750`, `0770`, any world bit, setgid) is refused, and so is + a `0700` home for a Grok agent, because its worker could not read its own + spills. Everything Daimon creates inside a runtime home is `0700` + (`runtimeHomeLayout.ts`: telemetry, turn traces, world trajectories, + `tool-state`, the engine XDG directories, `.tmp`), so a traversable home + still exposes nothing but `tool-output/`. A deployment-provisioned memory + home under that runtime home must stay `0700` for the same reason; - spills (`toolResultSpill.ts`) are written `0640`; provision `/tool-output` as `2000: 2750` (setgid) under a runtime home the worker can traverse, so each spill carries that agent's From d485784ec7df45d9e595ea4e40938316956a3c8f Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 21:08:42 +0200 Subject: [PATCH 074/124] fix: name the mounted tools in the agent identity envelope --- src/runtime/engineDispatcher.ts | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/runtime/engineDispatcher.ts b/src/runtime/engineDispatcher.ts index 4666aea..a5864a5 100644 --- a/src/runtime/engineDispatcher.ts +++ b/src/runtime/engineDispatcher.ts @@ -56,7 +56,8 @@ export async function startOrganizationRuntimeEngine( readablePaths: codexSandboxReadablePaths(canonicalAgent) } : undefined; - const adapter = adapterFor(canonicalAgent, controlTokenEnv, readiness.verify, readiness.executablePath, readiness.engineHomePath, paths?.verify, agyBusAddress, [...await createProductionAgentTools(canonicalAgent, wakeContext), ...(agent.attention !== undefined && attention !== undefined ? attentionTools(agent.id, attention) : [])], wakeContext, grokSandbox,grokBroker,codexSandboxPaths); + const mountedTools = [...await createProductionAgentTools(canonicalAgent, wakeContext), ...(agent.attention !== undefined && attention !== undefined ? attentionTools(agent.id, attention) : [])]; + const adapter = adapterFor(canonicalAgent, controlTokenEnv, readiness.verify, readiness.executablePath, readiness.engineHomePath, paths?.verify, agyBusAddress, mountedTools, wakeContext, grokSandbox,grokBroker,codexSandboxPaths, mountedTools.map((tool) => tool.name)); const handle = await adapter.startAgent({ id: canonicalAgent.id, name: canonicalAgent.name, @@ -118,16 +119,16 @@ export function codexSandboxReadablePaths( return [path.join(currentAgent.runtimeHomePath, "tool-output")]; } -function adapterFor(agent: OrganizationRuntimeAgentConfig, controlTokenEnv: string, verifyExecutable: () => Promise, executablePath: string, engineHomePath: string, verifyRuntimePaths?: () => Promise, agyBusAddress?: string, productionTools: readonly import("@earendil-works/pi-coding-agent").ToolDefinition[] = [], wakeEnvironmentContext: import("../pi/piAgentWakeSupport.js").PiWakeEnvironmentContextRef = {}, verifyGrokSandbox?: () => Promise,grokBroker?:EngineBrokerTurnClient,codexSandboxPaths?: { readonly protectedPaths: readonly string[]; readonly readablePaths: readonly string[] }): PiHarnessAdapter { +function adapterFor(agent: OrganizationRuntimeAgentConfig, controlTokenEnv: string, verifyExecutable: () => Promise, executablePath: string, engineHomePath: string, verifyRuntimePaths?: () => Promise, agyBusAddress?: string, productionTools: readonly import("@earendil-works/pi-coding-agent").ToolDefinition[] = [], wakeEnvironmentContext: import("../pi/piAgentWakeSupport.js").PiWakeEnvironmentContextRef = {}, verifyGrokSandbox?: () => Promise,grokBroker?:EngineBrokerTurnClient,codexSandboxPaths?: { readonly protectedPaths: readonly string[]; readonly readablePaths: readonly string[] }, mountedToolNames: readonly string[] = []): PiHarnessAdapter { const engine = agent.engine.kind; const sessionFactory = createCliSessionFactory( engine === "agy" - ? { engine, maxToolTurns: AGY_MAX_TOOL_TURNS, timeoutMs: 180_000, dbusSessionBusAddress: agyBusAddress, redactedEnvironmentNames: [controlTokenEnv], identityPrompt: identityEnvelope(agent), command: executablePath, engineHomePath, verifyExecutable, verifyRuntimePaths, + ? { engine, maxToolTurns: AGY_MAX_TOOL_TURNS, timeoutMs: 180_000, dbusSessionBusAddress: agyBusAddress, redactedEnvironmentNames: [controlTokenEnv], identityPrompt: identityEnvelope(agent, mountedToolNames), command: executablePath, engineHomePath, verifyExecutable, verifyRuntimePaths, // AGY has no broker to meter it, so the session hands its decoded // terminal-frame usage straight to the same ledger the Grok broker // appends to. `recordTurnUsage` is advisory and never rejects. onTurnUsage: (usage, outcome) => recordTurnUsage(resolveTurnUsageLedgerPath(), { agent: agent.id, wake: wakeEnvironmentContext.current ?? "wake", engine: "agy", usage, outcome }) } - : { engine, redactedEnvironmentNames: [controlTokenEnv], identityPrompt: identityEnvelope(agent), command: executablePath, engineHomePath, verifyExecutable, verifyRuntimePaths, + : { engine, redactedEnvironmentNames: [controlTokenEnv], identityPrompt: identityEnvelope(agent, mountedToolNames), command: executablePath, engineHomePath, verifyExecutable, verifyRuntimePaths, ...(engine === "codex" ? { // Codex has no broker to meter it, so publish terminal-frame usage // to the shared advisory ledger — on the wake that published and on @@ -176,11 +177,24 @@ function grokBrokerTurnFor(agent: OrganizationRuntimeAgentConfig, grokBroker: En * CLI engines do not consume Pi's resource loader. Frame the same immutable * identity in JSON so arbitrary names/instructions cannot change its shape. */ -function identityEnvelope(agent: OrganizationRuntimeAgentConfig): string { +/** + * The caller-owned prompt preamble. + * + * It names the mounted tools explicitly. A CLI engine reaches Daimon's tools + * over MCP, and Grok exposes MCP tools only through a deferred `search_tool` + * catalog, so an agent whose instructions name another engine's tool spelling + * can finish a turn having called nothing. The declared names are the caller's + * own configuration, not engine-supplied text. + */ +function identityEnvelope(agent: OrganizationRuntimeAgentConfig, mountedToolNames: readonly string[] = []): string { return [ "", JSON.stringify({ id: agent.id, name: agent.name, instructions: agent.instructions }), "", + ...(mountedToolNames.length === 0 ? [] : [ + `Your mounted tools are exactly: ${mountedToolNames.join(", ")}. Call them by these names; ` + + "your instructions may spell them differently. No other tool reaches the newsroom." + ]), "Colleagues only hear you when you call moltnet_send; your terminal response is a private note to the runtime, not a message to anyone — keep it to one line or leave it empty. " + "Do not seek transport credentials or invoke a transport CLI unless the caller explicitly mounted an authenticated transport tool.", "The following is the current wake event." From 0c16fc5d2294eb9b8489e08bacdb117d7c8bc204 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 21:15:29 +0200 Subject: [PATCH 075/124] build: stage the packaged Linux engine broker on every packing host --- src/runtime/native/copyArtifact.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/runtime/native/copyArtifact.ts b/src/runtime/native/copyArtifact.ts index 1166aa6..be3c30d 100644 --- a/src/runtime/native/copyArtifact.ts +++ b/src/runtime/native/copyArtifact.ts @@ -4,8 +4,14 @@ import { fileURLToPath } from 'node:url'; await import('./verifyArtifacts.ts'); const root = path.dirname(fileURLToPath(import.meta.url)); -const architecture = process.arch; -if (!['x64', 'arm64'].includes(architecture) || process.platform !== 'linux') { +// The broker artifacts are prebuilt, provenance-verified Linux executables checked into +// this repository, so staging one is a packaging step and not a host capability: the +// published tarball must carry `dist/runtime/native/daimon-engine-broker` on every packing +// host, because the runtime image installs that tarball with no lifecycle script that could +// stage it later. `DAIMON_ENGINE_BROKER_ARCH` selects the packaged Linux target when it is +// not the host's own architecture. +const architecture = process.env.DAIMON_ENGINE_BROKER_ARCH?.trim() || process.arch; +if (!['x64', 'arm64'].includes(architecture)) { if (process.env.DAIMON_REQUIRE_ENGINE_BROKER === '1') throw new Error('native engine broker is Linux x64/arm64 only'); process.exit(0); } From 1af01fd4a58ac67a2cf4cbfedb340665f5bc5a45 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 21:51:03 +0200 Subject: [PATCH 076/124] fix: refuse broker proxy policy misses non-retryably with a named reason --- src/runtime/grokBrokerProxy.test.ts | 7 +++-- src/runtime/grokBrokerProxy.ts | 41 +++++++++++++++++++++++--- src/runtime/grokInferenceProxy.test.ts | 18 ++++++----- src/runtime/grokInferenceProxy.ts | 14 +++++++-- 4 files changed, 62 insertions(+), 18 deletions(-) diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index 0e6e6de..81714d7 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -46,7 +46,8 @@ test("proxy refuses a fail-open tool set or an undeclared effort without calling const token = proxy.capabilities.issue("agent", "turn"); arm(proxy, async () => undefined); const full = [...lean, ...["search_replace", "todo_write", "write", "monitor"].map((name) => ({ type: "function", function: { name } }))]; for (const payload of [leanBody({ tools: full }), leanBody({ tools: [{ type: "function", function: { name: "session_title" } }] }), leanBody({ reasoning_effort: "high" }), leanBody({ reasoning_effort: undefined })]) { - assert.equal(await post(proxy.port, token, payload), 503); + // A policy miss is non-retryable: 400, so Grok fails fast instead of retrying a 503. + assert.equal(await post(proxy.port, token, payload), 400); } assert.equal(calls, 0); assert.equal(await post(proxy.port, token, leanBody()), 200); assert.equal(calls, 1); assert.ok(accessed >= 1); @@ -68,7 +69,7 @@ test("the session-title sink is refused before capability, guard, credential, or try { const token = proxy.capabilities.issue("agent", "turn", 60_000, 1); arm(proxy, async () => { guarded++; }); const title = JSON.stringify({ model: "disabled", max_tokens: 100, temperature: 0, stream: true, messages: [{ role: "user", content: "prompt-derived" }], tool_choice: { type: "function", function: { name: "session_title" } }, tools: [{ type: "function", function: { name: "session_title" } }] }); - assert.equal(await post(proxy.port, GROK_SESSION_TITLE_SINK_KEY, title), 503); + assert.equal(await post(proxy.port, GROK_SESSION_TITLE_SINK_KEY, title), 400); assert.deepEqual({ calls, accessed, guarded }, { calls: 0, accessed: 0, guarded: 0 }); // The turn capability (budget 1 request) is untouched and still serves the real request. assert.equal(await post(proxy.port, token, leanBody()), 200); @@ -85,7 +86,7 @@ test("the isolation guard is awaited before the first upstream call, and a faili order.push("guard-start"); await new Promise((resolve) => setTimeout(resolve, 30)); order.push("guard-end"); if (fail) throw new Error("no enforcement evidence"); }); - assert.equal(await post(proxy.port, token, leanBody()), 503); + assert.equal(await post(proxy.port, token, leanBody()), 400); assert.equal(upstreamCalls, 0); assert.deepEqual(order, ["guard-start", "guard-end"]); fail = false; order.length = 0; diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index 4d1e6d0..294754e 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -34,14 +34,28 @@ export async function startGrokBrokerProxy(authority: GrokBrokerCredentialAuthor return { port: address.port, capabilities,registerIsolationGuard(turnId,guard){guards.set(turnId,guard);},revokeIsolationGuard(turnId){guards.delete(turnId);},registerTurn(turnId,turn){turns.set(turnId,{policy:parseGrokBrokerModelPolicy(turn.policy),meter:turn.meter});},revokeTurn(turnId){turns.delete(turnId);}, close: () => new Promise((resolve, reject) => server.close((error) => error === undefined ? resolve() : reject(error))) }; } +/** + * A refusal the caller must not retry. + * + * Every internal refusal used to collapse into one bare 503. Grok treats 503 as + * transient and blind-retries the same request (observed: 14 retries, ~141k + * estimated tokens, then `exit 1`), so a policy miss burned a turn's budget and + * reported itself as an engine crash. Policy refusals now answer 400 with a + * reason, and only genuinely transient faults keep 503. + */ +export class GrokBrokerProxyRefusal extends Error { + constructor(readonly reason: string) { super(`grok broker proxy refused: ${reason}`); } +} + async function serve(request: IncomingMessage, response: ServerResponse, authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream, capabilities: EngineBrokerCapabilities,guards:MapPromise>,turns:Map,fallback:GrokBrokerModelPolicy,grants?:GrokInferenceGrants): Promise { let settle:((usage:ReturnType)=>void)|undefined; try { const body = await readBody(request); const headers = Object.fromEntries(Object.entries(request.headers).map(([key, value]) => [key, Array.isArray(value) ? value[0] : value])); const match=headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u); - if(match&&match[1]!.startsWith(GROK_ENGINE_BROKER.inferenceGrants.tokenPrefix)){if(!grants)throw new Error();return await serveGrokInferenceGrant({method:request.method??"",pathname:new URL(request.url??"/","http://127.0.0.1").pathname,headers,body,token:match[1]!},response,grants,authority,upstream);} - const scope=match?capabilities.inspectToken(match[1]!):undefined;if(!scope)throw new Error();const guard=guards.get(scope.turnId),turn=turns.get(scope.turnId);if(!guard||!turn)throw new Error();await guard(); - let token = await authority.accessToken(false);const rejectedDigest=createHash("sha256").update(token).digest("hex"); let prepared = authorizeGrokBrokerProxyRequest({ method: request.method ?? "", pathname: new URL(request.url ?? "/", "http://127.0.0.1").pathname, headers, body }, capabilities, token, turn.policy ?? fallback); token = ""; + if(match&&match[1]!.startsWith(GROK_ENGINE_BROKER.inferenceGrants.tokenPrefix)){if(!grants)throw new GrokBrokerProxyRefusal("inference_grants_unavailable");return await serveGrokInferenceGrant({method:request.method??"",pathname:new URL(request.url??"/","http://127.0.0.1").pathname,headers,body,token:match[1]!},response,grants,authority,upstream);} + const scope=match?capabilities.inspectToken(match[1]!):undefined;if(!scope)throw new GrokBrokerProxyRefusal("unknown_capability");const guard=guards.get(scope.turnId),turn=turns.get(scope.turnId);if(!guard||!turn)throw new GrokBrokerProxyRefusal("no_active_turn"); + try { await guard(); } catch (error) { throw new GrokBrokerProxyRefusal("worker_isolation_unverified"); } + let token = await authority.accessToken(false);const rejectedDigest=createHash("sha256").update(token).digest("hex"); let prepared = authorizeRequestOrRefuse({ method: request.method ?? "", pathname: new URL(request.url ?? "/", "http://127.0.0.1").pathname, headers, body }, capabilities, token, turn.policy ?? fallback); token = ""; // The spend gate runs after the body is proven a real lean worker request // (a refused session-title body never counts) and before any upstream call. const admission=turn.meter.admit(); @@ -52,7 +66,26 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori if (result.status === 401) { token = authority.refreshAfterRejection?await authority.refreshAfterRejection(rejectedDigest):await authority.accessToken(true);const refreshedDigest=createHash("sha256").update(token).digest("hex"); prepared = { ...prepared, headers: { ...prepared.headers, authorization: `Bearer ${token}` } }; token = ""; result = await upstream(prepared,admission.signal);if(result.status===401)await authority.markRejected(refreshedDigest); } settle?.(parseGrokUpstreamUsage(result.body,result.headers["content-type"])); response.writeHead(result.status, { "content-type": result.headers["content-type"] ?? "application/json", "cache-control": "no-store" }); response.end(result.body); - } catch { settle?.(undefined); response.writeHead(503, { "content-type": "application/json", "cache-control": "no-store" }); response.end('{"error":"broker unavailable"}'); } + } catch (error) { + settle?.(undefined); + // Name the refusal on the broker's own stderr (reason code only, never a body + // or a token) so a failing turn is diagnosable without a stub harness. + const refusal = error instanceof GrokBrokerProxyRefusal ? error.reason : "broker_unavailable"; + process.stderr.write(`[grok-proxy] refused: ${refusal}\n`); + if (error instanceof GrokBrokerProxyRefusal) { + response.writeHead(400, { "content-type": "application/json", "cache-control": "no-store" }); + response.end(JSON.stringify({ error: "broker refused this request", reason: refusal })); + return; + } + response.writeHead(503, { "content-type": "application/json", "cache-control": "no-store" }); + response.end('{"error":"broker unavailable"}'); + } +} + +/** The body gate, refused non-retryably: a rejected body is a policy miss, never a transient fault. */ +function authorizeRequestOrRefuse(...args: Parameters): ReturnType { + try { return authorizeGrokBrokerProxyRequest(...args); } + catch { throw new GrokBrokerProxyRefusal("request_body_rejected"); } } async function readBody(request: IncomingMessage): Promise { const chunks: Buffer[] = []; let bytes = 0; for await (const chunk of request) { const value = Buffer.from(chunk); bytes += value.length; if (bytes > 2 * 1024 * 1024) throw new Error("too large"); chunks.push(value); } return Buffer.concat(chunks); } const defaultUpstream: GrokBrokerUpstream = async (request, signal) => { const result = await fetch(request.url, { method: "POST", headers: request.headers, body: Buffer.from(request.body), ...(signal === undefined ? {} : { signal }) }); return { status: result.status, headers: { "content-type": result.headers.get("content-type") ?? "application/json" }, body: new Uint8Array(await result.arrayBuffer()) }; }; diff --git a/src/runtime/grokInferenceProxy.test.ts b/src/runtime/grokInferenceProxy.test.ts index 8fcb20e..1bf6aee 100644 --- a/src/runtime/grokInferenceProxy.test.ts +++ b/src/runtime/grokInferenceProxy.test.ts @@ -60,9 +60,10 @@ test("a grant refuses any tools member, the session_title request, and undeclare judgeBody({ messages: [{ role: "tool", content: "x" }] }), judgeBody({ messages: [{ role: "assistant", content: null, tool_calls: [] }] }), judgeBody({ response_format: { type: "json_object" } }) ]; - for (const body of refused) assert.equal((await post(port, token, body)).status, 503, body.slice(0, 120)); - assert.equal((await post(port, token, judgeBody(), "1.0.30")).status, 503); - assert.equal((await post(port, GROK_SESSION_TITLE_SINK_KEY, titleBody)).status, 503); + // Policy misses are non-retryable: 400, so a judge fails fast instead of retrying a 503. + for (const body of refused) assert.equal((await post(port, token, body)).status, 400, body.slice(0, 120)); + assert.equal((await post(port, token, judgeBody(), "1.0.30")).status, 400); + assert.equal((await post(port, GROK_SESSION_TITLE_SINK_KEY, titleBody)).status, 400); assert.equal(bodies.length, 0); assert.equal(rows.length, 0); }); }); @@ -70,8 +71,8 @@ test("a grant refuses any tools member, the session_title request, and undeclare test("an expired, released or unknown grant is refused", async () => { await withProxy(async ({ port, grants, bodies }) => { const released = grants.issue({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }); grants.release(released.grantId); - assert.equal((await post(port, released.token, judgeBody())).status, 503); - assert.equal((await post(port, `inference_${"A".repeat(43)}`, judgeBody())).status, 503); + assert.equal((await post(port, released.token, judgeBody())).status, 400); + assert.equal((await post(port, `inference_${"A".repeat(43)}`, judgeBody())).status, 400); assert.equal(bodies.length, 0); }); let now = 5_000; @@ -81,7 +82,7 @@ test("an expired, released or unknown grant is refused", async () => { const { token } = grants.issue({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }); assert.equal((await post(proxy.port, token, judgeBody())).status, 200); now += 600_000; - assert.equal((await post(proxy.port, token, judgeBody())).status, 503); + assert.equal((await post(proxy.port, token, judgeBody())).status, 400); } finally { grants.close(); await proxy.close(); } }); @@ -93,8 +94,9 @@ test("a grant token never authorizes a subject turn and a turn capability never proxy.registerTurn("turn-a", { policy: { model: "grok-4.6", reasoningEffort: "low" }, meter: new GrokBrokerTurnMeter({ maxRequests: 4, maxTokens: 10_000, timeoutMs: 60_000 }) }); const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); const leanBody = JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: lean }); - assert.equal((await post(port, grantToken, leanBody)).status, 503); - assert.equal((await post(port, turnToken, judgeBody())).status, 503); + // Cross-use is a policy miss on both paths: refused 400, never a retryable 503. + assert.equal((await post(port, grantToken, leanBody)).status, 400); + assert.equal((await post(port, turnToken, judgeBody())).status, 400); assert.equal(bodies.length, 0); assert.equal((await post(port, turnToken, leanBody)).status, 200); assert.equal((await post(port, grantToken, judgeBody())).status, 200); diff --git a/src/runtime/grokInferenceProxy.ts b/src/runtime/grokInferenceProxy.ts index 95169c1..34acc99 100644 --- a/src/runtime/grokInferenceProxy.ts +++ b/src/runtime/grokInferenceProxy.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto"; import type { ServerResponse } from "node:http"; +import { GrokBrokerProxyRefusal } from "./grokBrokerProxy.js"; import type { GrokBrokerCredentialAuthority, GrokBrokerUpstream } from "./grokBrokerProxy.js"; import { parseGrokUpstreamUsage } from "./grokBrokerTurnMeter.js"; import type { GrokInferenceGrants } from "./grokInferenceGrants.js"; @@ -30,8 +31,10 @@ export async function serveGrokInferenceGrant(input: GrokInferenceProxyInput, re let settle: ((usage: ReturnType) => void) | undefined; try { const grant = grants.authorize(input.token); - if (grant === undefined) throw new Error("inference grant unavailable"); - let prepared = authorizeGrokInferenceProxyRequest(input, "pending", grant.policy); + if (grant === undefined) throw new GrokBrokerProxyRefusal("unknown_or_expired_grant"); + let prepared: ReturnType; + try { prepared = authorizeGrokInferenceProxyRequest(input, "pending", grant.policy); } + catch { throw new GrokBrokerProxyRefusal("request_body_rejected"); } let token = await authority.accessToken(false); const rejectedDigest = createHash("sha256").update(token).digest("hex"); prepared = withBearer(prepared, token); token = ""; const admission = grant.meter.admit(); @@ -47,9 +50,14 @@ export async function serveGrokInferenceGrant(input: GrokInferenceProxyInput, re } settle?.(parseGrokUpstreamUsage(result.body, result.headers["content-type"])); json(response, result.status, result.body, result.headers["content-type"]); - } catch { + } catch (error) { settle?.(undefined); + // Same rule as the subject path: a policy miss is non-retryable (400), because a + // retryable 503 makes the client re-send a request the broker will never accept, + // charging estimated usage for every attempt. 503 stays for transient faults only. + process.stderr.write(`[grok-proxy] inference refused: ${error instanceof GrokBrokerProxyRefusal ? error.reason : "broker_unavailable"}\n`); if (authority.isStale?.() === true) json(response, 401, GROK_INFERENCE_AUTH_STALE_BODY); + else if (error instanceof GrokBrokerProxyRefusal) json(response, 400, JSON.stringify({ error: "broker refused this request", reason: error.reason })); else json(response, 503, '{"error":"broker unavailable"}'); } } From 7b9a92c9e182bdcc7bb08f7034b78398a3597f25 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 21:58:43 +0200 Subject: [PATCH 077/124] fix: keep the Grok session-title sink on its transient refusal shape --- src/runtime/grokBrokerProxy.test.ts | 3 ++- src/runtime/grokBrokerProxy.ts | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index 81714d7..79e8685 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -69,7 +69,8 @@ test("the session-title sink is refused before capability, guard, credential, or try { const token = proxy.capabilities.issue("agent", "turn", 60_000, 1); arm(proxy, async () => { guarded++; }); const title = JSON.stringify({ model: "disabled", max_tokens: 100, temperature: 0, stream: true, messages: [{ role: "user", content: "prompt-derived" }], tool_choice: { type: "function", function: { name: "session_title" } }, tools: [{ type: "function", function: { name: "session_title" } }] }); - assert.equal(await post(proxy.port, GROK_SESSION_TITLE_SINK_KEY, title), 400); + // The title sink keeps its transient 503 shape: a 4xx there ends Grok's session. + assert.equal(await post(proxy.port, GROK_SESSION_TITLE_SINK_KEY, title), 503); assert.deepEqual({ calls, accessed, guarded }, { calls: 0, accessed: 0, guarded: 0 }); // The turn capability (budget 1 request) is untouched and still serves the real request. assert.equal(await post(proxy.port, token, leanBody()), 200); diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index 294754e..1b89d58 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -4,6 +4,7 @@ import type { AddressInfo } from "node:net"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; import { DEFAULT_GROK_BROKER_MODEL_POLICY, parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; import { authorizeGrokBrokerProxyRequest } from "./grokBrokerProxyRequest.js"; +import { GROK_SESSION_TITLE_SINK_KEY } from "./grokBrokerWorkerConfig.js"; import { parseGrokUpstreamUsage, type GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; import { serveGrokInferenceGrant } from "./grokInferenceProxy.js"; @@ -49,8 +50,10 @@ export class GrokBrokerProxyRefusal extends Error { async function serve(request: IncomingMessage, response: ServerResponse, authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream, capabilities: EngineBrokerCapabilities,guards:MapPromise>,turns:Map,fallback:GrokBrokerModelPolicy,grants?:GrokInferenceGrants): Promise { let settle:((usage:ReturnType)=>void)|undefined; + let titleSink = false; try { const body = await readBody(request); const headers = Object.fromEntries(Object.entries(request.headers).map(([key, value]) => [key, Array.isArray(value) ? value[0] : value])); + titleSink = headers.authorization === `Bearer ${GROK_SESSION_TITLE_SINK_KEY}`; const match=headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u); if(match&&match[1]!.startsWith(GROK_ENGINE_BROKER.inferenceGrants.tokenPrefix)){if(!grants)throw new GrokBrokerProxyRefusal("inference_grants_unavailable");return await serveGrokInferenceGrant({method:request.method??"",pathname:new URL(request.url??"/","http://127.0.0.1").pathname,headers,body,token:match[1]!},response,grants,authority,upstream);} const scope=match?capabilities.inspectToken(match[1]!):undefined;if(!scope)throw new GrokBrokerProxyRefusal("unknown_capability");const guard=guards.get(scope.turnId),turn=turns.get(scope.turnId);if(!guard||!turn)throw new GrokBrokerProxyRefusal("no_active_turn"); @@ -72,6 +75,14 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori // or a token) so a failing turn is diagnosable without a stub harness. const refusal = error instanceof GrokBrokerProxyRefusal ? error.reason : "broker_unavailable"; process.stderr.write(`[grok-proxy] refused: ${refusal}\n`); + // Grok's own session-title call is refused by design. It must keep the transient + // 503 shape it has always had: a hard 4xx on that internal request ends Grok's + // session, which surfaces as the worker exiting 1 mid-turn. + if (titleSink) { + response.writeHead(503, { "content-type": "application/json", "cache-control": "no-store" }); + response.end('{"error":"broker unavailable"}'); + return; + } if (error instanceof GrokBrokerProxyRefusal) { response.writeHead(400, { "content-type": "application/json", "cache-control": "no-store" }); response.end(JSON.stringify({ error: "broker refused this request", reason: refusal })); From 1a6ee5d2d326ad0bdc516d8ac53bb48fdc5a4ac6 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 21:58:57 +0200 Subject: [PATCH 078/124] test: keep the grant-path title sink refusal transient --- src/runtime/grokInferenceProxy.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/runtime/grokInferenceProxy.test.ts b/src/runtime/grokInferenceProxy.test.ts index 1bf6aee..8e1f44c 100644 --- a/src/runtime/grokInferenceProxy.test.ts +++ b/src/runtime/grokInferenceProxy.test.ts @@ -63,7 +63,8 @@ test("a grant refuses any tools member, the session_title request, and undeclare // Policy misses are non-retryable: 400, so a judge fails fast instead of retrying a 503. for (const body of refused) assert.equal((await post(port, token, body)).status, 400, body.slice(0, 120)); assert.equal((await post(port, token, judgeBody(), "1.0.30")).status, 400); - assert.equal((await post(port, GROK_SESSION_TITLE_SINK_KEY, titleBody)).status, 400); + // The title sink keeps its transient 503 shape on the grant path too. + assert.equal((await post(port, GROK_SESSION_TITLE_SINK_KEY, titleBody)).status, 503); assert.equal(bodies.length, 0); assert.equal(rows.length, 0); }); }); From b54f66f9c9cf333dddafd30f2f797baff3ac4f31 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 22:55:08 +0200 Subject: [PATCH 079/124] fix: lead an inbox turn with each delivery's own text and keep the accounting after it --- src/runtime/attentionDispatcher.test.ts | 14 +++++++++ src/runtime/attentionDispatcher.ts | 41 ++++++++++++++++++++++--- 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/src/runtime/attentionDispatcher.test.ts b/src/runtime/attentionDispatcher.test.ts index 4774ae2..71dce93 100644 --- a/src/runtime/attentionDispatcher.test.ts +++ b/src/runtime/attentionDispatcher.test.ts @@ -200,3 +200,17 @@ test("claim-renewal failure revokes execution authority, stops cognition, and la const accepted = await f.control.accept(request("after-fence")); assert.equal(accepted.state, "stopped"); assert.equal(accepted.blocked!.reason, "ledger_unavailable"); } finally { WakeAcceptanceStore.prototype.renewClaim = original; await f.cleanup(); } }); + +test("an inbox turn leads with each delivery's own text and keeps the accounting after the work", async () => { + const f = await fixture(); + try { + await f.control.accept(request("d-1")); await until(() => f.core.wakes.length === 1); + const text = f.core.wakes[0]!.event.text; + // The task comes first: a delivery's text is the work, not a JSON payload to account for. + assert.match(text, /^Carry out this delivery\./u); + assert.match(text, //u); + const task = text.indexOf("Handle d-1"), accounting = text.indexOf("daimon_inbox_disposition"); + assert.ok(task >= 0 && accounting > task, "accounting must follow the delivery text"); + assert.ok(text.indexOf("Machine-readable payload:") > accounting, "payload stays a trailing appendix"); + } finally { await f.cleanup(); } +}); diff --git a/src/runtime/attentionDispatcher.ts b/src/runtime/attentionDispatcher.ts index b1a8f77..208bd5d 100644 --- a/src/runtime/attentionDispatcher.ts +++ b/src/runtime/attentionDispatcher.ts @@ -171,14 +171,47 @@ export function selectBatch(records: readonly StoredWakeAcceptanceRecord[], agen return selected.length ? selected : [first]; } +/** One claimed delivery, rendered as the task it is. */ +function deliveryBlock(message: unknown, index: number): string | undefined { + if (message === null || typeof message !== "object") return undefined; + const row = message as Record; + const text = typeof row.text === "string" ? row.text : undefined; + if (text === undefined) return undefined; + const from = typeof row.from === "string" ? row.from : undefined; + const kind = typeof row.kind === "string" ? row.kind : "delivery"; + const id = typeof row.delivery_id === "string" ? row.delivery_id : `#${index + 1}`; + return [``, text, ""].join("\n"); +} + +/** + * The inbox turn, task first. + * + * A delivery's own text *is* the work. Leading with bookkeeping and handing the + * model `JSON.stringify(messages)` buried the task: an agent read the JSON, did + * the accounting and deferred without doing the job (observed on Grok: nine + * model requests, no tool calls, nothing filed). The deliveries are therefore + * rendered as labelled blocks and the `daimon_inbox` accounting follows them as + * what to do *after* the work, with the machine-readable payload kept as a + * trailing appendix while it fits the same budget. + */ function inboxPrompt(messages: readonly unknown[], maxBytes = 12000): string { const body = JSON.stringify(messages); + const blocks = messages.map(deliveryBlock).filter((block): block is string => block !== undefined); + const accounting = "\nWhen the work above is done, record each delivery with daimon_inbox_disposition (complete), or defer the ones you could not finish; use daimon_inbox for deliveries and remaining allowances. Reading or ending this turn never completes a delivery, and deferred work waits for a later external wake.\n"; + const header = blocks.length === 1 ? "Carry out this delivery.\n" : `Carry out these ${blocks.length} deliveries.\n`; + const fits = (value: string): boolean => Buffer.byteLength(value) <= ORGANIZATION_RUNTIME_MAX_WAKE_TEXT_BYTES + && [...value].length <= ORGANIZATION_RUNTIME_MAX_STRING_CODEPOINTS; + if (blocks.length > 0 && Buffer.byteLength(blocks.join("\n\n")) <= maxBytes) { + const task = header + blocks.join("\n\n") + accounting; + const withPayload = `${task}\nMachine-readable payload: ${body}`; + // The inbox budget bounds selection; the v1 execution boundary independently + // bounds the complete prompt, including metadata, escaping, and instructions. + if (Buffer.byteLength(body) <= maxBytes && fits(withPayload)) return withPayload; + if (fits(task)) return task; + } const prefix = "Handle this inbox turn. Use daimon_inbox for deliveries and remaining allowances. Explicitly call daimon_inbox_disposition for each handled delivery (complete) or unfinished delivery (defer). Reading or ending this turn never completes a delivery. Deferred work waits for a later external wake.\n"; const prompt = prefix + body; - // The inbox budget bounds selection; the v1 execution boundary independently - // bounds the complete prompt, including metadata, escaping, and instructions. - if (Buffer.byteLength(body) > maxBytes || Buffer.byteLength(prompt) > ORGANIZATION_RUNTIME_MAX_WAKE_TEXT_BYTES - || [...prompt].length > ORGANIZATION_RUNTIME_MAX_STRING_CODEPOINTS) { + if (Buffer.byteLength(body) > maxBytes || !fits(prompt)) { return prefix + "The selected payload exceeds the prompt budget; read it with daimon_inbox."; } return prompt; From 8496d44c32a023b8608b420863b5800723cb80f1 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 23:30:42 +0200 Subject: [PATCH 080/124] fix: name a failed brokered worker's own reason instead of its exit code --- src/contracts/runtimeContractManifest.ts | 6 +-- src/runtime/AGENTS.md | 19 +++++++++ src/runtime/engineBrokerControlClient.ts | 2 +- src/runtime/engineBrokerNativeClient.test.ts | 26 +++++++++--- src/runtime/engineBrokerNativeClient.ts | 38 ++++++++++++++---- src/runtime/engineBrokerProtocol.test.ts | 15 +++++++ src/runtime/engineBrokerProtocol.ts | 15 ++++++- src/runtime/engineBrokerService.test.ts | 6 +++ src/runtime/native/AGENTS.md | 7 ++++ .../artifacts/daimon-engine-broker-arm64 | Bin 67456 -> 67456 bytes ...daimon-engine-broker-arm64.provenance.json | 2 +- .../native/artifacts/daimon-engine-broker-x64 | Bin 55392 -> 55392 bytes .../daimon-engine-broker-x64.provenance.json | 2 +- src/runtime/native/engineBrokerLauncher.h | 15 ++++--- .../engineBrokerLauncherIntegrationCore.inc | 8 ++-- ...ngineBrokerLauncherIntegrationLauncher.inc | 29 +++++++++++++ .../engineBrokerLauncherIntegrationMain.inc | 3 +- .../native/engineBrokerLauncherModes.inc | 10 +++-- .../native/engineBrokerLauncherServer.inc | 32 +++++++++++---- src/runtime/native/fixtureWorker.c | 2 +- 20 files changed, 195 insertions(+), 42 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 95a6673..41baa05 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -133,9 +133,9 @@ export const GROK_ENGINE_BROKER = { projectionVersion: "noopolis.daimon.grok-broker-projection.v1", slotPreflightVersion: "noopolis.daimon.grok-slot-preflight.v2", artifacts: { - sourceSha256: "c0082d4b366ffdb860d8154ee7f402b8f8be1f09d4eda277eda6965198ab6a75", - x64Sha256: "69e2865c722606a71501bc38d8b9c4c2c397e748327a8077e51e040319d1280d", - arm64Sha256: "16a3f89d84b7139d556b070a626c75ece224d5e05c52d48e045b38086c0a0382" + sourceSha256: "356a8e56e44dca9ab4784ac343587c0fc4d57e23f961124d8491cf6f504f8e98", + x64Sha256: "a21efd6a47059de7c5fe6add8b23799946728dfb44845ea9b6602402e5cfad02", + arm64Sha256: "a8bf311ca82ed004dd4efd69d1b7ae9edc1264876ca9bbb94c77226d40c75381" } } as const; export const AGY_SUBSCRIPTION_REALM = { diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 4d83329..0863d31 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -77,6 +77,25 @@ model (`grok-4.6-build` → `grok-4.6`), otherwise the turn fails as rejected an is still metered. Control protocol v2 is refused-v1 on the wire because both ends ship in this package. +A failed brokered turn also carries the worker's own last words. The launcher +gives the worker one pipe for stdout and stderr and publishes no output for a +failure, so a `worker_failed` turn used to reach the host as nothing but +`exit=1` — the reason the worker printed died with the container's tmpfs. +`DBL_MAX_DIAGNOSTIC` (512 bytes) is now the launcher's bounded tail of that +pipe, sent beside the fixed result frame in `diagnostic_length` and kept only +for a worker that exited on its own account: an output-limit tail would be the +very payload the bound refused, a cancelled turn has no reader left, and a +prelaunch failure ran nothing. `engineBrokerNativeClient.ts` redacts that tail +exactly as the CLI child path redacts a failed engine child +(`redactCredentialText` with the turn's own provider/MCP capabilities as exact +secrets, the same `CLI_ENGINE_MAX_DIAGNOSTIC_BYTES` bound) and flattens it to +one line as `diagnostic.reason`. It is an optional, control-character-free +member of the sealed terminal response's closed diagnostic — admitted by +`engineBrokerProtocol.ts` only for the statuses where a worker ran and spoke — +so it replays with the sealed record and reaches the operator through +`engineBrokerControlClient.ts`'s failure message. Nothing new is written to +disk: the reason travels inside the response the broker already seals. + Evaluator inference grants (`grokInferenceGrants.ts`) let Paideia judges and the DSPy optimizer — uid 2000, the trusted evaluator side — spend the broker's Grok credential without holding it. `request_inference_grant {model, diff --git a/src/runtime/engineBrokerControlClient.ts b/src/runtime/engineBrokerControlClient.ts index 6d5c845..a02d758 100644 --- a/src/runtime/engineBrokerControlClient.ts +++ b/src/runtime/engineBrokerControlClient.ts @@ -45,6 +45,6 @@ export class EngineBrokerControlClient implements EngineBrokerTurnClient { const turnId=createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"),requestId=randomUUID();const request={version:ENGINE_BROKER_VERSION,kind:"start_turn",requestId,turnId,agentId,wakeId,prompt,mcpEndpoint,...(options.limits===undefined?{}:{limits:options.limits})} as const;const socket=createConnection({path:this.socketPath});const decoder=new EngineBrokerFrameDecoder(); return new Promise((resolve,reject)=>{let accepted=false,settled=false;const fail=()=>{if(settled)return;settled=true;cleanup();reject(new Error("engine broker unavailable"));};const cleanup=()=>{signal?.removeEventListener("abort",abort);socket.destroy();};const abort=()=>fail();signal?.addEventListener("abort",abort,{once:true});if(signal?.aborted)return abort();socket.once("connect",()=>socket.write(encodeEngineBrokerFrame(request)));socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const response=parseEngineBrokerResponse(value);if((response.kind!=="accepted"&&response.kind!=="completed"&&response.kind!=="failed")||response.requestId!==requestId||response.turnId!==turnId)throw new Error();if(response.kind==="accepted"){if(accepted)throw new Error();accepted=true;continue;}if(!accepted||settled)throw new Error();settled=true;cleanup(); if(options.model!==undefined&&response.model!==options.model){reject(new Error(`engine broker turn used model ${response.model}, not the declared ${options.model}`));return;} - if(response.kind==="completed")resolve(response.text);else reject(new Error(`engine broker turn failed (${response.code}${response.limitReason==="none"?"":`; limit=${response.limitReason}`}${response.diagnostic ? `; ${response.diagnostic.stage}/${response.diagnostic.failureClass}; exit=${response.diagnostic.exitCode}; signal=${response.diagnostic.termSignal}` : ""})`));}}catch{fail();}});socket.once("error",fail);socket.once("close",()=>{if(!settled)fail();});}); + if(response.kind==="completed")resolve(response.text);else reject(new Error(`engine broker turn failed (${response.code}${response.limitReason==="none"?"":`; limit=${response.limitReason}`}${response.diagnostic ? `; ${response.diagnostic.stage}/${response.diagnostic.failureClass}; exit=${response.diagnostic.exitCode}; signal=${response.diagnostic.termSignal}${response.diagnostic.reason===undefined?"":`; reason=${response.diagnostic.reason}`}` : ""})`));}}catch{fail();}});socket.once("error",fail);socket.once("close",()=>{if(!settled)fail();});}); } } diff --git a/src/runtime/engineBrokerNativeClient.test.ts b/src/runtime/engineBrokerNativeClient.test.ts index 754dcfa..f960d63 100644 --- a/src/runtime/engineBrokerNativeClient.test.ts +++ b/src/runtime/engineBrokerNativeClient.test.ts @@ -1,10 +1,11 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { decodeNativeBrokerResult,encodeNativeBrokerTurn,ENGINE_BROKER_NATIVE_RESULT_BYTES,NativeBrokerTurnFailure } from "./engineBrokerNativeClient.js"; +import { CLI_ENGINE_MAX_DIAGNOSTIC_BYTES } from "../pi/cliChildOutput.js"; +import { decodeNativeBrokerResult,encodeNativeBrokerTurn,ENGINE_BROKER_NATIVE_DIAGNOSTIC_BYTES,ENGINE_BROKER_NATIVE_RESULT_BYTES,NativeBrokerTurnFailure } from "./engineBrokerNativeClient.js"; const turnId="turn-1"; -function frame(values:Readonly<{status?:number;uid?:number;pid?:number;exit?:number;signal?:number;ticks?:bigint;stage?:number;failure?:number;profile?:number;reserved?:number;text?:string}>={}):Buffer{ - const text=Buffer.from(values.text??"");const out=Buffer.alloc(ENGINE_BROKER_NATIVE_RESULT_BYTES+text.length);out.writeUInt32LE(2,0);out.writeUInt32LE(values.status??0,4);out.writeUInt32LE(values.uid??2200,8);out.writeUInt32LE(text.length,12);out.writeInt32LE(values.pid??42,16);out.writeInt32LE(values.exit??0,20);out.writeInt32LE(values.signal??0,24);out.writeBigUInt64LE(values.ticks??123n,32);out.write(turnId,40);out.writeUInt32LE(values.stage??7,108);out.writeUInt32LE(values.failure??0,112);out.writeUInt32LE(values.profile??0,116);out.writeUInt32LE(values.reserved??0,120);text.copy(out,ENGINE_BROKER_NATIVE_RESULT_BYTES);return out; +function frame(values:Readonly<{status?:number;uid?:number;pid?:number;exit?:number;signal?:number;ticks?:bigint;stage?:number;failure?:number;profile?:number;diagnosticLength?:number;diagnostic?:string;text?:string}>={}):Buffer{ + const text=Buffer.from(values.text??""),diagnostic=Buffer.from(values.diagnostic??"");const out=Buffer.alloc(ENGINE_BROKER_NATIVE_RESULT_BYTES+text.length+diagnostic.length);out.writeUInt32LE(2,0);out.writeUInt32LE(values.status??0,4);out.writeUInt32LE(values.uid??2200,8);out.writeUInt32LE(text.length,12);out.writeInt32LE(values.pid??42,16);out.writeInt32LE(values.exit??0,20);out.writeInt32LE(values.signal??0,24);out.writeBigUInt64LE(values.ticks??123n,32);out.write(turnId,40);out.writeUInt32LE(values.stage??7,108);out.writeUInt32LE(values.failure??0,112);out.writeUInt32LE(values.profile??0,116);out.writeUInt32LE(values.diagnosticLength??diagnostic.length,120);text.copy(out,ENGINE_BROKER_NATIVE_RESULT_BYTES);diagnostic.copy(out,ENGINE_BROKER_NATIVE_RESULT_BYTES+text.length);return out; } test("encodes ABI v2 and decodes a closed successful result",()=>{ @@ -21,7 +22,22 @@ test("returns bounded typed diagnostics for closed native failures",()=>{ ])assert.throws(()=>decodeNativeBrokerResult(frame(value),turnId),(error:unknown)=>error instanceof NativeBrokerTurnFailure&&error.diagnostic.stage!=="none"); }); -test("rejects unknown, reserved, output-bearing, and cross-class failure frames",()=>{ - for(const value of [{status:9},{reserved:1},{status:1,stage:7,failure:4,pid:0,uid:0,ticks:0n},{status:2,stage:6,failure:6,text:"secret"}])assert.throws(()=>decodeNativeBrokerResult(frame(value),turnId),/^Error: engine broker turn failed$/u); +test("rejects unknown, diagnostic-bearing success, output-bearing, and cross-class failure frames",()=>{ + for(const value of [{status:9},{diagnostic:"late words"},{status:1,stage:4,failure:4,pid:0,uid:0,ticks:0n,exit:-1,diagnostic:"no worker ran"},{status:2,stage:6,failure:5,exit:1,diagnosticLength:ENGINE_BROKER_NATIVE_DIAGNOSTIC_BYTES+1},{status:1,stage:7,failure:4,pid:0,uid:0,ticks:0n},{status:2,stage:6,failure:6,text:"secret"}])assert.throws(()=>decodeNativeBrokerResult(frame(value),turnId),/^Error: engine broker turn failed$/u); for(const offset of [28,31,105,107,124,127]){const hostile=frame({text:"done"});hostile[offset]=1;assert.throws(()=>decodeNativeBrokerResult(hostile,turnId),/^Error: engine broker turn failed$/u);} }); + +test("a failed worker's own last words cross as a redacted, bounded reason",()=>{ + const words=`{"type":"error","message":"session store unwritable"}\nBearer provider-cap-secret-value\ngrok: exiting 1\n`; + assert.throws(()=>decodeNativeBrokerResult(frame({status:2,stage:6,failure:5,exit:1,diagnostic:words}),turnId,["provider-cap-secret-value"]),(error:unknown)=>{ + assert.ok(error instanceof NativeBrokerTurnFailure); + const reason=error.diagnostic.reason; + assert.ok(reason!==undefined,"the worker's own reason must reach the diagnostic"); + assert.match(reason,/session store unwritable/u); + assert.doesNotMatch(reason,/provider-cap-secret-value/u,"the turn capability must never reach a diagnostic"); + assert.doesNotMatch(reason,/[\n\r\u0000-\u001f]/u,"the reason is one bounded line"); + assert.ok(Buffer.byteLength(reason,"utf8")<=CLI_ENGINE_MAX_DIAGNOSTIC_BYTES); + return true; + }); + assert.throws(()=>decodeNativeBrokerResult(frame({status:2,stage:6,failure:5,exit:1}),turnId,[]),(error:unknown)=>error instanceof NativeBrokerTurnFailure&&error.diagnostic.reason===undefined,"a worker that said nothing reports no reason rather than an empty one"); +}); diff --git a/src/runtime/engineBrokerNativeClient.ts b/src/runtime/engineBrokerNativeClient.ts index 8d905bd..18fff46 100644 --- a/src/runtime/engineBrokerNativeClient.ts +++ b/src/runtime/engineBrokerNativeClient.ts @@ -1,14 +1,18 @@ import { spawn } from "node:child_process"; +import { redactCredentialText } from "../core/credentialRedaction.js"; +import { CLI_ENGINE_MAX_DIAGNOSTIC_BYTES } from "../pi/cliChildOutput.js"; import { terminateChild, trackCliChild } from "../pi/cliProcess.js"; export const ENGINE_BROKER_NATIVE_REQUEST_BYTES = 396; export const ENGINE_BROKER_NATIVE_RESULT_BYTES = 128; +/** `DBL_MAX_DIAGNOSTIC`: the launcher's bounded tail of a failed worker's own merged stdout/stderr. */ +export const ENGINE_BROKER_NATIVE_DIAGNOSTIC_BYTES = 512; const MAX_PROMPT = 65_536, MAX_CAPABILITY = 4_096, MAX_OUTPUT = 65_536; const statuses = ["ok", "prelaunch_failed", "worker_failed", "output_failed", "cancelled"] as const; const stages = ["none", "peer", "request", "registration", "executable", "exec", "wait", "output", "attestation"] as const; const failures = ["none", "peer", "protocol", "registration", "executable", "exec", "wait", "output_limit", "cancelled", "profile_missing", "profile_invalid"] as const; -export interface NativeBrokerDiagnostic { exitCode:number;failureClass:typeof failures[number];profileApplied:boolean;stage:typeof stages[number];startTicks:string;status:typeof statuses[number];termSignal:number;workerPid:number;workerUid:number } +export interface NativeBrokerDiagnostic { exitCode:number;failureClass:typeof failures[number];profileApplied:boolean;reason?:string;stage:typeof stages[number];startTicks:string;status:typeof statuses[number];termSignal:number;workerPid:number;workerUid:number } export class NativeBrokerTurnFailure extends Error { constructor(readonly diagnostic:NativeBrokerDiagnostic){super("engine broker turn failed");} } export type NativeBrokerTurn = Readonly<{slot:number;requestId:string;turnId:string;agentId:string;wakeId:string;prompt:string;providerCapability:string;mcpCapability:string}>; export interface NativeBrokerTurnResult {text:string;workerPid:number;workerUid:number;startTicks:bigint;diagnostic:NativeBrokerDiagnostic} @@ -16,22 +20,40 @@ export interface NativeBrokerTurnResult {text:string;workerPid:number;workerUid: export async function runNativeBrokerTurn(executable:string,input:NativeBrokerTurn,signal?:AbortSignal):Promise>{ const frame=encodeNativeBrokerTurn(input),child=trackCliChild(spawn(executable,["--client"],{detached:process.platform!=="win32",env:{LANG:"C",LC_ALL:"C",TZ:"UTC"},stdio:["pipe","pipe","ignore"],...(signal===undefined?{}:{signal})}));const chunks:Buffer[]=[];let bytes=0; child.stdout!.on("data",(chunk:Buffer)=>{bytes+=chunk.length;if(bytes<=ENGINE_BROKER_NATIVE_RESULT_BYTES+MAX_OUTPUT)chunks.push(chunk);});child.stdin!.end(frame);frame.fill(0); - try{const code=await new Promise((resolve,reject)=>{child.once("error",reject);child.once("exit",resolve);});if(code!==0||bytes>ENGINE_BROKER_NATIVE_RESULT_BYTES+MAX_OUTPUT)throw new Error();return decodeNativeBrokerResult(Buffer.concat(chunks),input.turnId);}catch(error){if(error instanceof NativeBrokerTurnFailure)throw error;throw new Error("engine broker turn failed");}finally{await terminateChild(child).catch(()=>undefined);} + try{const code=await new Promise((resolve,reject)=>{child.once("error",reject);child.once("exit",resolve);});if(code!==0||bytes>ENGINE_BROKER_NATIVE_RESULT_BYTES+MAX_OUTPUT)throw new Error();return decodeNativeBrokerResult(Buffer.concat(chunks),input.turnId,[input.providerCapability,input.mcpCapability]);}catch(error){if(error instanceof NativeBrokerTurnFailure)throw error;throw new Error("engine broker turn failed");}finally{await terminateChild(child).catch(()=>undefined);} } export function encodeNativeBrokerTurn(input:NativeBrokerTurn):Buffer{if(!Number.isInteger(input.slot)||input.slot<0)throw new TypeError("invalid engine broker turn");const p=Buffer.from(input.prompt),provider=Buffer.from(input.providerCapability),mcp=Buffer.from(input.mcpCapability);if(p.length<1||p.length>MAX_PROMPT||provider.length<1||provider.length>MAX_CAPABILITY||mcp.length<1||mcp.length>MAX_CAPABILITY||provider.equals(mcp))throw new TypeError("invalid engine broker turn");const c=Buffer.alloc(4+provider.length+mcp.length);c.writeUInt16LE(provider.length,0);provider.copy(c,2);c.writeUInt16LE(mcp.length,2+provider.length);mcp.copy(c,4+provider.length);const frame=Buffer.alloc(ENGINE_BROKER_NATIVE_REQUEST_BYTES+8+p.length+c.length);frame.writeUInt32LE(2,0);frame.writeUInt32LE(input.slot,4);field(frame,8,65,input.requestId);field(frame,73,65,input.turnId);field(frame,138,129,input.agentId);field(frame,267,129,input.wakeId);let o=ENGINE_BROKER_NATIVE_REQUEST_BYTES;frame.writeUInt32LE(p.length,o);o+=4;p.copy(frame,o);o+=p.length;frame.writeUInt32LE(c.length,o);o+=4;c.copy(frame,o);p.fill(0);provider.fill(0);mcp.fill(0);c.fill(0);return frame;} -export function decodeNativeBrokerResult(output:Buffer,turnId:string):NativeBrokerTurnResult{ +export function decodeNativeBrokerResult(output:Buffer,turnId:string,secrets:readonly string[]=[]):NativeBrokerTurnResult{ if(output.lengthbytes.every((byte)=>byte===0)); - if(output.length!==ENGINE_BROKER_NATIVE_RESULT_BYTES+length||output.readUInt32LE(0)!==2||status>=statuses.length||stage>=stages.length||failure>=failures.length||profile>1||reserved!==0||!paddingZero||observed!==turnId||length>MAX_OUTPUT)throw new Error("engine broker turn failed"); - const diagnostic:NativeBrokerDiagnostic={status:statuses[status]!,stage:stages[stage]!,failureClass:failures[failure]!,profileApplied:profile===1,exitCode,termSignal,workerPid:pid,workerUid:uid,startTicks:ticks.toString()}; - const success=status===0&&stage===7&&failure===0&&profile===0&&pid>0&&uid>=2200&&ticks>0n&&exitCode===0&&termSignal===0; - const prelaunch=status===1&&stage>=1&&stage<=5&&failure>=1&&failure<=5&&profile===0&&pid===0&&uid===0&&ticks===0n; + if(output.length!==ENGINE_BROKER_NATIVE_RESULT_BYTES+length+diagnosticLength||output.readUInt32LE(0)!==2||status>=statuses.length||stage>=stages.length||failure>=failures.length||profile>1||!paddingZero||observed!==turnId||length>MAX_OUTPUT||diagnosticLength>ENGINE_BROKER_NATIVE_DIAGNOSTIC_BYTES)throw new Error("engine broker turn failed"); + const reason=workerReason(output.subarray(ENGINE_BROKER_NATIVE_RESULT_BYTES+length),secrets); + const diagnostic:NativeBrokerDiagnostic={status:statuses[status]!,stage:stages[stage]!,failureClass:failures[failure]!,profileApplied:profile===1,...(reason===undefined?{}:{reason}),exitCode,termSignal,workerPid:pid,workerUid:uid,startTicks:ticks.toString()}; + const success=status===0&&stage===7&&failure===0&&profile===0&&pid>0&&uid>=2200&&ticks>0n&&exitCode===0&&termSignal===0&&diagnosticLength===0; + const prelaunch=status===1&&stage>=1&&stage<=5&&failure>=1&&failure<=5&&profile===0&&pid===0&&uid===0&&ticks===0n&&diagnosticLength===0; const worker=status===2&&stage===6&&(failure===5||failure===6)&&profile===0&&pid>0&&uid>=2200&&ticks>0n; const outputFailure=status===3&&stage===7&&failure===7&&profile===0&&pid>0&&uid>=2200&&ticks>0n; const cancelled=status===4&&stage===6&&failure===8&&profile===0&&pid>0&&uid>=2200&&ticks>0n; if(!success){if(length!==0||(!prelaunch&&!worker&&!outputFailure&&!cancelled))throw new Error("engine broker turn failed");throw new NativeBrokerTurnFailure(diagnostic);} return{text:output.subarray(ENGINE_BROKER_NATIVE_RESULT_BYTES).toString("utf8"),workerUid:uid,workerPid:pid,startTicks:ticks,diagnostic}; } +/** + * The worker's own last words, fit to cross a boundary. + * + * A failed brokered turn otherwise reports nothing but `exit=1`: the launcher + * merges the worker's stdout and stderr into one pipe and publishes no output + * for a failure, so this bounded tail is the only account of why it failed. + * It is worker-controlled text, so it is redacted exactly as the CLI child + * path redacts a failed engine child (`redactCredentialText` with the turn's + * own capabilities as exact secrets, the same diagnostic bound) and flattened + * to one line, because it travels inside a failure message. + */ +function workerReason(tail:Buffer,secrets:readonly string[]):string|undefined{ + if(tail.length===0)return undefined; + const flattened=tail.toString("utf8").replace(/[\u0000-\u001f\u007f]+/gu," ").replace(/\s+/gu," ").trim(); + const reason=redactCredentialText(flattened,secrets,CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); + return reason.length===0?undefined:reason; +} function field(target:Buffer,offset:number,length:number,value:string):void{if(!/^[A-Za-z0-9][A-Za-z0-9._:-]*$/u.test(value)||Buffer.byteLength(value)>=length)throw new TypeError("invalid engine broker turn");target.write(value,offset,"utf8");} diff --git a/src/runtime/engineBrokerProtocol.test.ts b/src/runtime/engineBrokerProtocol.test.ts index ae762bc..2ce17e4 100644 --- a/src/runtime/engineBrokerProtocol.test.ts +++ b/src/runtime/engineBrokerProtocol.test.ts @@ -62,3 +62,18 @@ test("start_turn limits are an optional closed subset inside their bounds", () = assert.throws(() => parseEngineBrokerRequest({ ...start, limits }), /invalid broker frame/u); } }); + +test("a failed worker's redacted reason is an optional bounded member of its diagnostic",()=>{ + const worker={status:"worker_failed",stage:"wait",failureClass:"exec",profileApplied:false,exitCode:1,termSignal:0,workerPid:31,workerUid:2200,startTicks:"9"} as const; + const failed={version:start.version,kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",outcome:"failed",usage:null,model:"grok-4.6",requests:4,limitReason:"none"} as const; + const named={...failed,diagnostic:{...worker,reason:"grok: session store unwritable"}} as const; + assert.deepEqual(parseEngineBrokerResponse(named),named); + const prelaunch={status:"prelaunch_failed",stage:"executable",failureClass:"executable",profileApplied:false,exitCode:-1,termSignal:0,workerPid:0,workerUid:0,startTicks:"0"} as const; + for(const bad of [ + {...failed,diagnostic:{...worker,reason:""}}, + {...failed,diagnostic:{...worker,reason:"x".repeat(769)}}, + {...failed,diagnostic:{...worker,reason:"line\nbreak"}}, + {...failed,diagnostic:{...worker,reason:7}}, + {...failed,diagnostic:{...prelaunch,reason:"no worker ran"}} + ])assert.throws(()=>parseEngineBrokerResponse(bad),/invalid broker frame/u); +}); diff --git a/src/runtime/engineBrokerProtocol.ts b/src/runtime/engineBrokerProtocol.ts index 9b9f644..eedf428 100644 --- a/src/runtime/engineBrokerProtocol.ts +++ b/src/runtime/engineBrokerProtocol.ts @@ -20,7 +20,15 @@ export type EngineBrokerRequest = | Readonly<{ version: typeof VERSION; kind: "cancel_turn"; requestId: string; turnId: string }> | EngineBrokerInferenceRequest; -export interface EngineBrokerFailureDiagnostic { status:string;stage:string;failureClass:string;profileApplied:boolean;exitCode:number;termSignal:number;workerPid:number;workerUid:number;startTicks:string } +/** + * `reason` is the worker's own last words (`engineBrokerNativeClient.ts`), + * already redacted and flattened to one bounded line by the broker. It is the + * only field a failed turn carries that the worker itself wrote, so it is + * optional, bounded, control-character free, and admitted only for the + * statuses where a worker actually ran and spoke. + */ +export interface EngineBrokerFailureDiagnostic { status:string;stage:string;failureClass:string;profileApplied:boolean;reason?:string;exitCode:number;termSignal:number;workerPid:number;workerUid:number;startTicks:string } +export const ENGINE_BROKER_MAX_DIAGNOSTIC_REASON_BYTES = 768; export type EngineBrokerResponse = | Readonly<{ version: typeof VERSION; kind: "ready"; requestId: string; brokerUid: 2100; providerProxyPort: 43123; mcpFacadePort: 43124; registrations: number; credentialStale: false; realmLease: true; workerIsolation: true }> @@ -105,7 +113,7 @@ function parseTerminal(input: JsonRecord, expected: typeof VERSION | typeof V1): const codes: readonly string[] = expected === VERSION ? ENGINE_BROKER_FAILURE_CODES : ENGINE_BROKER_FAILURE_CODES.filter((code) => code !== "limit_exceeded"); if (!codes.includes(input.code as string)) throw new TypeError("invalid broker frame"); let diagnostic:EngineBrokerFailureDiagnostic|undefined; - if(input.diagnostic!==undefined){const value=record(input.diagnostic);exact(value,["status","stage","failureClass","profileApplied","exitCode","termSignal","workerPid","workerUid","startTicks"]);const status=["prelaunch_failed","worker_failed","output_failed","cancelled"],stage=["peer","request","registration","executable","exec","wait","output","attestation"],failureClass=["peer","protocol","registration","executable","exec","wait","output_limit","cancelled","profile_missing","profile_invalid"];if(!status.includes(value.status as string)||!stage.includes(value.stage as string)||!failureClass.includes(value.failureClass as string)||typeof value.profileApplied!=="boolean"||![value.exitCode,value.termSignal,value.workerPid,value.workerUid].every(Number.isSafeInteger)||typeof value.startTicks!=="string"||!/^(0|[1-9][0-9]*)$/u.test(value.startTicks)||!closedDiagnostic(value))throw new TypeError("invalid broker frame");diagnostic=value as unknown as EngineBrokerFailureDiagnostic;} + if(input.diagnostic!==undefined){const value=record(input.diagnostic);const fields=["status","stage","failureClass","profileApplied","exitCode","termSignal","workerPid","workerUid","startTicks"];exact(value,value.reason===undefined?fields:[...fields,"reason"]);if(value.reason!==undefined&&(typeof value.reason!=="string"||value.reason.length===0||Buffer.byteLength(value.reason,"utf8")>ENGINE_BROKER_MAX_DIAGNOSTIC_REASON_BYTES||/[\u0000-\u001f\u007f]/u.test(value.reason)))throw new TypeError("invalid broker frame");const status=["prelaunch_failed","worker_failed","output_failed","cancelled"],stage=["peer","request","registration","executable","exec","wait","output","attestation"],failureClass=["peer","protocol","registration","executable","exec","wait","output_limit","cancelled","profile_missing","profile_invalid"];if(!status.includes(value.status as string)||!stage.includes(value.stage as string)||!failureClass.includes(value.failureClass as string)||typeof value.profileApplied!=="boolean"||![value.exitCode,value.termSignal,value.workerPid,value.workerUid].every(Number.isSafeInteger)||typeof value.startTicks!=="string"||!/^(0|[1-9][0-9]*)$/u.test(value.startTicks)||!closedDiagnostic(value))throw new TypeError("invalid broker frame");diagnostic=value as unknown as EngineBrokerFailureDiagnostic;} const base = { kind: "failed", requestId: id(input.requestId), turnId: id(input.turnId), code: input.code as EngineBrokerFailureCode, ...(diagnostic ? { diagnostic } : {}) } as const; if (expected === V1) return { version: V1, ...base } as V1Failed; const accountingValue = parseEngineBrokerTurnAccounting(input, "failed"); @@ -115,6 +123,9 @@ function parseTerminal(input: JsonRecord, expected: typeof VERSION | typeof V1): function closedDiagnostic(value:JsonRecord):boolean{ if(value.profileApplied!==false||(value.workerPid as number)<0||(value.workerUid as number)<0)return false; + // Only a worker that ran and wrote something can have said why it failed: + // no prelaunch failure and no attestation refusal carries worker words. + if(value.reason!==undefined&&!((value.status==="worker_failed"&&value.stage==="wait")||value.status==="output_failed"||value.status==="cancelled"))return false; const noWorker=value.workerPid===0&&value.workerUid===0&&value.startTicks==="0"; const worker=(value.workerPid as number)>0&&(value.workerUid as number)>=2200&&value.startTicks!=="0"; if(value.status==="prelaunch_failed")return noWorker&&({peer:"peer",request:"protocol",registration:"registration",executable:"executable",exec:"exec"} as Record)[value.stage as string]===value.failureClass; diff --git a/src/runtime/engineBrokerService.test.ts b/src/runtime/engineBrokerService.test.ts index d33f2bf..c48a192 100644 --- a/src/runtime/engineBrokerService.test.ts +++ b/src/runtime/engineBrokerService.test.ts @@ -56,3 +56,9 @@ test("a limit failure reaches the client with its code and limit reason", async await assert.rejects(client.turn("agent-a","wake-a","hello","http://127.0.0.1:44001/mcp"),/engine broker turn failed \(limit_exceeded; limit=requests\)/u); },async()=>{throw new EngineBrokerTurnFailure("limit_exceeded",undefined,{outcome:"failed",usage:{input:1,cacheRead:0,cacheWrite:0,output:1,total:2},model:"grok-4.6",requests:3,limitReason:"requests"});}); }); + +test("a failed worker's own reason reaches the client instead of a bare exit code", async () => { + await withService(async (client) => { + await assert.rejects(client.turn("agent-a","wake-a","hello","http://127.0.0.1:44001/mcp"),/engine broker turn failed \(engine_failed; wait\/exec; exit=1; signal=0; reason=grok: session store unwritable\)/u); + },async()=>{throw new EngineBrokerTurnFailure("engine_failed",{status:"worker_failed",stage:"wait",failureClass:"exec",profileApplied:false,exitCode:1,termSignal:0,workerPid:31,workerUid:2200,startTicks:"9",reason:"grok: session store unwritable"},{outcome:"failed",usage:null,model:"grok-4.6",requests:4,limitReason:"none"});}); +}); diff --git a/src/runtime/native/AGENTS.md b/src/runtime/native/AGENTS.md index 8fd8f77..5e8f4f9 100644 --- a/src/runtime/native/AGENTS.md +++ b/src/runtime/native/AGENTS.md @@ -39,3 +39,10 @@ Holding one verified descriptor and `execveat`-ing it would not make a replaced binary unrunnable: Grok 1.0.34 re-executes itself inside bubblewrap by path (`/usr/local/bin/grok`), so the image path's root ownership, not the launcher descriptor, is what protects the sandboxed process. + +The result frame's last word is `diagnostic_length`, not padding: on +`DBL_STATUS_WORKER_FAILED` the supervisor keeps the last `DBL_MAX_DIAGNOSTIC` +bytes of the worker's merged stdout/stderr and sends them after the fixed +frame, while `output_length` stays 0 as before. Every other failure sends none, +and `closed_result` refuses a frame that mixes the two. The bytes are the +worker's own, so the broker redacts them before they cross any boundary. diff --git a/src/runtime/native/artifacts/daimon-engine-broker-arm64 b/src/runtime/native/artifacts/daimon-engine-broker-arm64 index 6c393474c393788fcddb983d037e3d6986f75d10..8f89d08f2297daaa569b69b164f914dd6d52455a 100755 GIT binary patch delta 8940 zcmc&)eRNdSwLfR>Op+l9BqU)bBw=O}zA}-IKqenT%$LsbYOqgaXtk0&X+hw@^wl=0t&g_ws!|CV&>iKx>SHb<8qJt3NIm zBa;ym=EQgz_*~F*P&Q}=C6$GTL$XlOY^x5IiK(cMN}AFRkT z3aM4 zs#aoaFwDyfX;E}N7m9d6Ml6yR#};}V-$4gP%{PgBx;m;>rc!l=EjPYkSkD_eA_Yl8 ze=)7ju$A(9jHiKwQZ){OZyS!bGNF}1yE1HaJR>cAX98-W6F`T6?oB|GX&C4YnQ7@q z6HqHn$+YE;NYH;Qp)`%Yk!hnHXmct79Z8P?{T0yj31|kr3iLCe0}1G8`W$Hf=(O~d zFTb)on+itT=oX;%1au571iBgMgKEbRmOA<+5+G@nibP0LD4Uz~uB zqx38r%?7$60d-O}&{aS;B%p<~8t8LCwPOx;Ft`K*M8fg@04T zQ^T8j_I4{vPsaV-x9K;XeOO+ZznwRjVWO!vTk7k`6_#sGljGz;)ux(BQ>IfO@0L-Q zzSKJcR^h?Hvj4EjLm%W#@VD!FN53N6-N^$LVI`;QIOMPhH2w@c0z9l_b$y{o_aD*L zlqNDG%9yUOlI{Kr>r#Y=+iXc{Ym{iSkK3A}m@E%8S|R%kIomBwfyN%R2}>@|azBnT zhf^8%4l7e+5c^o57fO2XMZp*7IG_xaKT5mui~Xq@8{q+tQ?1N}!Lsb~rE1f;17L;> z+!ajKN?5U3+~#CFJeY1{ z*Xc){KW9U5$v~qMl~cPP?`KV!(8R+GU(qXYk$B^&-|{fCyRNh-oeL;;T#>&3H7*%5Fs7>( zTB6>!!TVY6e}e3@uqU&-OjnITG)`dK%u$TJ--bG>A!oUzInY?emPBXForJoPttKr1 zOU8h1MHxpyP>3==#(Oder~0@89KG+2D=KDJbr0)hD-E~gxIcx=R`{92#aCMDI(MV& z6Pn;Ck>}}lN1i-Ds~vg8@1utEQ{BT65>ZTuCX@RFu)z8Mi5;Q?4hOCYZ=z)neeRek z50T3`Nq&zSoOx4968gCr{hS@f$RtmDwtGCv5~p!5X-=E(b8IaEh0#)?45cAJ3<_~3_Rb-8d1m$(Y#$JF4OHEJ44j8MF$SDamzM&z0R zkAz$fIgZ|Qjn6pMUCqSrmvFi$>7#f5nV+hL*;&OWO|C(MlS{|SieKSIrg z6}ekbBP<5WgC>}hAzHh?)Qz~@Ne2p-W^O>~B@8v9)iZ7rx2mP$BccE9-H8o zuTuKNr8(8m+zkq&o;W{$P{kk*QQO2LvXH7K-621s$0pU_T6O~Y56Lz8pnQVfnLJ7U zfQBdEXBxgn-z#1w7tx!=B{@I1hQ%WXj&KN{!9ZMXfkrcBx*b_ZQG(?#dH@Dsb9zVL zI)>`p)j93JO=_T*H@iX+yeMJ%cGBD>at^)fwppfPIR)rFchQq2GjJ`PSyJj>i6)L@ zQR*|I%5)o5Gb zy(@@Cbx`+&x2mCtzPr(67Pl`Ia^>Cuz3FmnmnekJK}B?l0iDj4+-X?`c^_5ZFyFr( zxnPbs98RqbIZ{NcH%B!23WerM3G{vd&E46e%r{XQ^1Iv<%WjkA$_Vtn#pPL$r$UOv;pohCqgBa=~#%XvTKIO4xN3 zZJeV9DvUT19xc*h(rPV%#)mhn`&u&8!7qRM=oMk+0$>Eo^5(8u?Q&|gJL>ozl1y&q*}(K;FVbL+jK zb1Ow?w^Ow4RYcR-P2SMi~y*}S2TT14nKs@l5OB$}GxrSMNgvsMV#9fJ{q z54cgNH0qY(b9oDkoufoqkFdD8182HuVt0o}fTxL8oK;P{)&nW?Hu_TPH>!}sbt&4` zBBAk_6;blLzG)VtEQmqso|+?rbaHB~e=EksN7J zZ2eYP{HYPdJv(?1%h!p+ETZHYav=h9FMJTVq95i-;0bVIitmEK!E3tbES$$}&tL)= zUnWecC)g0XXpLObn_9BP?SA-a8}f03rf@!H&>#BQAb-@NAT06%_z(&nLG-tPcEg#= zcFx(j?TcumGZ>W+COle5claV!3q}*Xs2~1wZ1moXsP`LnPymZ}xbMPO9X4t>YrHHaf3&I9iJ)JIfnED6l%hJ20izb=OJ~3sEADS zKIAYL+j2j`)xEXr>Y7#5M*z2%Im*1P9R~^!u zX0rwNsUf~a?ZMu3o5!u7NwvcQb5 zep8D7f=vzdu79%Q#|O}O9WKW{n;P=rn&Qh-L%6G!@iyw%>m62O25{#Wxg*A3@D2wO zaK3~w&Q7_4W1Ah~c-cGbhv5&vCx+p00k2+*o1Ptg7aLwqId9_#6BpwQ8e{T zz%~cJvObOhC-l9@bxVXN3!Jygpj$L?KLT_sFsECTd-|bB*{o7>=Oev%Z_!ulE7vZy zOxiB7_wreM9_7rr^TAlBDz#Q-HGBsu@c3ec=w(HAjf9L4z3}-&(Nwdtyb|}PE*B1& z4B+ZSaY&}9Rd{))MJ9-{WiUa(Q-jP`E1T&=rDHA+oV7)2#2~v$ju(d_XQgJ6f!;wr zl8Sf%Mz7PlKczDWTY@u%2Ls|~i(VeMlS->>{w3Vs4BWYv)EZWm-WpnDRR+l^arAD6R?j`yJaGst1cyg7M6LC}A2ofLop2&d@8jr0BA z1LRP~u9Vn=@cy$8Z-)9-dMmUAZ50~c1oz=(ZeMYvLQ8Q~YIrBwhZk;rpZF@Z*MpTB zUc>g`r9!iFYIIcsJW#+-A{I_E@w} z@-jQ|YONiQrHk7?>9%OEizMx%tkIo#4c$H-^rA0GdnAygalGRtQeSPjTyu#E?b^H~ zZQ!R#8X^rk%#p?VaO*Bn-X-jXUWrDFCKb3@=4Bo@LXnpyI zSp3qx1!$zDc)JUVukccbvOiL1^*SY4+d!_Ha+DP9eyX}@ZgGm1BCT4?>+PQet=j8h zt9Gu=q@CTW7>>4xEseF#8|kH+3Zz26zv)NTX6_%RFQq$cY8onH1L0sQi{+bjc4i8=*)zw&E>gO^wR7_Lm)JZ$F&Y3y) zkG!klMM4C2V3X-8LHodv?%9ol_6+X9QnVYJlEmYI=pLbQAAf-!Os`wTLpb;GFnu~_ zsVt|4xwCSGNykVOltUl3Ypa=I_Y-u)CU9_*Nx)Vzq^7bbU?cxhSe&qjh4Y z+s@On`p4y3x=_DV-b=SPY?oDZx?#C|;^5o`byD7R@X?z$IpqTfEvu4~{pwy_{|sBl zqW!wQ8C!}tsOvXimHWU|@OE$&3UKEkUEcw&f(OADfgj~^aNZq!hjl%hS3fv^ z9a!~my1o`%9MSa`z}LL0>vrrI?cmcD;TPs#>-u5XJGCeQ3hZjKY)U&q2^!6)sb!{P@+6&fAk}uQT zbrq(*gY@vaiL=*m9eA_}o?XKO@Qw_o){YGR$y~SVP{Z>F8*1s5hNliR)IGhw;hA4F z%Jv*BO58caVQ@!q2Oqxo{=28%Ir-Mxr~fiicvGFw_8cl@ zP|tdYr5m|(^wIhb3s3lYYTK~hatgUyXn4c=DNUH{z<=ucEoj4ueYyhJuM)6Dz%~OL zpgj-WHsuZE%x`=(Bi!N~uzoyz;UqN@f2AA%_8ApyoK!F#eqQtDS1q^-&*N<1JE?hN z*4QTGf){jsJQpFcKREo3?>+eVMpK|U{*@~JQEJB|qnf}(8fgaH#3?TG8sU@{WkR(@ zGwCt8e^oRS|4Ld<2d+8fVtVsh4L0q}A%{Flr9;y)*Tld2#M*73)*(mcF&;7!!rVyh zv}foxIh`(`-A>9J_WSWA zepyPjBO)ap7?V4endH<=JQf=%&X0ENG0AC}JNd5wNOAs5M+C5SI{!}sX?jPlA}g%z zF|oa43B>DR8xP+N9eWfsQe!ivcD%>6{`bVK>2R6Rqu@UraEkH%j%L8t_(JOMc-AZ{ zMr5pxR?1l2@xED>Td)5gq$Ae?-KW+@i{pHMN3#V=;@_m}JDx=?Yy6Az;*R%Ny#7y< Wj@%^Zh%d=C!%CgmZuE%#6;x-*>`S*F^9r9_JxL z6c30@Zp~a7tFu^U!n(Xk1JMu~-E* z2Kn)z3{V589zs7cE)HW}Ok$^_X!rJ!k`c(Nl?3mOZW3!-F-GWzvtS?4zZn}Ga? znA{lSRp2v0<)AE3B`6u>0M&t}gD4e68LI)G6U*m-TS2KY`KB1J0G}1_$pv8sXp&Ic zmWhQ!Cjb0oo@O2$tD7cH@O*B@uQf9rSV;`e2cM`M%(}HW2IR*0s2C?!21<#^Bf)95 z-Joia2UG|u0Of%s&}>jPh>3%SIou=$4JG`)gu`e>o_^G);19u9$!> zpV8X(YQH^HU#x~P2T&RSUx@)6H#hlZBR7azV>aKfseDwc-Yk}wfKiA5h z)<&$;YMc_6QgSs^(P}MB&E-Ry+^3Zr$QRIgF!gB9BWcAt(Vg)q!jI$61$!xo?I6GM zG-k-i<^iL`nVvagD zexTt5v|M})^fb`E1hi75=fKZ7@N+CeC=#kf9ne~!mITx-9sqhjPqr+wA)HhudE78rC!A13lY4+;-%pkbc?8K45(a-OM*< z4_b=E_pR2Rr>xTrPZrD{!By2So0%K?^?}2>DdNt8alYq)UBR9ZN*<^Q>vOt4fE;FW z^Dn?7z{C2i?#p`aJb6R)>(W?eL_exKtj~6S23giKH?`TC)ZWa|CP-~{QH)p1%{h>< zk)Tetlk4Q>V`vlRBxgCFLm7rJ=6x#E^FXtX$-Pk0yAlPL(Xp%_sD4Tm7MA)<3K`)6 zj?>2Fz}QH3cuh(zbpTA0fja^wrG^w6*mv!Or_0Sa^0WVdj&Yo?S;bjP07M3uQLhL_ zQ=g2PVWL<>Usr zc@|j`oiXP~)QxP@DKab>1-b)e6a@jsTt1BVWD*n8ZH0WKSZpgUC0B)pRo5!bEjiAs zs7+SvAPyJzw>EZlqU^HRXDj1Z#0gtIe@$Gp<(K{)HSFhAH$_N9uR}EHoTq@vpZ-rQ zBr5GT{wJ}>p3jeo2kccmB=*`%_-65rJ%3V7LO(a7pD%_nGMsZ1Iu$5OoW?^U(_uBg zK>dbQcbcel)9x9`%dMO87A;Xp?MfFjfRYw$lQf!3R%@A37Yk7X92&$iEct zIHr%7juI^tud6j@?>399qH*xZv?3dKip51^EiZ-YXj(~$xg9|*r!~nO3_pv?9{Q~uTU0Z?2r~}i0pTVFlgXs zpo{6`-m7=&y883kARiWg96S2v8z^qqhLF`Q(6oP>QP~@lpMcB}_n7JKoybRG_z#g^ z9>X6+9tV9#Yp7gQjhj|^5c(n*@)_(B6k9pe4tbcD59k*ZC0Szb3`yQO*Y;54TBOVZGIP`4&&t)jJ!yXb}yr zn%RXu74>FGp}ibSM%fD2^;0+xs_yVMDHu^Zn{@`ZXc^8ov*Dzj$rp9N#wNq#SlQoZ1kynb?|0Hf!+~F~wn$dq09^Z#Hvz$8$~oGvsg< zWT#GXSmfUKsGQ`<22FmK%11KS8L~v5*Ny4wVBSiFHlG1lat4blf^AI01C>gds)}us z=FYWSSc?Om)8aC=13+qQab+EP-)A44`pgn`RNV$~8TFAwLB=9uIso+@LvT62auc>LX z3_RRuQnnQ{g$^krg6%E?8xheLbdR>YQA>l!xBR{q?mIwRN}l8yruv=rY_P!cW^JM6 ztykrN9>eqMPF%@x3`ATu5-0G`(HX%iAa7s8LPn&y zBl>(zE<%(Z#;i#F>S3A@3}i;H9TR>#1<-Yzm~bgIChV4&>2b4Z2u2v^D8M=&x0Mq2l>S5egtet4(xQu$QOM~Gr)x+`O?E3PX z;gnJyZNit-Ltz^%DUJpP8&qBYDkfE0&EsJcon`yQg35fpPuyGC$XmqAl~xmN1VrME z%0S`KK~%$j-_Ex8{wHGIYhtcQE20C~QqexerZZm?Kbdj|zex;FS&@T9>&K&gHtmPB zT)Zhv`IXpI^^oyCluQzrs>Ye8D2y@O*s6rhRmrc4MXs#UC2+5Vik{OP~5651FHX3L=Dxu4sJK#Wlw=e&;mGyh0&f5 zi>|4z= z*dTw$70v_AYfx^7JOZ2!7xK)_H(@g!hrJ8lCVe!&b`0gWRJV+|E5K?03prT}2hKpN zfJsi~b7vx>-y(_RuI;^etp2)H`9ICTahpy@Rde zchJH)KMxoMb2_X$K~D>xHV8rYb|<5R(+J>* zDa)6O((5SQrq4%zI3;7P&f@f=d7<-{u&_SYN!rqY%>+hgd3c2sjMqPh(@~ozoQmg0 zEGX48y|a}WoM%hvv9Z7jr!*I$tyh!toy#HT4XAhd!5H6we zTb1BznCn*CzO;Qkw*z#Flt6Z(HP_=Rg*S-N2j@pb`i$w0*P+mi#Yz>oVr4{HOLx>k zaX&3qDEqs(cg8xsLD?-LGjjAs;rA9}vSXN%H?lDbcE*sNs~pxMszSD~~` z3QD?x>h&5n%6X13A27%ffM1Sk1JTSchMaf*wZ(>g6<$R%*BGwoRy!f3|~)ud6|?u zG@q=(I%P?)3WxUvv3zDD|GC&Vv)=sH6)Ka0v&2PUE>StFYSxFe%Oyo)cZOmhom`o;^Kh1hvN*NKZHM zK?z8t8e4E`fi!f%h%&<56*9b`|#t@fVy5Ziuzu`@z?OORuQv zi{RT|Q&qeiU@wAC)#Iy?{Wq%W1+X1_E%?}ws{R^WI-;rujP{;4RP`S4w~wMD@X#?; zJq_-CPgOV49`+|yU5$NBdLIJ-?*M;>_BmV)E`VPF*FG`*8HQni2XJqe!21SN_4Z_B zEQ3%WPOL4iOT&vlypO`ioa8YUu!*4CfmINOM3@9x3yi*mQL=+tK+lWHb;Xm8q6D^S zDI32#K_394mdul3*`s@bQx7kKj*IqnHM;B`@ym5Ze7pE~T?xM+jO%N36^F!(_2X+d zqD}`qx*HyziGiSfAh{tU@Rp%*&y}VZE;luZFPnB>YHEC{uj%P8nr8X`(e%v4re`lS zJ@km|v&C$$m3joPm#^&Tw`wPa@+W@d?qtj{uzMeZIk zxW0JweB@sJC7h3PN;5%TV5VP*>J1IXmyq>}hc`5u`;c>CI>(B4HcXxHWXeaUPks33 z58gld-k(1Hizh>t4W10n5@BiwUWZmuy3xfS5_fGJWqc8sNo?9^Gro=7X|ZRc z-$+v)7SlFuFou!aE)H+nFlh;h{c<~JLv=DhTi>V zPN>BNU>AVJzg^O2N(TR)6Yp;>F-}BorQmH@Q^@BXmsC~z5*V91YQF^dGr-4EJ0$W8 z1)!m;XLg%T7J;_V^7yBr_?Mux60Iu3bZQBEgO=hleYT>M7G=z2jb`W*B_-GWQBGeL zDJ>LT*KOP@-nm}S-xlT}8^<@`q4La)@eeW4YM7vVhisWIk=aNHb}g+I&klW;Tf}*^ z+bGP#KA%0--?zlE3w)~v_^^(@O#{&Xw`#yR3XA{G11rTrwFX}=dp=XY;=I-$;oPU| zOY+~X>Ih;kH5ugWs7Np_w0<)0f>zP0TX1^zQ#TchK2>?|XmVdG7yzp7WgN zJm)zd_lmzU%-B;o(9T^g)i%A&&lBowk0VS@&agTaBgM{9<41(3Jk&Smd} z^`CJQp7L;~;=UR8SlpSob8ypT#%;x&k9!Di>d_?JdfXFn8*q=p9glkgiwoZ)=-4OW zV^hWhlMd+$xclJN>1z1zB$g2|QYd1xBeI1KRuhpV{LFtn>>V)C>c!L{!h8wKrza!X%=<3J^?tS2ahnC&j zRNmLAXD$5_Tmr(_hG-gj7%{9%mh{uY#UQRsUJ^yab`6-W-sGe~#ucX1WwHF&F|0b)Ft9P0lEE_F)J9EdD|P94ou>)X# z1#1dn)7Vw8nf(#p5OyTX>W}z>%?)8Q*aEPt!Ac?QXyygm3U*!yo5i+)Jqxxpgw1B> zzz$A8d_!0(OG!X{!LABn$Fdn*sURK9yKgiEO?;8$^_brGf+ z73k!6A7NS>13r$o6UN-q1sXYSAxsN-pn>Ce2}cmF=lBg55wxHNsyXorVaohKImgcv zrg<4C;dm`!Ol)1CnB%7iM-#Sj`~=~?gmXB4jBpHLGsjB_W1-Onj2ye}CnA;z9VZqN z772G>1Ce$oVOpdDevWS^97nj5&Yq2me*31<`T{*}t_BuuA>fS=>r3DZd;(8=+1 z!gTrw_&AgF5!~v z1{T-jh?2{E!%KurHq?^hTItp4obu_5uVPy@=-O$PD#>tP-sSO$eCc+|-zs~5QPp;u zV9L|zt0bSY%e$0YVbv*LlFxgK!;}%o0*|k9q4iSBILth%bO@@TX0 zp>VV#8mcmj`|dgum;$7gL-`nd3XB6|^|jIZqQ&n|nt9JHRB;(9)+&PN&H_S@UPBes zdmODsw#hApQX{c=%=(?A(f38VRDT?J?go*>N~h6d9z-98d3Cav2;>jU})E6 zRc)q(M`$Ja`->a30eT!3!O#GWy`l}!DH@=}pi^(5271;>F@%+NsG||jdeD5Jh&=v_ zn#0)|V$Jx0tIN1%<--`PBI`LN2~hG@Sc=2jE4z2(&_V0Dp6I!Y0=G+WHh*;yAh;g=*|)rK6-)iHt3x>R=3<@ z!0;*k(%8eJlZByd_2}W^)gM*0jW%x@!tNhg+vviX&o=3FKQ104rny1S&Qb<<>YcKGTJ26@w@x=L?{MXXUblLED)Mwyp+}!8;b`ApdrhqX`dPWuP3S5| z=M@D=s@X~1bjUr_dHVwUUG}?oSoyr3*TU!Zzoz?hIyVH6U$zE0QW)M`&6gbZK*&U>*iYKIpw|6yfJZ- zr>jC_W^1C$=#b~aK>PPt9-v@wqB$c!AZZ@W=N>s_*jApx>Jx0GQik4q?pBzY+*$^= ztg5_Sl0R1#-p~_03s4|=9hTo)wb?GoyCnoWUHR-hthvZ(B&X=P8G5y?d_eW0C!N=7 zwa`0bn;iqBJQ!ls9mR{?mJG}ZUM$Ib$f{A98mjgsU&LE2_mf%;SxFBq-f>O6gD(j3{b>j^sQKt7s0E4VG+gZQn+Cfm>l1eC zb-8cj|(FKzhYZ-~cV@(5M^7!oXc5#(}^Ekz_yqV6? zPFZcCvlO=8rpF07Ear=^A_Ea(r|fcCOxoCrFclnIa2_97G>e|UBh|2wIpiN`oh(71 z$wIEw@rBt5!p~c9T5_bQea&?0=Eg3*N$rGYF69drLJcIOS`KYacd(>)p=Lx(&S_QBHXVfjm16;i!AjG7q)wHNViuBZoZS z;uN2x(+l+GqCgOzY~y;Z7ANV|ss{1N&0y->mSPZa%~0DDEk3z{xYd@k5H+sL)G^E( zNqn*fgj82yF}B$8L}9j3%nhhZqn%5k%~6gAJC{S_o~t=*tCZlx*Qgc#$Rx=-;CI!$ zRU)7yIqKl6bWivblF+I8wTZt^4Rj-mU2rq9vTrT6lStA13M$Y-MsI{xUo*W!aEGMp z5yO6XxRI=y=tSny zDQ||3fcfX6G*anB1J%Io@DJ|w5j{&#nOrpB=s}OIJgs!%)5SnFnE!C}SsZP}m4zTE z7Cc)-cMhOKJ{q-G^rZ2}Pwn1>zBrQcBt}$SmK-hpcVox;Kk}fP)&@A6PDulQbxOgR zP(>_C!OYqn)HL(l;-^h&giqAq^0^&b`t>%9yr#=y5b+t z_N|d>%FJ(>Kkp&^)o`_Lo73}2-B98rL6e|os&?WZ>msf_%1Ul2aXC+O-&d8vN%7wf%al!m z=%Kl5{a}+x{7Z);>WfPb%S@2}23aJZwMXq2J+WBv@Mb|X!N8{hIy@Pc@^)1u;*k2h z;`s`Lh&RA`tlF8VsH3j&bGq^vnlJL4tIVBNqv}}U#L=!-zD8F(KG#VEX`;Nn_Df{J z)2Jk2u`-MA?Ud~as35x664#6)J3b?8AvN!@g@@6zES13@9NnDqcR2AmGjIUgY?+ND zk-X#jD&=T%x;K3E;u~XLe|?ut!HG?&yg)+)doeJFQ7x@1@_a42+XZ&IV08cB$yd4O zOWrNfz497soj5SB3L}r6rBkGDX`YOlJR0^^>ar3CZ63VJp~;%c>33A)NYwN)lv#dscKtjuf~R%Uy! zy1nd7dvC!~6f8suoiHWYUKWV%^~KsXL%!M@Ww|(9+G8uDadF61IQL@~P+<7tr(D80 zaR!ZG7Ah0XN9l9rbMS$I;1;v3la`K%55Bw|#L`Vi@m%bm2AwhpVWP9%zd4i@pJH03`9Sp#QVKiAj=V3m)slk~oeU$Xkz(OrDt30hkOjrI{>f*Sl4 z5)NO1T{JS09C1LW#Yi7V+xlRfHrTX#?St0Pa1fVdiHY*y^nGw>LNH4_O3En^$aJS9BH=7MdbEOE%HavsgVqgBI+ zq`5%vG{=KvERmF`MD7ZbmvKntO+)IGf1YH|Oim$Z$%-Ew|8!y3np|>~>Yn5N%<#j?CEm{}F$0Ig8ixZM}3qN%%mc$45qST7gHd=|C^3RLKvJT=q!RM8+ z{96ZQ{vK6;#zKoq{CJk|j5PU(D6woi(F*7(Wvu4b{`qFD!mYsyJt!&EGVXy`CMsHPMBlX*E07z#rJ&Ne7`^?i+QaZ?Ik`>QR*~_~1htHfW$YTx(_* zQIbYo{-f5CKUm|bU=7+*tvf+L%#Ls>#>tW9;pz z&qN>9I{0m z;nSwGl2s7yXG5l&WAE2;!AZmM?h)pkKEjcV^-(_MIZ+j%WFO(J=6Fx)a4Vj-YD8up(X=~+!oQycL7%u2^ z(U=ki`xoA51BQ)<(K&e!Egg1x$Hs^6J<1;sv*9z#@PGC7GxO8BG*ev&vX)?~^32C* zi>hn`N$)E6vhyJ`~Y>5A)2<3i9R~{B!rPU9(}K zY5eUrK{(K~Y|hL`!QHg$&X=Z-#os>QOHC|3*LGRe-mh`?I}m{LBKDfOcg(>-*03j{ z28uklHayDKzdAw4V+UVdIWU(d>Urhif2u0x%9(dDL9j|@y`IY+cx{ak+oZlG7~XBH zkBHBGLRHVzsOkpXoi0^A5yCEwejThL6=26JB+0A2x2IgrVKG1Z(s!|iZ7wz=)^|OR^H1`+gvqxK| zvcW9|p^T-qj1;!887-srrV930%S?TG1>4s$CZY=CdW2nVvFLptvf=NiB^p+!>QJ)h zh&87}^kC2}E7)D{k4h{>(G)7eCE=z#*tmkd^!{|c!NtCNe^kVNm#Tis2DYZ_{Q?uV z4TzO+Kyi{jhJ1c4TiiNJ@Ur(>Q}KJG^=^T&o3~9D@TVo)iZj-!>Lk*{Mct^z*l!Nu zO2DlHcXS=QvhALX^{R>_;1_AfP-cKbnD~{6Uo*Ig>)Fb-Y}d2@t-S~(N5RooBf7{! z6}Yd#VRG?{%2tE(gF8bLY-&`PA?%c}1COM?31wT5i+{i~UBtD5Ga@hV=8VR5f?GTv zB=$~zk>b}Nw}P7+!qF@XM-qqX#UOD@z?FvRnZdP&a1NA>1DF0Hd%k_DsS#nMd4|h| zyB;O`n^bihEAS0xgMBH&c9!qE6)V`|z88eAS#n3A@D01OqtJy-dKlHhMJ`Ins|s-R zU5_r}=7TH$4Ob4X>^GbnTq(FDUQ?^PA!sj@+YGKcShs21fSM>nmR7^ZUjOiJ{GUtI zezS3tMhy(w+Q9Pmry~9TXMdq!X3yby7Hi(09oKVGQ$SAgYaYA2KQ+!sZsWo{`gg$C;79boI(v zOgNMsSKd!!d*$QIekePtJIKQPp$SnX!N`@c=0g*%JJRdWJSH4=p&K=avsw3HBl8{3 ziPu>)tG)6iH`9NT9Tf~f!zPyh$)%{?KYc*IFzpC~9#Bgh^+S z4LxqPee6#Fe%DtE`A6#`8N%sJ}cTwdpehHXZukvMkiV zOQ}YV?=N*4NIKkGw!AAB$p3Hc;y~?UiU@g1E}%srgA#8+)kNH*15M|`&Nnx-9d*#vr@+a z@x@xKE#vNkTas$TZ$6C-8zJY@tguY7><^{7uvqyt)r7^%i|E6!OxZ-o@VrYfaTL>M ziu_0D#>;C3vrqgU6~D{)ml`7zXm#I&6@3%r_XUyFH;byn6XZKYwbk@sWTrfupQ$S% zL5`%eeKTdJklifUBIrIX@JjJJL>RDDbn&>*=z&ItkZFQd1ZziY9RkZj_LS(oDI!ar zD{vRdk(pEmEw_*~_DvwmzEIj5F$S&iAm1b#Y^xMfbDvOoD1B&(rPP}iv@E~r4ig>e z*Wbm#j$6V1YVi031Z^pm8%k}trmT>c{5+eg|Jz%2rYN&5o0QQ%OiCxAZ^I7B)b zz!gc4L~yh&0Q1$Gl^k7(Jkf(9syQieHPiCw_?Y#9tQB&wQ|MpOaBXyW%%{JQ)RL^k zuTa7h5Bx&V(Jb1NdMH(iPt<-PUHy?EJwvmUc=;kdqSVScbPCV;lx&Wt!RDxbmLrBv zUlb;q4%P)WD~JuFMPRGI+Jo3|dLQf>uth;^Bz+C`FxcWCHj4Vi#8XsERKKzyHk$Im z=7FsaV$HM=Y$@2rAhtif4)!&$O+jodeGax0Y;zDBM_0jK0qYB56DXs9JdNoe)vqgv zO``k2&I8*M#15tgu*<-n31WxP2C(g5rK2}|W}#DHzXxj$Vuw=l0Qh$R{2Rok(R8pb zuvtOu2wDWT3amYd9ZBzlT?4i#h#gH|gFOtkIEc-lezEXxEc_e9W>P-bJh0V4tc@0e zEd|>c#NI-$gMAHbQxH3zJ_p+gwmFEMKv%(D0qYB5b17pW{5uf-4Px_J9vt}BNGcnc zx55(Z_DGUiyES|Zdru$#%?A9Y9mlVdV#gEX1uXLS=(y4jGUln|?-ID1F|UPwpTN5q zhcIpycr#;Oy8TT8w=oW5+$iu`Hxs;o`l|);E@RGnf0@8}ZW6eSaT?>stLR@(Tg$|7 zCaQ(PyNpLLE))1o#_5cU1%8F`NXA71uVBnuwcjrA9~kqYz@H`XV#d76`z-=L$2fzr z`6~L?)1G8vEEAGYc$9G_<1<%S{t#n6H~4!5zK1cNFZ^8s&t}Z$3%^g`X^i=F;%^rC zHpYB%@HYvZ%b3q3{>IUQxS5GtnWz?c6yphu%dR&tzmYBuTOgOxgVtpC_6A9Ec~6{s zyFZ5mAVZu9t*rMy78y#LX{QF$XzCT%yyrFxG#yxX)|kS^~9@9Bo35N$pp z?(tPTWV_Hd4zr9a4KS#g9nEN!iJJ|gpo3?D_q5i{QA4G%PJpM;@O&l3DM_t-WU_VJ z(6Uy9KD=8zzGt7*Dt^}W+U(Hs9_=J7QS0vyR~yD3TZTvPQ7r+rVQ}3SXc_dL04)dT zPJ`|Y(BGj|*4`;mRa&z97GYOqW_b=;nhcp?2ZqA#*a&avnp?EwKIxwfWXPeM2xb2` zFg9NYuO){6T8!pB6S?BwQL$E&mAVWdjOYZ0ftp_+uTdNT3bpCl8#H)$f}Bm`hiA(V z&|||B+;^iMk6c;$9n4c3GXDKXc(LIlYZs3N4TW9S^;S-9xK{Q4aPBQ5Kpkii3{lIL|{p`4HvNPvY#Ua}T_$c`@9UC!N{wrM@F-X3hlF|ps|Djvc zQ{_UMo1SbMahaY;&y+9Fs`MehU3C#{BZZ$G;Hb^BhL(Sejs{#pN3X#RqGykK%C(3`+U18SGIOY0 zO&OU916Q3D9>F+!?!C`3Z;KP_6wy7IquiClWSIY&wHT)mwPB653^R6z)2g!7SaR^_ z;zw6QmGvQ4!&}xa4pYdFy57bwBen}XkLV|yJL;_4`59O3 zrqWIRkaOyL>pA4}y-sVfHjP5E^5qh8WM#O^*pi%W6@Im0rF9<6X>@serq*ti(ZJN4 zy%lGB&FZzO(9>Os5q+sS(Z8einqCg{lXj~>R|Pt)rL%1{TX~pH??oQG<37jzjt92b z#JpD9#Js-kif$FjKNgOz$M749^cqY7M0v>|P*`VN80Z>`vBl+tS~U}9g{wZwiXhK* zU|Kn9KgYcr_b%PlP-iWH7whL(RhReEsSQyvs;9d=1{+ebEzS)aW38HW8mmKntks3i z!u^}E_8IoLggq{AlvQiMN@Da;Q4cMbFe${sCRM^Do9^ALdXH%Hud68@KJ%#!PU{q# zG0|1;b`^e3&{EGBZK)nNW_T{tYTHF9O^RoXsMKyffUtBp5E`xgMw4~XLRx1{#AFas z)w_e+4A*7^X=!5dZnxgUS~c88R*-a8Kzaz$X@5Keymx9+IPagpj;Rr zs8ygG-tAg`!0U(x$B1^`JlM^iR^f?SpS;~9xpI6JXE_y=x)JEiQF|3n$#}AL-`M%t zyDkjDtZfUpg`;5h_#EEN%F>lc2=-xdD<6|xUcHTvO4w~BH$$h@a_(&;9bD`3x~y)q z25bM6v zeO;Fm3KCO5`%^sMRA*IDU-5hb0J#%DR+jMI?5uyyng{5p{TUNto;6Qd!h5r`zFlkt zj#@oZS;FgV{W9xe6g8K45O7xZGf$}X+%}eC{o^ zew@`fT7AQ=g=ftig@m6Dhv)y(>{L9%Q6i+?T{u~Z)D-C`(ooZ}#oow-fJYV2IiAZ1 zKmw-Ef{o?A@6kAo{3?lQBx?!^*rxT^r2!&BjkXB?jVdjvcwLl~+o zNfqcckPgHLdZYSKQsp^u zPMIWZ5hp%lLKL5)4<pqezu(wQ&=2^G2kE3Y|ggwIGhn*(F%_vba(-4#2PvV z$`TJv9Fw(N=oNZSRb)@C(IwLJMlXEm-x!L==l&W_%JXin{l=|$`k<1^sjsZt>UG4T zg5k*8D(e^omY3i*>`2o3KKSup#*g;ix1vu)?6zVoy`4LFX^=3oI!ULLg297+4OKC$ ziliHcmB0-D_jGr`Fw?8Y>30Q5iLWED965{ag2UVC*ubseL^-#TJ}4OZtINGoZ&8B( z6hK=D zH$Qu-MN8iH`%ug^UI|&|@?J)grfZ)-A?%=m_?Iz>^zY6^mv{rDOitLGn-{8m<{QCPX z2-Zo#8?m;Z1IVwxF+k!q?kzAZmRLOE7)IDIu)K^uu&0^* z_wO;3p9-+yf^9!)u=j!0UcheO`wDx(=G%zj;te*?S_w2Z4hIv^5N}>E_%hl>CwXj$ z6FO~X{(jie2maMEkCOYgxbVk#tJI!2Mz7x%pLm2v_EVd=SBZxR>9Hg9xgkYk;i9_G z!9fqw50B8*+lH4K#x9+KsRAu?2dAkP8ep@*c5qakUR^tlZeY>UVN7zi^Q*wY02$3B z=NpsT0^}upR~22Ol(c^wp%-pXW@EXP9&qA~-(ht5>mJ&6``9UvI%>Deg*n$Mj^Uke z!a`nb&MChSYkC=bKfYQNqv!dm^4O=4+92)V6~yKJX}(griTN(@ISVP*IcUrSr~-5g zv{=%EXBE##lP{Q3N;fmDfu6jOYMcXd(~SzZ2P#~IWRjt|rMJrhV1AeOVML-kw(a6e zwA&fyhG=Td*5z!VQo5B-9ofP*9>`6BdVGN6D(x@v)HL47Ar3XV*~Szf=s;a6HG1j^ z)L0Ry!Dk$s4?S#WJ9vNN_NrdOe=UOC!6YxyOqQVvxJ}HN16+y0ZD!5_PHQ<#uNJ0- zqc^4qkwf-Y(V#>?XB zLq^JgW_L*0cCJf&G|@rY6v@rp!ux6iqfAgs@c}+gY<#=LgZJ* zz_s6eMN_60Ozw+yj~yWrUAy!Hws~YRQvS5|J%A9wrIL1}7m3-_)r)v0X`AV@sTTP_ z%lA`lviw^ba)%}Qw?;0wI2>!gB-b6ood>bFd5?MyRfcJqUy7!)JUYNo0Cm&PmVj)4 zAsZ)TeE@L460ha>2)3Wr-7yK{yl}@n`AfRf^_JBaq|O)rr5(uf(8@%z|}2kv$oYf2q|?GS zB=SGM^WhtPpkZkr2It+uONRr>{5f)mcJDqKHnUXTLya?Y)Akrmb;HRC8xwQN5&nMzf1^BDU6aUb4%$1XC`M~OG$ap7*7Oq+@D=qpeIs9*B1Ld{N zjbZ&4*XVj4?w{PcelPA}H7##`{G;s7D%Euhf^SE85eZ#fuIrzoui^?_&jw$H(uJ}M zw61SujPeZ1tY>uHj!bVxS%%VFt?NFNc9eUWM=5dcqwiRh&5L!N|9YYcxw>3u%euY>W!Ik(Mr<(VS9EvbrbQNG+4FPP>x&=H7gls=R_DEFc)dJ_$! z^r5_h(*8GHH}jgeUf1WMw11-OrHps!dNY?;FwUdQ!n<-KjwL-PyHJ*4;V?&H#3-#O zn^9(?^r3VO#6uSzicy|HS&q_thK9F}k{fAO>n-^8v<^!Q$0>}v=)I5~(mdQ2+SEG0 zH25icu5B_6ZVQ*kP+HpvxtylAjW!)%dWK1RigvY)4a-Jg=h4+RtEqVh4f`Z5POZ}Q zRBq52ZAnLQ5m7IxqWeD?6}K9q$t=R9;^ss+UPXWTWSU9+J$?7dsIVxc(2JDVo^JA) zNZt@1g-Jh*-kYZibj&I7qBKS9z``mzb1$) z2FKrs4iH`5@bJh#={j;iT&x|%i2)8?64w~~TEOM~iJtGsbieq&+9i-20GANZ<`yc! z9S5fbIF>B~*8`6KN<*3w5fUD9RNjI|)?W?TM@Yp7@yr)&6_e*_#>BJj`gVnOgTwpys`1 z(~(*-?aPb^47_O- 0 && r.worker_uid == 2200 && + r.start_ticks > 0 && r.output_length == 0 && + r.diagnostic_length > 0 && + r.diagnostic_length <= DBL_MAX_DIAGNOSTIC, + "worker failure diagnostic length"); + char *reason = calloc(1, r.diagnostic_length + 1); + check(read_all(s, reason, r.diagnostic_length) && + strstr(reason, "grok: session store unwritable"), + "worker failure reason"); + char extra; + check(read(s, &extra, 1) == 0, "worker failure EOF"); + free(reason); + close(s); + close(p); + close(c); +} static void org_cases(void) { check(!setgid(DBL_BROKER_UID) && !setuid(DBL_BROKER_UID), "drop broker uid"); client_case(); @@ -90,6 +118,7 @@ static void org_cases(void) { client_input_reject(5, 1); output_boundary_case("exact-output", 0); output_boundary_case("overflow-output", 1); + worker_failure_case(); int s = connect_socket(), p = sealed("prompt"), c = sealed_bundle("provider.Token-1", "mcp.Token-2"); struct dbl_request q = request(); diff --git a/src/runtime/native/engineBrokerLauncherIntegrationMain.inc b/src/runtime/native/engineBrokerLauncherIntegrationMain.inc index f773c55..ef59692 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationMain.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationMain.inc @@ -114,7 +114,8 @@ int main(void) { DBL_RESULT_FAILURE_CLASS_OFFSET && offsetof(struct dbl_result, profile_applied) == DBL_RESULT_PROFILE_APPLIED_OFFSET && - offsetof(struct dbl_result, reserved) == DBL_RESULT_RESERVED_OFFSET, + offsetof(struct dbl_result, diagnostic_length) == + DBL_RESULT_DIAGNOSTIC_LENGTH_OFFSET, "result ABI"); mkdir("/run/daimon-engine-broker", 0755); mkdir("/etc/daimon-engine-broker", 0755); diff --git a/src/runtime/native/engineBrokerLauncherModes.inc b/src/runtime/native/engineBrokerLauncherModes.inc index ac78544..012da6b 100644 --- a/src/runtime/native/engineBrokerLauncherModes.inc +++ b/src/runtime/native/engineBrokerLauncherModes.inc @@ -2,7 +2,7 @@ static int client_mode(void) { struct dbl_request request; uint32_t prompt_length = 0, capability_length = 0; unsigned char *prompt = NULL, capability[DBL_MAX_CAPABILITY_BUNDLE] = {0}, - output[DBL_MAX_OUTPUT] = {0}, extra; + output[DBL_MAX_OUTPUT + DBL_MAX_DIAGNOSTIC] = {0}, extra; struct dbl_result result; struct sockaddr_un a = {.sun_family = AF_UNIX}; int s = -1, p = -1, c = -1, ok = -1; @@ -37,7 +37,8 @@ static int client_mode(void) { 1) || client_send(s, &request, p, c) || full_read(s, &result, sizeof(result)) || !closed_result(&result, request.turn_id) || - full_read(s, output, result.output_length) || read(s, &extra, 1) != 0) { + full_read(s, output, result.output_length + result.diagnostic_length) || + read(s, &extra, 1) != 0) { struct dbl_result failure; memset(&failure, 0, sizeof(failure)); failure.version = DBL_VERSION; @@ -50,8 +51,11 @@ static int client_mode(void) { ok = 0; goto done; } + /* The trailer is the turn's output on success and the worker's bounded + diagnostic tail on failure; `closed_result` keeps the two exclusive. */ if (full_write(STDOUT_FILENO, &result, sizeof(result)) || - full_write(STDOUT_FILENO, output, result.output_length)) + full_write(STDOUT_FILENO, output, + result.output_length + result.diagnostic_length)) goto done; ok = 0; done: diff --git a/src/runtime/native/engineBrokerLauncherServer.inc b/src/runtime/native/engineBrokerLauncherServer.inc index 75f452b..1d4cd76 100644 --- a/src/runtime/native/engineBrokerLauncherServer.inc +++ b/src/runtime/native/engineBrokerLauncherServer.inc @@ -225,12 +225,25 @@ static void supervise(int client, pid_t pid, int output, out->output_length = (uint32_t)used; } if (out->status != DBL_STATUS_OK) { + /* A failed turn publishes no output, but a worker that exited on its own + account said why on the pipe it shares with stdout, and that tail is the + only reason the host can ever see: without it a failure reads `exit=1`. + Keep a bounded tail of it and erase the rest here; the broker redacts it + before it crosses any boundary. The other failures get none: an + output-limit tail is the very payload the bound refused to publish, a + cancelled turn has no reader left, and a prelaunch failure ran nothing. */ + size_t keep = out->status != DBL_STATUS_WORKER_FAILED ? 0 + : used > DBL_MAX_DIAGNOSTIC ? DBL_MAX_DIAGNOSTIC + : used; + memmove(bytes, bytes + (used - keep), keep); + erase(bytes + keep, sizeof(bytes) - keep); out->output_length = 0; - erase(bytes, sizeof(bytes)); + out->diagnostic_length = (uint32_t)keep; + used = keep; } if (!disconnected) { full_write(client, out, sizeof(*out)); - if (out->status == DBL_STATUS_OK) + if (used) full_write(client, bytes, used); } erase(bytes, sizeof(bytes)); @@ -334,7 +347,8 @@ static int client_send(int socket_fd, const struct dbl_request *r, int prompt, return sendmsg(socket_fd, &m, MSG_NOSIGNAL) == (ssize_t)sizeof(*r) ? 0 : -1; } static int closed_result(const struct dbl_result *r, const char turn_id[65]) { - if (r->version != DBL_VERSION || r->reserved || r->profile_applied > 1 || + if (r->version != DBL_VERSION || r->diagnostic_length > DBL_MAX_DIAGNOSTIC || + r->profile_applied > 1 || r->stage > DBL_STAGE_ATTESTATION || r->failure_class > DBL_FAILURE_ATTESTATION_PROFILE_INVALID || r->output_length > DBL_MAX_OUTPUT || memcmp(r->turn_id, turn_id, 65)) @@ -343,12 +357,14 @@ static int closed_result(const struct dbl_result *r, const char turn_id[65]) { return r->stage == DBL_STAGE_OUTPUT && r->failure_class == DBL_FAILURE_NONE && r->profile_applied == 0 && r->worker_pid > 0 && r->worker_uid >= 2200 && r->start_ticks && - r->exit_code == 0 && r->term_signal == 0; + r->exit_code == 0 && r->term_signal == 0 && + r->diagnostic_length == 0; if (r->status == DBL_STATUS_PRELAUNCH_FAILED) return r->stage >= DBL_STAGE_PEER && r->stage <= DBL_STAGE_EXEC && r->failure_class >= DBL_FAILURE_PEER && r->failure_class <= DBL_FAILURE_EXEC && r->worker_pid == 0 && - r->worker_uid == 0 && r->start_ticks == 0 && r->output_length == 0; + r->worker_uid == 0 && r->start_ticks == 0 && r->output_length == 0 && + r->diagnostic_length == 0; if (r->status == DBL_STATUS_WORKER_FAILED) return r->stage == DBL_STAGE_WAIT && (r->failure_class == DBL_FAILURE_EXEC || @@ -358,10 +374,12 @@ static int closed_result(const struct dbl_result *r, const char turn_id[65]) { if (r->status == DBL_STATUS_OUTPUT_FAILED) return r->stage == DBL_STAGE_OUTPUT && r->failure_class == DBL_FAILURE_OUTPUT_LIMIT && r->worker_pid > 0 && - r->worker_uid >= 2200 && r->start_ticks && r->output_length == 0; + r->worker_uid >= 2200 && r->start_ticks && r->output_length == 0 && + r->diagnostic_length == 0; if (r->status == DBL_STATUS_CANCELLED) return r->stage == DBL_STAGE_WAIT && r->failure_class == DBL_FAILURE_CANCELLED && r->worker_pid > 0 && - r->worker_uid >= 2200 && r->start_ticks && r->output_length == 0; + r->worker_uid >= 2200 && r->start_ticks && r->output_length == 0 && + r->diagnostic_length == 0; return 0; } diff --git a/src/runtime/native/fixtureWorker.c b/src/runtime/native/fixtureWorker.c index cb92d4e..d2f3674 100644 --- a/src/runtime/native/fixtureWorker.c +++ b/src/runtime/native/fixtureWorker.c @@ -6,4 +6,4 @@ #include #include #include "engineBrokerLauncher.h" -int main(int argc,char**argv){unsigned char bundle[8200]={0};ssize_t size=pread(4,bundle,sizeof(bundle),0);if(size<6)return 21;size_t offset=0;unsigned provider_length=bundle[0]|(bundle[1]<<8);offset=2;if(offset+provider_length+2>(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output");if(!strcmp(provider,"sleep"))sleep(30);if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;id_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s tmpdir=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target,getenv("TMPDIR")?getenv("TMPDIR"):"unset");for(int i=0;i(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output");if(!strcmp(provider,"sleep"))sleep(30);if(!strcmp(provider,"stderr-failure")){fprintf(stderr,"grok: session store unwritable\n");return 1;}if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;id_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s tmpdir=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target,getenv("TMPDIR")?getenv("TMPDIR"):"unset");for(int i=0;i Date: Thu, 17 Sep 2026 23:33:39 +0200 Subject: [PATCH 081/124] fix: stop naming a healthy turn's own two proxy requests as refusals --- src/runtime/AGENTS.md | 15 +++++++++++++-- src/runtime/grokBrokerProxy.test.ts | 17 +++++++++++++++++ src/runtime/grokBrokerProxy.ts | 29 ++++++++++++++++++++++++----- 3 files changed, 54 insertions(+), 7 deletions(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 0863d31..7487b5d 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -163,8 +163,19 @@ refuses a turn whose worker config does not hash to the declared one. Three capability reaches the model through `env_key = "DAIMON_PROVIDER_CAPABILITY"` set by the native launcher (as exposed as `DAIMON_MCP_CAPABILITY`); the per-turn `session_title` request cannot be disabled by any key, so -`[models] session_summary` points it at a hidden model on closed loopback port -9; and effort is only sent when the model declares it, so the declared effort is +`[models] session_summary` points it at a hidden model +(`GROK_SESSION_TITLE_SINK_MODEL_ID`) whose `base_url` is the broker's own +provider proxy and whose `api_key` is a placeholder too short to ever be a turn +capability — so the request does reach the proxy and is refused there, before +any capability lookup, isolation guard, credential read or upstream call, and +Grok falls back to the truncated prompt as the title. That refusal and a bare +unauthenticated `GET /` probe are the two requests a healthy turn always makes +and the proxy never forwards; neither prints a `refused:` line, because for as +long as they did, every healthy turn read as broken. The sink keeps its 503 +shape because every live capture was taken with it: forcing 400 and 503 there +were both observed to end the turn `exit=0, result: success`, so a hard 4xx on +that request does *not* end Grok's session. And effort is only sent when the +model declares it, so the declared effort is the model's single `reasoning_efforts` entry. HTTP MCP needs CA certificates in the image even for a loopback `http://` URL ("Failed to build HTTP client"). diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index 79e8685..05caadf 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -95,3 +95,20 @@ test("the isolation guard is awaited before the first upstream call, and a faili assert.deepEqual(order, ["guard-start", "guard-end", "credential", "upstream"]); } finally { await proxy.close(); } }); + +test("the two requests every healthy turn makes are not named as refusals", async () => { + const lines: string[] = []; const original = process.stderr.write.bind(process.stderr); + process.stderr.write = ((chunk: string | Uint8Array, ...rest: unknown[]) => { lines.push(String(chunk)); return original(chunk as string, ...rest as []); }) as typeof process.stderr.write; + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => ({ status: 200, headers: { "content-type": "application/json" }, body: Buffer.from("{}") })); + try { + arm(proxy, async () => undefined); + const title = await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`, { method: "POST", headers: { authorization: `Bearer ${GROK_SESSION_TITLE_SINK_KEY}`, "content-type": "application/json" }, body: leanBody() }); + assert.equal(title.status, 503, "the title sink keeps its transient shape"); + const probe = await fetch(`http://127.0.0.1:${proxy.port}/`); + assert.equal(probe.status, 400, "the unauthenticated probe keeps its non-retryable shape"); + assert.deepEqual(lines, [], "expected per-turn traffic must not read as a refusal on the broker's stderr"); + const miss = await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`, { method: "POST", headers: { authorization: `Bearer ${"z".repeat(48)}`, "content-type": "application/json" }, body: leanBody() }); + assert.equal(miss.status, 400); + assert.deepEqual(lines, ["[grok-proxy] refused: unknown_capability\n"], "a genuine policy miss is still named with its reason code"); + } finally { process.stderr.write = original; await proxy.close(); } +}); diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index 1b89d58..e13d5ca 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -72,12 +72,15 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori } catch (error) { settle?.(undefined); // Name the refusal on the broker's own stderr (reason code only, never a body - // or a token) so a failing turn is diagnosable without a stub harness. + // or a token) so a failing turn is diagnosable without a stub harness — + // except for the two requests every healthy turn makes anyway. const refusal = error instanceof GrokBrokerProxyRefusal ? error.reason : "broker_unavailable"; - process.stderr.write(`[grok-proxy] refused: ${refusal}\n`); - // Grok's own session-title call is refused by design. It must keep the transient - // 503 shape it has always had: a hard 4xx on that internal request ends Grok's - // session, which surfaces as the worker exiting 1 mid-turn. + if (!titleSink && !expectedWorkerProbe(request)) process.stderr.write(`[grok-proxy] refused: ${refusal}\n`); + // Grok's own session-title call is refused by design, and keeps the transient + // 503 shape it has always had. Forcing 400 and 503 on it were both observed + // to end the turn `exit=0, result: success`, so the shape is kept because it + // is the one every live capture was taken with, not because a 4xx there ends + // Grok's session — it does not. if (titleSink) { response.writeHead(503, { "content-type": "application/json", "cache-control": "no-store" }); response.end('{"error":"broker unavailable"}'); @@ -93,6 +96,22 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori } } +/** + * The unauthenticated connectivity probe Grok sends before its own requests: a + * bare `GET /` with no Authorization header, which has no capability to look up + * and answers 400. + * + * It and the per-turn `session_title` POST are the only two requests a healthy + * turn makes that this proxy does not forward, and both used to print the same + * `refused: unknown_capability` line as a real policy miss — so every healthy + * turn read as two refusals and cost a live investigation. They answer exactly + * as before; they simply stop claiming a refusal on the broker's stderr, which + * is left for the misses that are actually worth reading. + */ +const expectedWorkerProbe = (request: IncomingMessage): boolean => + request.headers.authorization === undefined && (request.method ?? "") === "GET" && + new URL(request.url ?? "/", "http://127.0.0.1").pathname === "/"; + /** The body gate, refused non-retryably: a rejected body is a policy miss, never a transient fault. */ function authorizeRequestOrRefuse(...args: Parameters): ReturnType { try { return authorizeGrokBrokerProxyRequest(...args); } From 081b3cecd0ae8d2ee304a4ef73d2e12e5c7a0ce7 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 23:38:09 +0200 Subject: [PATCH 082/124] test: pin the no-active-turn refusal to its non-retryable shape --- src/runtime/grokBrokerTurnMeter.test.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/runtime/grokBrokerTurnMeter.test.ts b/src/runtime/grokBrokerTurnMeter.test.ts index 0f28a88..094ef65 100644 --- a/src/runtime/grokBrokerTurnMeter.test.ts +++ b/src/runtime/grokBrokerTurnMeter.test.ts @@ -85,13 +85,21 @@ test("a request after the elapsed deadline is refused, and every admitted reques assert.deepEqual(snapshot.timings, [{ startedAt: new Date(1_000_000).toISOString(), endedAt: new Date(1_000_000).toISOString(), usage: estimate, estimated: true }]); }); -test("a turn without a registered meter is never forwarded", async () => { +// A live capability with no registered meter means the turn is already over: the +// launcher registers a turn before it starts the worker, so nothing can arrive +// before the meter exists, and nothing can make a finished turn live again. The +// answer is therefore 400 (a named, non-retryable refusal) rather than the 503 it +// once was — a retryable shape here bought only Grok's blind retry storm, which +// spent ~141k tokens re-asking a question that could never start being answerable. +test("a turn without a registered meter is refused non-retryably and never forwarded", async () => { let calls = 0; const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; return { status: 200, headers: {}, body: Buffer.from("{}") }; }, undefined, 0); try { const token = proxy.capabilities.issue("agent", "turn"); proxy.registerIsolationGuard("turn", async () => undefined); - assert.equal((await post(proxy.port, token)).status, 503); + const answer = await post(proxy.port, token); + assert.equal(answer.status, 400); + assert.equal(JSON.parse(answer.text).reason, "no_active_turn"); assert.equal(calls, 0); } finally { await proxy.close(); } }); From 37363c398484c46a752afc595ed21afc5fea39fb Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 00:05:16 +0200 Subject: [PATCH 083/124] fix: state the Grok use_tool prefix rule once, from the worker contract both texts render --- src/contracts/grokWorkerContract.ts | 44 ++++++++++++++++++++- src/runtime/attentionDispatcher.test.ts | 35 +++++++++++++++-- src/runtime/attentionDispatcher.ts | 17 ++++++-- src/runtime/engineDispatcher.test.ts | 52 ++++++++++++++++++++++++- src/runtime/engineDispatcher.ts | 26 +++++++++---- 5 files changed, 157 insertions(+), 17 deletions(-) diff --git a/src/contracts/grokWorkerContract.ts b/src/contracts/grokWorkerContract.ts index c0cc666..fbe949d 100644 --- a/src/contracts/grokWorkerContract.ts +++ b/src/contracts/grokWorkerContract.ts @@ -14,15 +14,57 @@ * run directly and saves one `search_tool` round trip per tool (P0: 3 → 2 * requests). */ + +/** + * The atoms of that route, and its single definition. + * + * Both texts a Grok worker receives are rendered from them: the pinned system + * prompt below, and the caller's identity envelope + * ({@link grokMountedToolNamingRule}, used by `src/runtime/engineDispatcher.ts`). + * They were worded independently once - the envelope told the model to call the + * tools by their bare names - and a live turn produced zero tool calls with + * every tool correctly mounted: two authoritative naming rules, the wrong one + * last. One definition is what keeps them from diverging again. + */ +export const DAIMON_GROK_MCP_SERVER = "daimon" as const; +/** Grok's own name for an MCP tool of that server: exactly what `tool_name` must carry. */ +export const DAIMON_GROK_TOOL_PREFIX = `${DAIMON_GROK_MCP_SERVER}__` as const; +export const grokDaimonToolName = (tool: string): string => `${DAIMON_GROK_TOOL_PREFIX}${tool}`; +/** Grok's two MCP meta-tools, and the argument that names a tool for the first. */ +export const GROK_MCP_INVOKE_TOOL = "use_tool" as const; +export const GROK_MCP_SEARCH_TOOL = "search_tool" as const; +export const GROK_MCP_TOOL_NAME_ARGUMENT = "tool_name" as const; +/** Illustrative Daimon tools for the system prompt, which cannot know a wake's real mount. */ +const DAIMON_GROK_EXAMPLE_TOOLS = Object.freeze(["moltnet_read", "moltnet_send", "memory_search", "memory_register"] as const); + export const DAIMON_GROK_SYSTEM_PROMPT = [ "You are a headless Daimon agent; no human is present.", "Your identity, instructions and wake event are in the user prompt.", - "Daimon tools are MCP tools on server daimon: call a known one directly with use_tool (tool_name daimon__moltnet_read, daimon__moltnet_send, daimon__memory_search, daimon__memory_register, or another daimon__ name you were given); use search_tool only for a name you do not know.", + `Daimon tools are MCP tools on server ${DAIMON_GROK_MCP_SERVER}: call a known one directly with ${GROK_MCP_INVOKE_TOOL} (${GROK_MCP_TOOL_NAME_ARGUMENT} ${DAIMON_GROK_EXAMPLE_TOOLS.map(grokDaimonToolName).join(", ")}, or another ${DAIMON_GROK_TOOL_PREFIX} name you were given); use ${GROK_MCP_SEARCH_TOOL} only for a name you do not know.`, "If a tool result says output was saved to a file, read that path with read_file.", "If a tool fails, do not retry it in a loop: stop and report the failure.", "Your final answer is a private note to the runtime: one line, or empty." ].join(" "); +/** + * The same route, stated once for the caller's identity envelope, where a + * wake's real mounted tools are known. + * + * It asserts rather than corrects: it names the bare tool set once, states the + * `use_tool` prefix rule once with one example drawn from that set, and says + * plainly that a bare name is not callable here. It never claims the agent's + * own instructions spell a tool wrongly, never offers a shell or CLI route, and + * never repeats the tool list a second time in prefixed form. + */ +export const grokMountedToolNamingRule = (mountedToolNames: readonly string[]): string => { + const example = grokDaimonToolName(mountedToolNames[0] ?? DAIMON_GROK_EXAMPLE_TOOLS[0]); + return `Your mounted tools are exactly: ${mountedToolNames.join(", ")}. ` + + `On this engine each is an MCP tool on server ${DAIMON_GROK_MCP_SERVER}: invoke it with ${GROK_MCP_INVOKE_TOOL}, ` + + `${GROK_MCP_TOOL_NAME_ARGUMENT} = ${DAIMON_GROK_TOOL_PREFIX} (for example ${example}). ` + + "None appears in your direct tool list and none is callable by its bare name; " + + `${GROK_MCP_SEARCH_TOOL} lists them if a name is unknown.`; +}; + /** Closed declared-model vocabulary; defaults are `grok-4.6` at `low`. */ export const GROK_BROKER_MODELS = Object.freeze(["grok-4.6", "grok-4.5", "grok-build"] as const); export const GROK_BROKER_REASONING_EFFORTS = Object.freeze(["low", "medium", "high"] as const); diff --git a/src/runtime/attentionDispatcher.test.ts b/src/runtime/attentionDispatcher.test.ts index 71dce93..3e61593 100644 --- a/src/runtime/attentionDispatcher.test.ts +++ b/src/runtime/attentionDispatcher.test.ts @@ -1,5 +1,6 @@ import { randomUUID } from "node:crypto"; import { AttentionDispatcher } from "./attentionDispatcher.js"; +import { DAIMON_GROK_TOOL_PREFIX, grokDaimonToolName } from "../contracts/grokWorkerContract.js"; import { createOrganizationRuntimeHost } from "./organizationRuntimeHost.js"; import { WakeAcceptanceStore, WakeExecutionClaimLostError } from "./wakeAcceptanceStore.js"; import { parseWakeAcceptanceRequest } from "./wakeAcceptanceTypes.js"; @@ -14,7 +15,7 @@ import type { OrganizationRuntimeHost, OrganizationRuntimeWakeRequest, Organizat const token = "attention-test"; const storeOptions = { processIdentity: async () => ({ pid: 1, process_start: "test-start", boot_id: "test-boot", pid_namespace_dev: 1, pid_namespace_ino: 1 }), ownerLiveness: async () => true }; -const config = (maxExecutions = 20) => ({ version: "noopolis.daimon.organization-runtime.v1", host: { bindHost: "127.0.0.1", port: 4318, controlTokenEnv: "ATTENTION_TEST" }, agents: ["alpha", "beta"].map((id) => ({ id, name: id, instructions: "Act", workspacePath: `/workspace/${id}`, runtimeHomePath: `/home/${id}`, engine: { kind: "codex" }, attention: { maxBatchMessages: 3, maxExecutions } })) }); +const config = (maxExecutions = 20, engine: "codex" | "grok" = "codex") => ({ version: "noopolis.daimon.organization-runtime.v1", host: { bindHost: "127.0.0.1", port: 4318, controlTokenEnv: "ATTENTION_TEST" }, agents: ["alpha", "beta"].map((id) => ({ id, name: id, instructions: "Act", workspacePath: `/workspace/${id}`, runtimeHomePath: `/home/${id}`, engine: { kind: engine }, attention: { maxBatchMessages: 3, maxExecutions } })) }); const request = (id: string, agent_id = "alpha") => ({ token, agent_id, delivery_id: id, event: { version: "noopolis.daimon.wake.v2", kind: "message", text: `Handle ${id}`, occurred_at: "2026-09-11T00:00:00.000Z" } }); const pause = (ms = 5) => new Promise((resolve) => setTimeout(resolve, ms)); async function until(test: () => boolean | Promise): Promise { for (let n = 0; n < 200; n++) { if (await test()) return; await pause(); } throw new Error("expected side effect did not appear"); } @@ -29,13 +30,13 @@ class Core implements OrganizationRuntimeHost { async activity() { return { version: "noopolis.daimon.organization-runtime-activity.v1" as const, items: [] }; } async stop() { this.stops++; this.releases.forEach((release, index) => release({ version: "noopolis.daimon.wake-result.v1", status: "stopped", agentId: this.wakes[index]!.agentId, wakeId: this.wakes[index]!.event.id, code: "active_wake_aborted" })); return { version: "noopolis.daimon.organization-runtime-stop.v1" as const, state: "stopped" as const }; } } -async function fixture(limit = 20, maxWakes = 100, claimTtlMs = 240000) { +async function fixture(limit = 20, maxWakes = 100, claimTtlMs = 240000, engine: "codex" | "grok" = "codex") { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-attention-")); await chmod(root, 0o700); const usage = await mkdtemp(path.join(os.tmpdir(), "daimon-attention-usage-")); await writeFile(path.join(usage, "usage.jsonl"), ""); const registry: AttentionRegistry = new Map(); const core = new Core(); const options = { acceptanceStorePath: root, controlToken: token, storeOptions: { ...storeOptions, claimTtlMs }, attentionRegistryForTest: registry, fuseEnvironment: { DAIMON_WAKE_FUSE_DIRECTORY: usage, DAIMON_WAKE_FUSE_EPOCH: "attention", DAIMON_WAKE_FUSE_MAX_WAKES: String(maxWakes), DAIMON_WAKE_FUSE_MAX_TOKENS: "10000", DAIMON_TURN_USAGE_LEDGER_PATH: path.join(usage, "usage.jsonl") } }; - const control = createOrganizationRuntimeControlHostWithCoreForTest(config(limit), core, options); await control.start(); + const control = createOrganizationRuntimeControlHostWithCoreForTest(config(limit, engine), core, options); await control.start(); return { root, usage, registry, core, options, control, cleanup: async () => { await control.stop(); await rm(root, { recursive: true, force: true }); await rm(usage, { recursive: true, force: true }); } }; } @@ -214,3 +215,31 @@ test("an inbox turn leads with each delivery's own text and keeps the accounting assert.ok(text.indexOf("Machine-readable payload:") > accounting, "payload stays a trailing appendix"); } finally { await f.cleanup(); } }); + +/** + * `daimon_inbox_disposition` is the tool that records a finished wake as + * complete; an agent that cannot name it leaves its work recorded as deferred. + * On Grok the bare name reaches nothing, so the inbox prompt must name the + * `daimon__` form the engine can actually invoke. + */ +test("a Grok inbox turn names both inbox tools the way use_tool can call them", async () => { + const f = await fixture(20, 100, 240000, "grok"); + try { + await f.control.accept(request("g-1")); await until(() => f.core.wakes.length === 1); + const text = f.core.wakes[0]!.event.text!; + assert.ok(text.includes(grokDaimonToolName("daimon_inbox_disposition")), "disposition tool carries the daimon__ prefix"); + assert.ok(text.includes(grokDaimonToolName("daimon_inbox")), "inbox tool carries the daimon__ prefix"); + // No bare occurrence survives: every mention is the prefixed one. + assert.equal(text.split("daimon_inbox").length - 1, text.split(DAIMON_GROK_TOOL_PREFIX).length - 1); + } finally { await f.cleanup(); } +}); + +test("every other engine's inbox turn keeps the bare tool names", async () => { + const f = await fixture(); + try { + await f.control.accept(request("c-1")); await until(() => f.core.wakes.length === 1); + const text = f.core.wakes[0]!.event.text!; + assert.ok(text.includes("with daimon_inbox_disposition (complete)")); + assert.equal(text.includes(DAIMON_GROK_TOOL_PREFIX), false); + } finally { await f.cleanup(); } +}); diff --git a/src/runtime/attentionDispatcher.ts b/src/runtime/attentionDispatcher.ts index 208bd5d..5d842b5 100644 --- a/src/runtime/attentionDispatcher.ts +++ b/src/runtime/attentionDispatcher.ts @@ -6,6 +6,7 @@ import { WakeAcceptanceStore, WakeExecutionClaimLostError, type WakeExecutionCla import type { StoredWakeAcceptanceRecord } from "./wakeAcceptanceRecord.js"; import { WakeFuse } from "./wakeFuse.js"; import { ORGANIZATION_RUNTIME_MAX_STRING_CODEPOINTS, ORGANIZATION_RUNTIME_MAX_WAKE_TEXT_BYTES } from "../contracts/organizationRuntimeContract.js"; +import { grokDaimonToolName } from "../contracts/grokWorkerContract.js"; type Claimed = { record: StoredWakeAcceptanceRecord; claim: WakeExecutionClaim; done: boolean }; type Options = Readonly<{ store: WakeAcceptanceStore; host: OrganizationRuntimeHost; fuse: WakeFuse; agents: readonly OrganizationRuntimeAgentConfig[]; registry: AttentionRegistry; token: string | undefined; onIdle(agentId: string): void }>; @@ -111,7 +112,7 @@ export class AttentionDispatcher { result = await host.wake({ token, agentId: agent.id, event: { version: "noopolis.daimon.wake.v1", id: agent.attention === undefined ? first.delivery_id : executionId, kind: first.event.kind, occurredAt: first.event.occurred_at, - text: agent.attention === undefined ? first.event.text : inboxPrompt(messages, agent.attention.maxBatchBytes) + text: agent.attention === undefined ? first.event.text : inboxPrompt(messages, agent.engine.kind, agent.attention.maxBatchBytes) } }); } catch (error) { result = { version: "noopolis.daimon.wake-result.v1", status: "failed", agentId: agent.id, wakeId: executionId, code: "engine_failed", detail: engineFailureDetail(error) }; @@ -193,11 +194,19 @@ function deliveryBlock(message: unknown, index: number): string | undefined { * rendered as labelled blocks and the `daimon_inbox` accounting follows them as * what to do *after* the work, with the machine-readable payload kept as a * trailing appendix while it fits the same budget. + * + * Both tools are named the way the agent's own engine can call them. On Grok a + * Daimon tool is an MCP tool of server `daimon` and its bare name reaches + * nothing (`grokDaimonToolName`, the same contract module the worker's system + * prompt and identity envelope render from), so an agent handed the bare name + * cannot mark its work complete — and an unmarked, finished wake is recorded as + * deferred. */ -function inboxPrompt(messages: readonly unknown[], maxBytes = 12000): string { +function inboxPrompt(messages: readonly unknown[], engine: OrganizationRuntimeAgentConfig["engine"]["kind"], maxBytes = 12000): string { const body = JSON.stringify(messages); const blocks = messages.map(deliveryBlock).filter((block): block is string => block !== undefined); - const accounting = "\nWhen the work above is done, record each delivery with daimon_inbox_disposition (complete), or defer the ones you could not finish; use daimon_inbox for deliveries and remaining allowances. Reading or ending this turn never completes a delivery, and deferred work waits for a later external wake.\n"; + const tool = (name: string): string => engine === "grok" ? grokDaimonToolName(name) : name; + const accounting = `\nWhen the work above is done, record each delivery with ${tool("daimon_inbox_disposition")} (complete), or defer the ones you could not finish; use ${tool("daimon_inbox")} for deliveries and remaining allowances. Reading or ending this turn never completes a delivery, and deferred work waits for a later external wake.\n`; const header = blocks.length === 1 ? "Carry out this delivery.\n" : `Carry out these ${blocks.length} deliveries.\n`; const fits = (value: string): boolean => Buffer.byteLength(value) <= ORGANIZATION_RUNTIME_MAX_WAKE_TEXT_BYTES && [...value].length <= ORGANIZATION_RUNTIME_MAX_STRING_CODEPOINTS; @@ -209,7 +218,7 @@ function inboxPrompt(messages: readonly unknown[], maxBytes = 12000): string { if (Buffer.byteLength(body) <= maxBytes && fits(withPayload)) return withPayload; if (fits(task)) return task; } - const prefix = "Handle this inbox turn. Use daimon_inbox for deliveries and remaining allowances. Explicitly call daimon_inbox_disposition for each handled delivery (complete) or unfinished delivery (defer). Reading or ending this turn never completes a delivery. Deferred work waits for a later external wake.\n"; + const prefix = `Handle this inbox turn. Use ${tool("daimon_inbox")} for deliveries and remaining allowances. Explicitly call ${tool("daimon_inbox_disposition")} for each handled delivery (complete) or unfinished delivery (defer). Reading or ending this turn never completes a delivery. Deferred work waits for a later external wake.\n`; const prompt = prefix + body; if (Buffer.byteLength(body) > maxBytes || !fits(prompt)) { return prefix + "The selected payload exceeds the prompt budget; read it with daimon_inbox."; diff --git a/src/runtime/engineDispatcher.test.ts b/src/runtime/engineDispatcher.test.ts index 0dae9a8..62f37fc 100644 --- a/src/runtime/engineDispatcher.test.ts +++ b/src/runtime/engineDispatcher.test.ts @@ -4,7 +4,8 @@ import os from "node:os"; import path from "node:path"; import test from "node:test"; -import { codexSandboxProtectedPaths, codexSandboxReadablePaths, grokSandboxProtectedPaths, startOrganizationRuntimeEngine } from "./engineDispatcher.js"; +import { codexSandboxProtectedPaths, codexSandboxReadablePaths, grokSandboxProtectedPaths, identityEnvelope, startOrganizationRuntimeEngine } from "./engineDispatcher.js"; +import { DAIMON_GROK_MCP_SERVER, DAIMON_GROK_SYSTEM_PROMPT, DAIMON_GROK_TOOL_PREFIX, GROK_MCP_INVOKE_TOOL, GROK_MCP_SEARCH_TOOL, GROK_MCP_TOOL_NAME_ARGUMENT, grokDaimonToolName } from "../contracts/grokWorkerContract.js"; import { AGY_SUBSCRIPTION_REALM, GROK_SUBSCRIPTION_REALM } from "./contractManifest.js"; import type { EngineBrokerTurnClient } from "./engineBrokerControlClient.js"; import { ORGANIZATION_RUNTIME_VERSION, type OrganizationRuntimeAgentConfig } from "./organizationRuntime.js"; @@ -71,6 +72,55 @@ test("Codex strict sandbox protects current credentials, ingress, shared roots, assert.deepEqual(codexSandboxReadablePaths(current), [path.join(current.runtimeHomePath, "tool-output")]); }); +/** + * The envelope's Grok wording is the second naming rule a Grok worker reads, + * after the pinned system prompt. When the two disagreed the later, more + * emphatic one won and a live turn made zero tool calls with every tool + * correctly mounted, so what is asserted here is agreement: the same route, + * stated once, and no instruction to use a bare name. + */ +const mounted = ["moltnet_read", "moltnet_send", "memory_search"] as const; +const envelopeToolSentence = (kind: OrganizationRuntimeAgentConfig["engine"]["kind"]): string => + identityEnvelope(rootConfig("/private/org", kind), mounted).split("\n").find((line) => line.startsWith("Your mounted tools are exactly"))!; + +test("the Grok envelope states the use_tool prefix rule once and never countermands the system prompt", () => { + const sentence = envelopeToolSentence("grok"); + // The route, asserted: one bare catalogue, one prefix rule, one example. + assert.equal(sentence.includes(`Your mounted tools are exactly: ${mounted.join(", ")}.`), true); + assert.match(sentence, new RegExp(`MCP tool on server ${DAIMON_GROK_MCP_SERVER}`, "u")); + assert.match(sentence, new RegExp(`invoke it with ${GROK_MCP_INVOKE_TOOL}, ${GROK_MCP_TOOL_NAME_ARGUMENT} = ${DAIMON_GROK_TOOL_PREFIX}`, "u")); + assert.match(sentence, new RegExp(`for example ${grokDaimonToolName(mounted[0])}`, "u")); + assert.match(sentence, /none is callable by its bare name/u); + assert.match(sentence, new RegExp(`${GROK_MCP_SEARCH_TOOL} lists them if a name is unknown`, "u")); + assert.match(sentence, /No other tool reaches the newsroom\.$/u); + // What it must never say: the bare names are callable, the agent's own + // instructions are wrong, or a shell reaches the tools. + assert.doesNotMatch(sentence, /Call them by these names/u); + assert.doesNotMatch(sentence, /spell them differently/u); + assert.doesNotMatch(sentence, /shell|terminal|CLI|run_terminal/u); + // One catalogue only: the prefixed names are a rule, not a second list. + for (const tool of mounted) assert.equal(sentence.split(tool).length - 1, tool === mounted[0] ? 2 : 1, tool); + assert.equal(sentence.split(DAIMON_GROK_TOOL_PREFIX).length - 1, 2, "prefix appears as the rule and its one example"); + // The transport prohibition is untouched and still follows the tool sentence. + assert.match(identityEnvelope(rootConfig("/private/org", "grok"), mounted), /Do not seek transport credentials or invoke a transport CLI/u); +}); + +test("the Grok envelope and the pinned worker system prompt state the same route", () => { + const sentence = envelopeToolSentence("grok"); + for (const atom of [DAIMON_GROK_MCP_SERVER, DAIMON_GROK_TOOL_PREFIX, GROK_MCP_INVOKE_TOOL, GROK_MCP_SEARCH_TOOL, GROK_MCP_TOOL_NAME_ARGUMENT]) { + assert.ok(DAIMON_GROK_SYSTEM_PROMPT.includes(atom), `system prompt states ${atom}`); + assert.ok(sentence.includes(atom), `envelope states ${atom}`); + } +}); + +test("only Grok gains the prefix rule: every other engine's envelope stays byte-identical", () => { + const unchanged = `Your mounted tools are exactly: ${mounted.join(", ")}. Call them by these names; your instructions may spell them differently. No other tool reaches the newsroom.`; + for (const kind of ["codex", "agy"] as const) assert.equal(envelopeToolSentence(kind), unchanged); + assert.notEqual(envelopeToolSentence("grok"), unchanged); + // An unmounted agent gets no tool sentence at all, on every engine. + for (const kind of ["codex", "agy", "grok"] as const) assert.doesNotMatch(identityEnvelope(rootConfig("/private/org", kind)), /Your mounted tools/u); +}); + test("production dispatcher starts each closed engine intent through Daimon", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-dispatcher-")); const priorPath = process.env.PATH; diff --git a/src/runtime/engineDispatcher.ts b/src/runtime/engineDispatcher.ts index a5864a5..24513af 100644 --- a/src/runtime/engineDispatcher.ts +++ b/src/runtime/engineDispatcher.ts @@ -1,4 +1,5 @@ import { attentionTools, type AttentionRegistry } from "./attention.js"; +import { grokMountedToolNamingRule } from "../contracts/grokWorkerContract.js"; import path from "node:path"; import type { AgentHandle } from "../core/types.js"; @@ -180,20 +181,29 @@ function grokBrokerTurnFor(agent: OrganizationRuntimeAgentConfig, grokBroker: En /** * The caller-owned prompt preamble. * - * It names the mounted tools explicitly. A CLI engine reaches Daimon's tools - * over MCP, and Grok exposes MCP tools only through a deferred `search_tool` - * catalog, so an agent whose instructions name another engine's tool spelling - * can finish a turn having called nothing. The declared names are the caller's - * own configuration, not engine-supplied text. + * It names the mounted tools explicitly, because a CLI engine reaches Daimon's + * tools over MCP and an agent whose instructions name another engine's tool + * spelling can finish a turn having called nothing. The declared names are the + * caller's own configuration, not engine-supplied text. + * + * On Grok the bare names are not the callable ones: every Daimon tool is a + * deferred MCP tool of server `daimon`, reached through `use_tool` with + * `tool_name` = `daimon__`. That rule is not restated here — it is + * rendered by `grokMountedToolNamingRule` in the same contract module that + * renders the worker's pinned system prompt, so this envelope can no longer + * contradict it. Every other engine's sentence is unchanged, byte for byte. */ -function identityEnvelope(agent: OrganizationRuntimeAgentConfig, mountedToolNames: readonly string[] = []): string { +export function identityEnvelope(agent: OrganizationRuntimeAgentConfig, mountedToolNames: readonly string[] = []): string { return [ "", JSON.stringify({ id: agent.id, name: agent.name, instructions: agent.instructions }), "", ...(mountedToolNames.length === 0 ? [] : [ - `Your mounted tools are exactly: ${mountedToolNames.join(", ")}. Call them by these names; ` - + "your instructions may spell them differently. No other tool reaches the newsroom." + (agent.engine.kind === "grok" + ? grokMountedToolNamingRule(mountedToolNames) + : `Your mounted tools are exactly: ${mountedToolNames.join(", ")}. Call them by these names; ` + + "your instructions may spell them differently.") + + " No other tool reaches the newsroom." ]), "Colleagues only hear you when you call moltnet_send; your terminal response is a private note to the runtime, not a message to anyone — keep it to one line or leave it empty. " + "Do not seek transport credentials or invoke a transport CLI unless the caller explicitly mounted an authenticated transport tool.", From cb7a745c2de357ea57ad80843cce9c058dec5259 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 00:10:46 +0200 Subject: [PATCH 084/124] fix: name the prefixed Grok tool form as the only valid one and add no third search_tool voice --- src/contracts/grokWorkerContract.ts | 37 +++++--- src/runtime/engineDispatcher.test.ts | 23 +++-- src/runtime/engineDispatcher.ts | 12 ++- src/runtime/grokBrokerProxy.ts | 17 +++- src/runtime/grokBrokerTurnMeter.test.ts | 54 ++++++++++- src/runtime/grokBrokerTurnMeter.ts | 110 ++++++++++++++++++---- src/runtime/grokEngineBrokerTurn.ts | 18 +++- src/runtime/grokEngineBrokerUsage.test.ts | 51 +++++++++- src/runtime/turnRequestLedger.ts | 24 ++++- 9 files changed, 289 insertions(+), 57 deletions(-) diff --git a/src/contracts/grokWorkerContract.ts b/src/contracts/grokWorkerContract.ts index fbe949d..2c49b12 100644 --- a/src/contracts/grokWorkerContract.ts +++ b/src/contracts/grokWorkerContract.ts @@ -21,10 +21,12 @@ * Both texts a Grok worker receives are rendered from them: the pinned system * prompt below, and the caller's identity envelope * ({@link grokMountedToolNamingRule}, used by `src/runtime/engineDispatcher.ts`). - * They were worded independently once - the envelope told the model to call the - * tools by their bare names - and a live turn produced zero tool calls with - * every tool correctly mounted: two authoritative naming rules, the wrong one - * last. One definition is what keeps them from diverging again. + * They were worded independently once, and the envelope told the model to call + * the tools by their bare names - which Grok 1.0.34 refuses outright, before + * any HTTP: `'moltnet_read' is not a valid MCP tool name. Tool names must be + * qualified as \`server__tool\`` (local rig, real CLI, real rendered config). + * There is exactly one valid spelling, so two independently worded naming rules + * are one rule too many; this is the single definition both render from. */ export const DAIMON_GROK_MCP_SERVER = "daimon" as const; /** Grok's own name for an MCP tool of that server: exactly what `tool_name` must carry. */ @@ -50,19 +52,28 @@ export const DAIMON_GROK_SYSTEM_PROMPT = [ * The same route, stated once for the caller's identity envelope, where a * wake's real mounted tools are known. * - * It asserts rather than corrects: it names the bare tool set once, states the - * `use_tool` prefix rule once with one example drawn from that set, and says - * plainly that a bare name is not callable here. It never claims the agent's - * own instructions spell a tool wrongly, never offers a shell or CLI route, and - * never repeats the tool list a second time in prefixed form. + * It contributes exactly what the pinned prompt cannot know - the wake's real + * mounted names - and the one rule that makes them callable. It asserts rather + * than corrects: one bare catalogue, one prefix rule, one example, and the + * prefixed form named as the *only* valid form, because that is the CLI's own + * verdict on a bare name rather than a preference. + * + * What it deliberately leaves out is as load bearing. It never claims the + * agent's own instructions spell a tool wrongly, never offers a shell or CLI + * route, never repeats the catalogue in prefixed form - and never restates the + * `search_tool` rule. A Grok worker already reads two authoritative sentences + * about `search_tool`: the pinned prompt's ("only for a name you do not know") + * and Grok's own injected notice, which says the model MUST call it before any + * MCP tool. Observed on the rig: that contradiction is not enforced, and + * `use_tool` works with no prior `search_tool`. A third wording would only add + * a voice, so this sentence stays out of that argument entirely. */ export const grokMountedToolNamingRule = (mountedToolNames: readonly string[]): string => { const example = grokDaimonToolName(mountedToolNames[0] ?? DAIMON_GROK_EXAMPLE_TOOLS[0]); return `Your mounted tools are exactly: ${mountedToolNames.join(", ")}. ` - + `On this engine each is an MCP tool on server ${DAIMON_GROK_MCP_SERVER}: invoke it with ${GROK_MCP_INVOKE_TOOL}, ` - + `${GROK_MCP_TOOL_NAME_ARGUMENT} = ${DAIMON_GROK_TOOL_PREFIX} (for example ${example}). ` - + "None appears in your direct tool list and none is callable by its bare name; " - + `${GROK_MCP_SEARCH_TOOL} lists them if a name is unknown.`; + + `On this engine each is an MCP tool on server ${DAIMON_GROK_MCP_SERVER}, and its only valid tool name is ` + + `${DAIMON_GROK_TOOL_PREFIX}: invoke it with ${GROK_MCP_INVOKE_TOOL}, ${GROK_MCP_TOOL_NAME_ARGUMENT} = ${example}. ` + + "A bare name is not a valid MCP tool name and reaches nothing."; }; /** Closed declared-model vocabulary; defaults are `grok-4.6` at `low`. */ diff --git a/src/runtime/engineDispatcher.test.ts b/src/runtime/engineDispatcher.test.ts index 62f37fc..dea7789 100644 --- a/src/runtime/engineDispatcher.test.ts +++ b/src/runtime/engineDispatcher.test.ts @@ -74,30 +74,33 @@ test("Codex strict sandbox protects current credentials, ingress, shared roots, /** * The envelope's Grok wording is the second naming rule a Grok worker reads, - * after the pinned system prompt. When the two disagreed the later, more - * emphatic one won and a live turn made zero tool calls with every tool - * correctly mounted, so what is asserted here is agreement: the same route, - * stated once, and no instruction to use a bare name. + * after the pinned system prompt. It used to instruct the bare form, which + * Grok 1.0.34 refuses outright as an invalid MCP tool name, so what is asserted + * here is agreement: the same route, stated once, the prefixed form named as + * the only valid one, and no third voice about `search_tool`. */ const mounted = ["moltnet_read", "moltnet_send", "memory_search"] as const; const envelopeToolSentence = (kind: OrganizationRuntimeAgentConfig["engine"]["kind"]): string => identityEnvelope(rootConfig("/private/org", kind), mounted).split("\n").find((line) => line.startsWith("Your mounted tools are exactly"))!; -test("the Grok envelope states the use_tool prefix rule once and never countermands the system prompt", () => { +test("the Grok envelope names the prefixed form as the only valid one, once", () => { const sentence = envelopeToolSentence("grok"); // The route, asserted: one bare catalogue, one prefix rule, one example. assert.equal(sentence.includes(`Your mounted tools are exactly: ${mounted.join(", ")}.`), true); assert.match(sentence, new RegExp(`MCP tool on server ${DAIMON_GROK_MCP_SERVER}`, "u")); - assert.match(sentence, new RegExp(`invoke it with ${GROK_MCP_INVOKE_TOOL}, ${GROK_MCP_TOOL_NAME_ARGUMENT} = ${DAIMON_GROK_TOOL_PREFIX}`, "u")); - assert.match(sentence, new RegExp(`for example ${grokDaimonToolName(mounted[0])}`, "u")); - assert.match(sentence, /none is callable by its bare name/u); - assert.match(sentence, new RegExp(`${GROK_MCP_SEARCH_TOOL} lists them if a name is unknown`, "u")); + assert.match(sentence, new RegExp(`only valid tool name is ${DAIMON_GROK_TOOL_PREFIX}`, "u")); + assert.match(sentence, new RegExp(`invoke it with ${GROK_MCP_INVOKE_TOOL}, ${GROK_MCP_TOOL_NAME_ARGUMENT} = ${grokDaimonToolName(mounted[0])}`, "u")); + // A bare name is invalid, not merely discouraged: that is the CLI's own verdict. + assert.match(sentence, /A bare name is not a valid MCP tool name and reaches nothing\./u); assert.match(sentence, /No other tool reaches the newsroom\.$/u); // What it must never say: the bare names are callable, the agent's own // instructions are wrong, or a shell reaches the tools. assert.doesNotMatch(sentence, /Call them by these names/u); assert.doesNotMatch(sentence, /spell them differently/u); assert.doesNotMatch(sentence, /shell|terminal|CLI|run_terminal/u); + // Fewer authoritative voices: the pinned prompt and Grok's own injected + // notice already give two rules for `search_tool`. This adds no third. + assert.equal(sentence.includes(GROK_MCP_SEARCH_TOOL), false); // One catalogue only: the prefixed names are a rule, not a second list. for (const tool of mounted) assert.equal(sentence.split(tool).length - 1, tool === mounted[0] ? 2 : 1, tool); assert.equal(sentence.split(DAIMON_GROK_TOOL_PREFIX).length - 1, 2, "prefix appears as the rule and its one example"); @@ -107,7 +110,7 @@ test("the Grok envelope states the use_tool prefix rule once and never counterma test("the Grok envelope and the pinned worker system prompt state the same route", () => { const sentence = envelopeToolSentence("grok"); - for (const atom of [DAIMON_GROK_MCP_SERVER, DAIMON_GROK_TOOL_PREFIX, GROK_MCP_INVOKE_TOOL, GROK_MCP_SEARCH_TOOL, GROK_MCP_TOOL_NAME_ARGUMENT]) { + for (const atom of [DAIMON_GROK_MCP_SERVER, DAIMON_GROK_TOOL_PREFIX, GROK_MCP_INVOKE_TOOL, GROK_MCP_TOOL_NAME_ARGUMENT]) { assert.ok(DAIMON_GROK_SYSTEM_PROMPT.includes(atom), `system prompt states ${atom}`); assert.ok(sentence.includes(atom), `envelope states ${atom}`); } diff --git a/src/runtime/engineDispatcher.ts b/src/runtime/engineDispatcher.ts index 24513af..40b06f1 100644 --- a/src/runtime/engineDispatcher.ts +++ b/src/runtime/engineDispatcher.ts @@ -187,11 +187,13 @@ function grokBrokerTurnFor(agent: OrganizationRuntimeAgentConfig, grokBroker: En * caller's own configuration, not engine-supplied text. * * On Grok the bare names are not the callable ones: every Daimon tool is a - * deferred MCP tool of server `daimon`, reached through `use_tool` with - * `tool_name` = `daimon__`. That rule is not restated here — it is - * rendered by `grokMountedToolNamingRule` in the same contract module that - * renders the worker's pinned system prompt, so this envelope can no longer - * contradict it. Every other engine's sentence is unchanged, byte for byte. + * deferred MCP tool of server `daimon`, and Grok 1.0.34 refuses an unqualified + * name before any HTTP ("Tool names must be qualified as `server__tool`"). This + * envelope used to instruct exactly that refused form. The correct rule is not + * restated here — it is rendered by `grokMountedToolNamingRule` in the same + * contract module that renders the worker's pinned system prompt, so the two + * texts cannot contradict each other again. Every other engine's sentence is + * unchanged, byte for byte. */ export function identityEnvelope(agent: OrganizationRuntimeAgentConfig, mountedToolNames: readonly string[] = []): string { return [ diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index e13d5ca..c848d91 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -5,7 +5,7 @@ import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; import { DEFAULT_GROK_BROKER_MODEL_POLICY, parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; import { authorizeGrokBrokerProxyRequest } from "./grokBrokerProxyRequest.js"; import { GROK_SESSION_TITLE_SINK_KEY } from "./grokBrokerWorkerConfig.js"; -import { parseGrokUpstreamUsage, type GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; +import { parseGrokResponseToolNames, parseGrokUpstreamUsage, type GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; import { serveGrokInferenceGrant } from "./grokInferenceProxy.js"; import type { GrokInferenceGrants } from "./grokInferenceGrants.js"; @@ -49,7 +49,7 @@ export class GrokBrokerProxyRefusal extends Error { } async function serve(request: IncomingMessage, response: ServerResponse, authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream, capabilities: EngineBrokerCapabilities,guards:MapPromise>,turns:Map,fallback:GrokBrokerModelPolicy,grants?:GrokInferenceGrants): Promise { - let settle:((usage:ReturnType)=>void)|undefined; + let settle:((usage:ReturnType,toolCalls?:readonly string[])=>void)|undefined; let titleSink = false; try { const body = await readBody(request); const headers = Object.fromEntries(Object.entries(request.headers).map(([key, value]) => [key, Array.isArray(value) ? value[0] : value])); @@ -64,10 +64,14 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori const admission=turn.meter.admit(); if("refused" in admission){response.writeHead(429,{"content-type":"application/json","cache-control":"no-store"});response.end(JSON.stringify({error:"turn limit reached",limit:admission.refused}));return;} if("busy" in admission){response.writeHead(429,{"content-type":"application/json","cache-control":"no-store"});response.end('{"error":"turn request in flight"}');return;} - settle=(usage)=>{turn.meter.settle(admission.index,usage,body.byteLength);settle=undefined;}; + settle=(usage,toolCalls)=>{turn.meter.settle(admission.index,usage,body.byteLength,toolCalls);settle=undefined;}; let result = await upstream(prepared,admission.signal); if (result.status === 401) { token = authority.refreshAfterRejection?await authority.refreshAfterRejection(rejectedDigest):await authority.accessToken(true);const refreshedDigest=createHash("sha256").update(token).digest("hex"); prepared = { ...prepared, headers: { ...prepared.headers, authorization: `Bearer ${token}` } }; token = ""; result = await upstream(prepared,admission.signal);if(result.status===401)await authority.markRejected(refreshedDigest); } - settle?.(parseGrokUpstreamUsage(result.body,result.headers["content-type"])); + // Names only, bounded, and never a reason to fail the request: the response + // is already buffered here for its usage block, so what the model tried to + // call is in hand. A decoder fault records no attempt rather than a false + // empty one, and never disturbs the turn. + settle?.(parseGrokUpstreamUsage(result.body,result.headers["content-type"]),toolCallsOrNothing(result.body,result.headers["content-type"])); response.writeHead(result.status, { "content-type": result.headers["content-type"] ?? "application/json", "cache-control": "no-store" }); response.end(result.body); } catch (error) { settle?.(undefined); @@ -112,6 +116,11 @@ const expectedWorkerProbe = (request: IncomingMessage): boolean => request.headers.authorization === undefined && (request.method ?? "") === "GET" && new URL(request.url ?? "/", "http://127.0.0.1").pathname === "/"; +/** Instrumentation must never fail a turn: a throwing decoder records nothing, exactly as an undecodable response does. */ +const toolCallsOrNothing = (body: Uint8Array, contentType: string | undefined): readonly string[] | undefined => { + try { return parseGrokResponseToolNames(body, contentType); } catch { return undefined; } +}; + /** The body gate, refused non-retryably: a rejected body is a policy miss, never a transient fault. */ function authorizeRequestOrRefuse(...args: Parameters): ReturnType { try { return authorizeGrokBrokerProxyRequest(...args); } diff --git a/src/runtime/grokBrokerTurnMeter.test.ts b/src/runtime/grokBrokerTurnMeter.test.ts index 094ef65..881b8fd 100644 --- a/src/runtime/grokBrokerTurnMeter.test.ts +++ b/src/runtime/grokBrokerTurnMeter.test.ts @@ -3,7 +3,7 @@ import { request as httpRequest } from "node:http"; import test from "node:test"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; -import { GrokBrokerTurnMeter, parseGrokUpstreamUsage } from "./grokBrokerTurnMeter.js"; +import { GROK_REQUEST_TOOL_CALLS_MAX, GROK_TOOL_CALL_INVALID, GROK_TOOL_CALL_TRUNCATED, GrokBrokerTurnMeter, parseGrokResponseToolNames, parseGrokUpstreamUsage } from "./grokBrokerTurnMeter.js"; const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); const body = JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: lean }); @@ -189,3 +189,55 @@ test("an implausible per-request usage block is never added, and missing usage s }); assert.deepEqual([blind.snapshot().limitReason, blind.snapshot().tokens, blind.snapshot().estimatedRequests], ["tokens", 12_891, 3]); }); + +const events = (chunks: readonly unknown[]): Uint8Array => + Buffer.from(chunks.map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("") + "data: [DONE]\n\n"); +/** One streaming tool call: the name arrives in one delta, the arguments in the next. */ +const callDeltas = (names: readonly string[]): unknown[] => [ + ...names.map((name, index) => ({ choices: [{ index: 0, delta: { tool_calls: [{ index, id: `call-${index}`, type: "function", function: { name, arguments: "" } }] } }] })), + ...names.map((_name, index) => ({ choices: [{ index: 0, delta: { tool_calls: [{ index, function: { arguments: '{"query":"secret"}' } }] } }] })) +]; + +/** + * Two live turns ended with every tool correctly mounted and no way to tell + * whether the model had tried to call anything. These names are that answer, + * and nothing more than that answer. + */ +test("a response's tool-call names are read, bounded, and stripped of everything but the names", async () => { + // Mutation guard: passing the response through instead of the names leaks arguments here. + const names = parseGrokResponseToolNames(events([...callDeltas(["use_tool", "search_tool"]), { choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], usage: { prompt_tokens: 1, completion_tokens: 1 } }]), "text/event-stream"); + assert.deepEqual(names, ["use_tool", "search_tool"]); + assert.equal(JSON.stringify(names).includes("secret"), false); + + // A non-streaming body carries its calls on the message; one tool called + // twice is two attempts, because neither carries a streaming call index. + assert.deepEqual(parseGrokResponseToolNames(Buffer.from(JSON.stringify({ choices: [{ message: { tool_calls: [{ function: { name: "use_tool" } }, { function: { name: "use_tool" } }] } }] })), "application/json"), ["use_tool", "use_tool"]); + + // Absence stays absence: a decoded response that called nothing is `[]`, and + // an undecodable one is nothing at all. A zero-length list must never be + // invented for a response nobody could read. + assert.deepEqual(parseGrokResponseToolNames(events([{ choices: [{ index: 0, delta: { content: "x" } }] }]), "text/event-stream"), []); + assert.equal(parseGrokResponseToolNames(Buffer.from("gateway"), "text/html"), undefined); + assert.equal(parseGrokResponseToolNames(Buffer.from(""), "text/event-stream"), undefined); + assert.equal(parseGrokResponseToolNames(Buffer.from("data: not-json\n\n"), "text/event-stream"), undefined); + + // A name that is not a plain short identifier is counted, never passed through. + assert.deepEqual(parseGrokResponseToolNames(events(callDeltas(["ok_tool", "a b/c", "x".repeat(65), "inject\nline"])), "text/event-stream"), ["ok_tool", GROK_TOOL_CALL_INVALID, GROK_TOOL_CALL_INVALID, GROK_TOOL_CALL_INVALID]); + + // Mutation guard: unbounded, a pathological response writes 400 names into one row. + const many = parseGrokResponseToolNames(events(callDeltas(Array.from({ length: 400 }, (_value, index) => `tool_${index}`))), "text/event-stream"); + assert.equal(many!.length, GROK_REQUEST_TOOL_CALLS_MAX); + assert.equal(many!.at(-1), GROK_TOOL_CALL_TRUNCATED); + assert.deepEqual(many!.slice(0, 2), ["tool_0", "tool_1"]); + // Exactly the bound is not truncated. + assert.equal(parseGrokResponseToolNames(events(callDeltas(Array.from({ length: GROK_REQUEST_TOOL_CALLS_MAX }, (_value, index) => `tool_${index}`))), "text/event-stream")!.includes(GROK_TOOL_CALL_TRUNCATED), false); +}); + +test("the meter carries each request's tool-call names without letting them touch the spend gate", async () => { + const meter = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 300_000, timeoutMs: 60_000 }); + await withProxy({ prompt_tokens: 11, completion_tokens: 2 }, meter, async (send) => { assert.equal((await send()).status, 200); }); + const [timing] = meter.snapshot().timings; + // The stub response carries no tool call, and says so rather than staying silent. + assert.deepEqual(timing!.toolCalls, []); + assert.deepEqual([meter.snapshot().tokens, meter.snapshot().limitReason, meter.snapshot().estimatedRequests], [13, "none", 0]); +}); diff --git a/src/runtime/grokBrokerTurnMeter.ts b/src/runtime/grokBrokerTurnMeter.ts index 3c14b7c..f266a16 100644 --- a/src/runtime/grokBrokerTurnMeter.ts +++ b/src/runtime/grokBrokerTurnMeter.ts @@ -1,8 +1,13 @@ import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; import { sumEngineBrokerTurnUsage, type EngineBrokerLimitReason, type EngineBrokerTurnLimits, type EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; -/** `estimated` marks a request whose response carried no valid usage and was charged {@link estimateGrokRequestUsage}. */ -export type GrokBrokerRequestTiming = Readonly<{ startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage; estimated?: true }>; +/** + * `estimated` marks a request whose response carried no valid usage and was + * charged {@link estimateGrokRequestUsage}. `toolCalls` are the tool-call names + * that request's response carried, names only ({@link parseGrokResponseToolNames}); + * absent means the response could not be decoded, `[]` that it called nothing. + */ +export type GrokBrokerRequestTiming = Readonly<{ startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage; estimated?: true; toolCalls?: readonly string[] }>; export type GrokBrokerTurnMeterSnapshot = Readonly<{ requests: number; tokens: number; limitReason: EngineBrokerLimitReason; usage: EngineBrokerTurnUsage | null; estimatedRequests: number; timings: readonly GrokBrokerRequestTiming[] }>; /** @@ -36,7 +41,7 @@ export type GrokBrokerTurnMeterSnapshot = Readonly<{ requests: number; tokens: n */ export class GrokBrokerTurnMeter { private readonly startedAt: number; - private readonly timings: { startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage; estimated?: true }[] = []; + private readonly timings: { startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage; estimated?: true; toolCalls?: readonly string[] }[] = []; private tokens = 0; private reason: EngineBrokerLimitReason = "none"; private inFlight: { index: number; controller: AbortController } | undefined; @@ -60,18 +65,21 @@ export class GrokBrokerTurnMeter { } /** - * Records one admitted request's end and its usage. A response without valid - * usage (absent, malformed, implausible, or a failed/aborted call) is charged - * a conservative estimate from the request body size, so a missing `usage` - * can never silently disable the token ceiling. + * Records one admitted request's end, its usage, and the tool-call names its + * response carried. A response without valid usage (absent, malformed, + * implausible, or a failed/aborted call) is charged a conservative estimate + * from the request body size, so a missing `usage` can never silently disable + * the token ceiling. `toolCalls` is observation only: it never affects + * admission, the running total, or any limit. */ - settle(index: number, usage: EngineBrokerTurnUsage | undefined, requestBytes: number): void { + settle(index: number, usage: EngineBrokerTurnUsage | undefined, requestBytes: number, toolCalls?: readonly string[]): void { const timing = this.timings[index]; if (timing === undefined || timing.endedAt !== undefined) return; if (this.inFlight?.index === index) this.inFlight = undefined; timing.endedAt = new Date(this.now()).toISOString(); if (usage === undefined) { timing.usage = estimateGrokRequestUsage(requestBytes); timing.estimated = true; } else timing.usage = usage; + if (toolCalls !== undefined) timing.toolCalls = toolCalls; this.tokens += timing.usage.total; } @@ -118,6 +126,26 @@ const count = (value: unknown): number | undefined => typeof value === "number" * zero-filled and never added — the meter charges an estimate instead. */ export function parseGrokUpstreamUsage(body: Uint8Array, contentType: string | undefined): EngineBrokerTurnUsage | undefined { + let found: EngineBrokerTurnUsage | undefined; + for (const candidate of decodeUpstreamResponse(body, contentType) ?? []) { + if (!isRecord(candidate) || !isRecord(candidate.usage)) continue; + // Last usage block wins even when invalid: an implausible final report + // must not fall back to an earlier, smaller block (the request is then + // charged the estimate instead). + found = decodeOpenAiUsage(candidate.usage); + } + return found; +} + +/** + * Every decodable JSON object of one upstream response: each `data:` event of + * an SSE stream, or the single body of a JSON response. + * + * `undefined` means *nothing* decoded — an unparseable or non-JSON response. + * Callers must keep that distinct from a decoded response that said nothing, + * because the ledger never fabricates an observation it did not make. + */ +function decodeUpstreamResponse(body: Uint8Array, contentType: string | undefined): unknown[] | undefined { const text = Buffer.from(body.buffer, body.byteOffset, body.byteLength).toString("utf8"); const candidates: unknown[] = []; if (contentType?.includes("text/event-stream") === true || text.startsWith("data:")) { @@ -125,20 +153,68 @@ export function parseGrokUpstreamUsage(body: Uint8Array, contentType: string | u if (!line.startsWith("data:")) continue; const payload = line.slice(5).trim(); if (payload === "[DONE]" || payload.length === 0) continue; - try { candidates.push(JSON.parse(payload)); } catch { /* a non-JSON event carries no usage */ } + try { candidates.push(JSON.parse(payload)); } catch { /* a non-JSON event carries neither usage nor a tool call */ } } } else { try { candidates.push(JSON.parse(text)); } catch { return undefined; } } - let found: EngineBrokerTurnUsage | undefined; - for (const candidate of candidates) { - if (!isRecord(candidate) || !isRecord(candidate.usage)) continue; - // Last usage block wins even when invalid: an implausible final report - // must not fall back to an earlier, smaller block (the request is then - // charged the estimate instead). - found = decodeOpenAiUsage(candidate.usage); + return candidates.length === 0 ? undefined : candidates; +} + +/** At most this many names per request row; a longer list ends in {@link GROK_TOOL_CALL_TRUNCATED}. */ +export const GROK_REQUEST_TOOL_CALLS_MAX = 16; +/** A `name` that is not a plain short identifier is counted, never passed through. */ +export const GROK_TOOL_CALL_INVALID = ""; +export const GROK_TOOL_CALL_TRUNCATED = ""; +const TOOL_CALL_NAME = /^[A-Za-z0-9_.-]{1,64}$/u; + +/** + * The tool-call NAMES one upstream response carried, and nothing else. + * + * Two live turns could not answer "did the model ever try `use_tool` or + * `search_tool`", because the per-request rows recorded timings and tokens but + * never an attempt. This is that answer, under four rules: + * + * - names only. No arguments, no message content, no tokens, no header. A + * `name` that is not a plain short identifier is recorded as + * {@link GROK_TOOL_CALL_INVALID} rather than passing provider bytes through; + * - bounded. At most {@link GROK_REQUEST_TOOL_CALLS_MAX} entries, the last being + * {@link GROK_TOOL_CALL_TRUNCATED} when the response carried more, so a + * pathological response cannot write an unbounded row; + * - absence stays absence. A decoded response that called nothing returns `[]`; + * a response that could not be decoded returns `undefined` and the row records + * no field at all; + * - one streaming call names itself in one delta and streams its arguments in + * the rest, so a repeat of the same `(choice, call)` index is that same call, + * not a second attempt. + */ +export function parseGrokResponseToolNames(body: Uint8Array, contentType: string | undefined): readonly string[] | undefined { + const candidates = decodeUpstreamResponse(body, contentType); + if (candidates === undefined) return undefined; + const names: string[] = [], seen = new Set(); + scan: for (const candidate of candidates) { + if (!isRecord(candidate) || !Array.isArray(candidate.choices)) continue; + for (const choice of candidate.choices) { + if (!isRecord(choice)) continue; + for (const source of [choice.delta, choice.message]) { + if (!isRecord(source) || !Array.isArray(source.tool_calls)) continue; + for (const call of source.tool_calls) { + if (!isRecord(call) || !isRecord(call.function)) continue; + const name = call.function.name; + // An arguments-only delta names nothing; it is not an attempt of its own. + if (typeof name !== "string" || name.length === 0) continue; + if (typeof choice.index === "number" && typeof call.index === "number") { + const key = `${choice.index}:${call.index}`; + if (seen.has(key)) continue; + seen.add(key); + } + names.push(TOOL_CALL_NAME.test(name) ? name : GROK_TOOL_CALL_INVALID); + if (names.length > GROK_REQUEST_TOOL_CALLS_MAX) break scan; + } + } + } } - return found; + return names.length > GROK_REQUEST_TOOL_CALLS_MAX ? [...names.slice(0, GROK_REQUEST_TOOL_CALLS_MAX - 1), GROK_TOOL_CALL_TRUNCATED] : names; } function decodeOpenAiUsage(usage: JsonRecord): EngineBrokerTurnUsage | undefined { diff --git a/src/runtime/grokEngineBrokerTurn.ts b/src/runtime/grokEngineBrokerTurn.ts index 674b572..3c66cdc 100644 --- a/src/runtime/grokEngineBrokerTurn.ts +++ b/src/runtime/grokEngineBrokerTurn.ts @@ -120,12 +120,22 @@ function streamOrMeterUsage(stream: GrokStreamUsage | undefined, snapshot: GrokB return snapshot.usage; } -/** Per-request rows: stream usage with proxy timing when both describe the same requests, else the proxy's own measured requests. */ +/** Per-request rows: stream usage with the proxy's own observation when both describe the same requests, else the proxy's measured requests. */ function requestRows(stream: GrokStreamUsage | undefined, snapshot: GrokBrokerTurnMeterSnapshot): BrokerTurnMeteringDetail["requests"] { if (stream !== undefined && stream.requests.length > 0) { const timed = snapshot.timings.length === stream.requests.length; - return stream.requests.map((value, index) => ({ ...value, usageSource: "stream" as const, ...(timed ? clock(snapshot.timings[index]!) : {}) })); + return stream.requests.map((value, index) => ({ ...value, usageSource: "stream" as const, ...(timed ? observed(snapshot.timings[index]!) : {}) })); } - return snapshot.timings.flatMap((timing, index) => timing.usage === undefined ? [] : [{ index, ...usageOf(timing.usage), usageSource: timing.estimated === true ? "estimated" as const : "upstream" as const, ...clock(timing) }]); + return snapshot.timings.flatMap((timing, index) => timing.usage === undefined ? [] : [{ index, ...usageOf(timing.usage), usageSource: timing.estimated === true ? "estimated" as const : "upstream" as const, ...observed(timing) }]); } -const clock = (timing: GrokBrokerTurnMeterSnapshot["timings"][number]) => ({ startedAt: timing.startedAt, ...(timing.endedAt === undefined ? {} : { endedAt: timing.endedAt }) }); +/** + * What only the proxy saw of one request: its clock, and the tool-call names the + * response carried. Both are attached on the stream path only when the two + * descriptions are request-for-request aligned, because an unaligned index would + * credit one request's attempt to another. + */ +const observed = (timing: GrokBrokerTurnMeterSnapshot["timings"][number]) => ({ + startedAt: timing.startedAt, + ...(timing.endedAt === undefined ? {} : { endedAt: timing.endedAt }), + ...(timing.toolCalls === undefined ? {} : { toolCalls: timing.toolCalls }) +}); diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index 9964bd9..e316a57 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -47,10 +47,19 @@ const untilAborted = (signal: AbortSignal): Promise => new Promise((_reso * talks to the proxy exactly as the native worker does (capability bearer, * pinned client version, lean body). Only the launcher and attestation are fakes. */ -const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId: string, worker: Worker, overrides?: Parameters[6], limits?: EngineBrokerServiceRegistration["limits"], turnStore?: string, syncDirectory?: (directory: string) => Promise) => ReturnType; usageRows: () => Promise[]>; requestRows: () => Promise[]>; upstreamCalls: () => number; upstreamAborts: () => number }>) => Promise, usageLedgerPath?: string, upstreamDelayMs: (call: number) => number = () => 15): Promise => { +/** The stub provider's own SSE response: usage only, and no tool call, unless a test says otherwise. */ +const upstreamResponse = (): string => `data: ${JSON.stringify({ choices: [], usage: upstreamUsage })}\n\ndata: [DONE]\n\n`; +/** One streaming tool call per name, arguments in a following delta, then the usage event. */ +const upstreamToolCallResponse = (names: readonly string[]): string => [ + ...names.map((name, index) => ({ choices: [{ index: 0, delta: { tool_calls: [{ index, id: `call-${index}`, type: "function", function: { name, arguments: "" } }] } }] })), + ...names.map((_name, index) => ({ choices: [{ index: 0, delta: { tool_calls: [{ index, function: { arguments: '{"tool_name":"daimon__moltnet_read"}' } }] } }] })), + { choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], usage: upstreamUsage } +].map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("") + "data: [DONE]\n\n"; + +const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId: string, worker: Worker, overrides?: Parameters[6], limits?: EngineBrokerServiceRegistration["limits"], turnStore?: string, syncDirectory?: (directory: string) => Promise) => ReturnType; usageRows: () => Promise[]>; requestRows: () => Promise[]>; upstreamCalls: () => number; upstreamAborts: () => number }>) => Promise, usageLedgerPath?: string, upstreamDelayMs: (call: number) => number = () => 15, upstreamBody: (call: number) => string = upstreamResponse): Promise => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-usage-")); let calls = 0, aborted = 0; - const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async (_request, signal) => { calls++; await new Promise((resolve, reject) => { const timer = setTimeout(resolve, upstreamDelayMs(calls)); signal?.addEventListener("abort", () => { clearTimeout(timer); aborted++; reject(new Error("aborted")); }, { once: true }); }); return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(`data: ${JSON.stringify({ choices: [], usage: upstreamUsage })}\n\ndata: [DONE]\n\n`) }; }, undefined, 0); + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async (_request, signal) => { calls++; await new Promise((resolve, reject) => { const timer = setTimeout(resolve, upstreamDelayMs(calls)); signal?.addEventListener("abort", () => { clearTimeout(timer); aborted++; reject(new Error("aborted")); }, { once: true }); }); return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(upstreamBody(calls)) }; }, undefined, 0); const ledger = usageLedgerPath ?? path.join(root, "usage.jsonl"); const rows = async (file: string) => (await readFile(file, "utf8").catch(() => "")).split("\n").filter((line) => line.length > 0).map((line) => JSON.parse(line) as Record); try { @@ -244,3 +253,41 @@ test("the broker meters only through the single sealing helper, on both terminal assert.equal((body.match(/finishBrokerTurnWithUsage\(/gu) ?? []).length, 2); assert.ok(body.indexOf("return replay(") < body.indexOf("finishBrokerTurnWithUsage("), "a replay returns before any metering"); }); + +/** + * The question two live turns could not answer: did the model ever *try* to + * call a tool? The rows carried timings and tokens and nothing about an + * attempt, so a turn with zero tool calls and a turn whose calls all failed + * read identically after the fact. + */ +test("each per-request row records the tool-call names that request's response carried, names only", async () => { + await withBroker(async ({ turn, requestRows }) => { + await turn("wake-tools", twoRequests); + const rows = await requestRows(); + // Mutation guard: without the field these are `[undefined, undefined]`. + assert.deepEqual(rows.map((row) => row.tool_calls), [["use_tool", "search_tool"], ["use_tool", "search_tool"]]); + const text = JSON.stringify(rows); + // Names only: no arguments, no message content, no bearer. + assert.equal(text.includes("daimon__moltnet_read"), false, "an argument value must never reach the ledger"); + assert.equal(text.includes("arguments"), false); + assert.equal(text.includes("provider-token"), false); + }, undefined, () => 1, () => upstreamToolCallResponse(["use_tool", "search_tool"])); +}); + +test("a response that called nothing records an empty list, and one that cannot be decoded records no field at all", async () => { + await withBroker(async ({ turn, requestRows }) => { + await turn("wake-silent", twoRequests); + // Decoded, and it called nothing: that is an observation, not a gap. + assert.deepEqual((await requestRows()).map((row) => row.tool_calls), [[], []]); + }, undefined, () => 1); + await withBroker(async ({ turn, requestRows }) => { + await turn("wake-undecodable", twoRequests); + const rows = await requestRows(); + // Mutation guard: a fabricated `[]` here would be byte-identical to the + // measured empty list above, and the ledger would claim an observation the + // proxy never made. + assert.deepEqual(rows.map((row) => Object.hasOwn(row, "tool_calls")), [false, false]); + assert.deepEqual(rows.map((row) => row.request), [0, 1], "the rows themselves are still written"); + }, undefined, () => 1, () => "bad gateway"); +}); + diff --git a/src/runtime/turnRequestLedger.ts b/src/runtime/turnRequestLedger.ts index bb33992..d0d2153 100644 --- a/src/runtime/turnRequestLedger.ts +++ b/src/runtime/turnRequestLedger.ts @@ -110,7 +110,12 @@ const requestClockFields = (request: Readonly<{ startedAt?: string; endedAt?: st * (the provider response the proxy saw), or `estimated` (no valid usage; the * proxy's conservative charge, see `grokBrokerTurnMeter.ts`). */ -export type GrokTurnRequest = Readonly<{ index: number; input: number; cacheRead: number; cacheWrite: number; output: number; total: number; startedAt?: string; endedAt?: string; usageSource?: "stream" | "upstream" | "estimated" }>; +/** + * `toolCalls` are the tool-call names that request's response carried, as the + * proxy read them (`grokBrokerTurnMeter.ts`): names only, bounded, `[]` for a + * response that called nothing, and absent when no response could be decoded. + */ +export type GrokTurnRequest = Readonly<{ index: number; input: number; cacheRead: number; cacheWrite: number; output: number; total: number; startedAt?: string; endedAt?: string; usageSource?: "stream" | "upstream" | "estimated"; toolCalls?: readonly string[] }>; /** * `requestCount` is the turn's admitted request count when it exceeds the rows: * a killed turn's in-flight request was sent upstream but never reported usage, @@ -125,6 +130,22 @@ export type GrokTurnRequestEntry = Readonly<{ agent: string; wake: string; turn: * reasoning tokens, so `reasoning` is absent rather than zero. `turn` is the * broker idempotency key and `thread` the Grok session id when the stream * named one. + * + * `tool_calls` is what a turn's rows could not say before: whether the model + * ever *tried* to call anything. Two live turns ended with correctly mounted + * tools and no visible attempt, and the rows recorded timings and tokens only, + * so the question could not be answered after the fact. It is names only — + * never arguments, never message content, never a bearer — bounded, `[]` for a + * response that called nothing, and absent for a response that could not be + * decoded, because a fabricated empty list is byte-identical to a measured one. + * + * It is an additive field inside the unchanged + * `noopolis.daimon.turn-requests.v1` row, deliberately without a version bump: + * Spawnfile's usage reader (`spawnfile/src/runtime/usageLedger.ts`) drops every + * line whose `v` it does not recognise while ignoring fields it does not know, + * and Paideia only relocates this stream's path + * (`DAIMON_TURN_REQUESTS_LEDGER_PATH`). A bump is what would blind them; a new + * field is not. */ export const renderGrokTurnRequestLines = (entry: GrokTurnRequestEntry): string => { const at = entry.at ?? new Date().toISOString(); @@ -146,6 +167,7 @@ export const renderGrokTurnRequestLines = (entry: GrokTurnRequestEntry): string output: request.output, total: request.total, ...(request.usageSource === undefined ? {} : { usage_source: request.usageSource }), + ...(request.toolCalls === undefined ? {} : { tool_calls: request.toolCalls }), ...requestClockFields(request) })}\n`).join(""); }; From 1c74e716c0ec76fe55fe71a4f7ee3fb3048c7b12 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 00:11:23 +0200 Subject: [PATCH 085/124] feat: record each brokered request's tool-call names in the per-request ledger --- src/runtime/AGENTS.md | 22 ++++++++++++++++++++++ src/runtime/grokBrokerTurnMeter.test.ts | 6 ++++++ 2 files changed, 28 insertions(+) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 7487b5d..1860ba4 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -353,6 +353,28 @@ is swallowed, because instrumentation must never fail a wake. The existing ledger's version, path, and field list are untouched, so Spawnfile's `v`-pinned reader is unaffected. +Each Grok row also carries `tool_calls`: the tool-call NAMES that request's +response carried, read by the proxy from the body it already buffers for usage +(`parseGrokResponseToolNames` in `grokBrokerTurnMeter.ts`). Timings and tokens +alone cannot answer "did the model ever *try* to call `use_tool` or +`search_tool`", which is exactly the question two live turns left open. Names +only — never arguments, never message content, never a bearer; a `name` that is +not a plain short identifier is recorded as `` rather than passed +through, and the list is bounded at `GROK_REQUEST_TOOL_CALLS_MAX` (16) entries +with a `` last entry, so a pathological response cannot write an +unbounded row. Absence stays absence, as everywhere in these ledgers: a decoded +response that called nothing records `[]`, and a response that could not be +decoded records *no field at all*, because a fabricated empty list is +byte-identical to a measured one. On the stream row path the names are attached +only when the proxy's timings and the worker's stream requests are aligned +request-for-request, since an unaligned index would credit one request's attempt +to another. It is an additive field inside the unchanged +`noopolis.daimon.turn-requests.v1` row and deliberately not a version bump: +Spawnfile's reader pins `v` and ignores fields it does not know, and Paideia +only relocates this stream's path. The whole path is advisory — the parse is +wrapped, and nothing it does can refuse, delay, or fail a turn, or reach the +spend gate. + `testRuntimeSubprocess.ts` is an unexported, explicit-test-only JSONL process surface for exercising the real control, schedule, and acceptance paths with a controlled clock and deterministic scripted cognition. Its ephemeral loopback diff --git a/src/runtime/grokBrokerTurnMeter.test.ts b/src/runtime/grokBrokerTurnMeter.test.ts index 881b8fd..852b514 100644 --- a/src/runtime/grokBrokerTurnMeter.test.ts +++ b/src/runtime/grokBrokerTurnMeter.test.ts @@ -224,6 +224,12 @@ test("a response's tool-call names are read, bounded, and stripped of everything // A name that is not a plain short identifier is counted, never passed through. assert.deepEqual(parseGrokResponseToolNames(events(callDeltas(["ok_tool", "a b/c", "x".repeat(65), "inject\nline"])), "text/event-stream"), ["ok_tool", GROK_TOOL_CALL_INVALID, GROK_TOOL_CALL_INVALID, GROK_TOOL_CALL_INVALID]); + // A hostile but decodable shape yields no attempt instead of throwing: + // instrumentation must never be able to fail the turn it observes. + assert.deepEqual(parseGrokResponseToolNames(events([ + { choices: "not-an-array" }, { choices: [null, 7, { delta: { tool_calls: "no" } }, { message: { tool_calls: [null, { function: null }, { function: { name: 42 } }, { function: { name: "" } }] } }] } + ]), "text/event-stream"), []); + // Mutation guard: unbounded, a pathological response writes 400 names into one row. const many = parseGrokResponseToolNames(events(callDeltas(Array.from({ length: 400 }, (_value, index) => `tool_${index}`))), "text/event-stream"); assert.equal(many!.length, GROK_REQUEST_TOOL_CALLS_MAX); From d8a67dcee896fb9c66da2f03c0b3da4aaf2cac04 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 00:12:12 +0200 Subject: [PATCH 086/124] test: split the identity-envelope tests into their own file under the line limit --- src/runtime/engineDispatcher.test.ts | 55 +---------------- src/runtime/engineDispatcherIdentity.test.ts | 65 ++++++++++++++++++++ 2 files changed, 66 insertions(+), 54 deletions(-) create mode 100644 src/runtime/engineDispatcherIdentity.test.ts diff --git a/src/runtime/engineDispatcher.test.ts b/src/runtime/engineDispatcher.test.ts index dea7789..0dae9a8 100644 --- a/src/runtime/engineDispatcher.test.ts +++ b/src/runtime/engineDispatcher.test.ts @@ -4,8 +4,7 @@ import os from "node:os"; import path from "node:path"; import test from "node:test"; -import { codexSandboxProtectedPaths, codexSandboxReadablePaths, grokSandboxProtectedPaths, identityEnvelope, startOrganizationRuntimeEngine } from "./engineDispatcher.js"; -import { DAIMON_GROK_MCP_SERVER, DAIMON_GROK_SYSTEM_PROMPT, DAIMON_GROK_TOOL_PREFIX, GROK_MCP_INVOKE_TOOL, GROK_MCP_SEARCH_TOOL, GROK_MCP_TOOL_NAME_ARGUMENT, grokDaimonToolName } from "../contracts/grokWorkerContract.js"; +import { codexSandboxProtectedPaths, codexSandboxReadablePaths, grokSandboxProtectedPaths, startOrganizationRuntimeEngine } from "./engineDispatcher.js"; import { AGY_SUBSCRIPTION_REALM, GROK_SUBSCRIPTION_REALM } from "./contractManifest.js"; import type { EngineBrokerTurnClient } from "./engineBrokerControlClient.js"; import { ORGANIZATION_RUNTIME_VERSION, type OrganizationRuntimeAgentConfig } from "./organizationRuntime.js"; @@ -72,58 +71,6 @@ test("Codex strict sandbox protects current credentials, ingress, shared roots, assert.deepEqual(codexSandboxReadablePaths(current), [path.join(current.runtimeHomePath, "tool-output")]); }); -/** - * The envelope's Grok wording is the second naming rule a Grok worker reads, - * after the pinned system prompt. It used to instruct the bare form, which - * Grok 1.0.34 refuses outright as an invalid MCP tool name, so what is asserted - * here is agreement: the same route, stated once, the prefixed form named as - * the only valid one, and no third voice about `search_tool`. - */ -const mounted = ["moltnet_read", "moltnet_send", "memory_search"] as const; -const envelopeToolSentence = (kind: OrganizationRuntimeAgentConfig["engine"]["kind"]): string => - identityEnvelope(rootConfig("/private/org", kind), mounted).split("\n").find((line) => line.startsWith("Your mounted tools are exactly"))!; - -test("the Grok envelope names the prefixed form as the only valid one, once", () => { - const sentence = envelopeToolSentence("grok"); - // The route, asserted: one bare catalogue, one prefix rule, one example. - assert.equal(sentence.includes(`Your mounted tools are exactly: ${mounted.join(", ")}.`), true); - assert.match(sentence, new RegExp(`MCP tool on server ${DAIMON_GROK_MCP_SERVER}`, "u")); - assert.match(sentence, new RegExp(`only valid tool name is ${DAIMON_GROK_TOOL_PREFIX}`, "u")); - assert.match(sentence, new RegExp(`invoke it with ${GROK_MCP_INVOKE_TOOL}, ${GROK_MCP_TOOL_NAME_ARGUMENT} = ${grokDaimonToolName(mounted[0])}`, "u")); - // A bare name is invalid, not merely discouraged: that is the CLI's own verdict. - assert.match(sentence, /A bare name is not a valid MCP tool name and reaches nothing\./u); - assert.match(sentence, /No other tool reaches the newsroom\.$/u); - // What it must never say: the bare names are callable, the agent's own - // instructions are wrong, or a shell reaches the tools. - assert.doesNotMatch(sentence, /Call them by these names/u); - assert.doesNotMatch(sentence, /spell them differently/u); - assert.doesNotMatch(sentence, /shell|terminal|CLI|run_terminal/u); - // Fewer authoritative voices: the pinned prompt and Grok's own injected - // notice already give two rules for `search_tool`. This adds no third. - assert.equal(sentence.includes(GROK_MCP_SEARCH_TOOL), false); - // One catalogue only: the prefixed names are a rule, not a second list. - for (const tool of mounted) assert.equal(sentence.split(tool).length - 1, tool === mounted[0] ? 2 : 1, tool); - assert.equal(sentence.split(DAIMON_GROK_TOOL_PREFIX).length - 1, 2, "prefix appears as the rule and its one example"); - // The transport prohibition is untouched and still follows the tool sentence. - assert.match(identityEnvelope(rootConfig("/private/org", "grok"), mounted), /Do not seek transport credentials or invoke a transport CLI/u); -}); - -test("the Grok envelope and the pinned worker system prompt state the same route", () => { - const sentence = envelopeToolSentence("grok"); - for (const atom of [DAIMON_GROK_MCP_SERVER, DAIMON_GROK_TOOL_PREFIX, GROK_MCP_INVOKE_TOOL, GROK_MCP_TOOL_NAME_ARGUMENT]) { - assert.ok(DAIMON_GROK_SYSTEM_PROMPT.includes(atom), `system prompt states ${atom}`); - assert.ok(sentence.includes(atom), `envelope states ${atom}`); - } -}); - -test("only Grok gains the prefix rule: every other engine's envelope stays byte-identical", () => { - const unchanged = `Your mounted tools are exactly: ${mounted.join(", ")}. Call them by these names; your instructions may spell them differently. No other tool reaches the newsroom.`; - for (const kind of ["codex", "agy"] as const) assert.equal(envelopeToolSentence(kind), unchanged); - assert.notEqual(envelopeToolSentence("grok"), unchanged); - // An unmounted agent gets no tool sentence at all, on every engine. - for (const kind of ["codex", "agy", "grok"] as const) assert.doesNotMatch(identityEnvelope(rootConfig("/private/org", kind)), /Your mounted tools/u); -}); - test("production dispatcher starts each closed engine intent through Daimon", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-dispatcher-")); const priorPath = process.env.PATH; diff --git a/src/runtime/engineDispatcherIdentity.test.ts b/src/runtime/engineDispatcherIdentity.test.ts new file mode 100644 index 0000000..b82dafa --- /dev/null +++ b/src/runtime/engineDispatcherIdentity.test.ts @@ -0,0 +1,65 @@ +import assert from "node:assert/strict"; +import path from "node:path"; +import test from "node:test"; + +import { identityEnvelope } from "./engineDispatcher.js"; +import { DAIMON_GROK_MCP_SERVER, DAIMON_GROK_SYSTEM_PROMPT, DAIMON_GROK_TOOL_PREFIX, GROK_MCP_INVOKE_TOOL, GROK_MCP_SEARCH_TOOL, GROK_MCP_TOOL_NAME_ARGUMENT, grokDaimonToolName } from "../contracts/grokWorkerContract.js"; +import type { OrganizationRuntimeAgentConfig } from "./organizationRuntime.js"; + +const rootConfig = (root: string, kind: OrganizationRuntimeAgentConfig["engine"]["kind"]): OrganizationRuntimeAgentConfig => ({ + id: `${kind}-agent`, name: kind, instructions: "Reply.", + workspacePath: path.join(root, "workspace", kind), runtimeHomePath: path.join(root, "runtime", kind), + engine: { kind } +}); + +/** + * The envelope's Grok wording is the second naming rule a Grok worker reads, + * after the pinned system prompt. It used to instruct the bare form, which + * Grok 1.0.34 refuses outright as an invalid MCP tool name, so what is asserted + * here is agreement: the same route, stated once, the prefixed form named as + * the only valid one, and no third voice about `search_tool`. + */ +const mounted = ["moltnet_read", "moltnet_send", "memory_search"] as const; +const envelopeToolSentence = (kind: OrganizationRuntimeAgentConfig["engine"]["kind"]): string => + identityEnvelope(rootConfig("/private/org", kind), mounted).split("\n").find((line) => line.startsWith("Your mounted tools are exactly"))!; + +test("the Grok envelope names the prefixed form as the only valid one, once", () => { + const sentence = envelopeToolSentence("grok"); + // The route, asserted: one bare catalogue, one prefix rule, one example. + assert.equal(sentence.includes(`Your mounted tools are exactly: ${mounted.join(", ")}.`), true); + assert.match(sentence, new RegExp(`MCP tool on server ${DAIMON_GROK_MCP_SERVER}`, "u")); + assert.match(sentence, new RegExp(`only valid tool name is ${DAIMON_GROK_TOOL_PREFIX}`, "u")); + assert.match(sentence, new RegExp(`invoke it with ${GROK_MCP_INVOKE_TOOL}, ${GROK_MCP_TOOL_NAME_ARGUMENT} = ${grokDaimonToolName(mounted[0])}`, "u")); + // A bare name is invalid, not merely discouraged: that is the CLI's own verdict. + assert.match(sentence, /A bare name is not a valid MCP tool name and reaches nothing\./u); + assert.match(sentence, /No other tool reaches the newsroom\.$/u); + // What it must never say: the bare names are callable, the agent's own + // instructions are wrong, or a shell reaches the tools. + assert.doesNotMatch(sentence, /Call them by these names/u); + assert.doesNotMatch(sentence, /spell them differently/u); + assert.doesNotMatch(sentence, /shell|terminal|CLI|run_terminal/u); + // Fewer authoritative voices: the pinned prompt and Grok's own injected + // notice already give two rules for `search_tool`. This adds no third. + assert.equal(sentence.includes(GROK_MCP_SEARCH_TOOL), false); + // One catalogue only: the prefixed names are a rule, not a second list. + for (const tool of mounted) assert.equal(sentence.split(tool).length - 1, tool === mounted[0] ? 2 : 1, tool); + assert.equal(sentence.split(DAIMON_GROK_TOOL_PREFIX).length - 1, 2, "prefix appears as the rule and its one example"); + // The transport prohibition is untouched and still follows the tool sentence. + assert.match(identityEnvelope(rootConfig("/private/org", "grok"), mounted), /Do not seek transport credentials or invoke a transport CLI/u); +}); + +test("the Grok envelope and the pinned worker system prompt state the same route", () => { + const sentence = envelopeToolSentence("grok"); + for (const atom of [DAIMON_GROK_MCP_SERVER, DAIMON_GROK_TOOL_PREFIX, GROK_MCP_INVOKE_TOOL, GROK_MCP_TOOL_NAME_ARGUMENT]) { + assert.ok(DAIMON_GROK_SYSTEM_PROMPT.includes(atom), `system prompt states ${atom}`); + assert.ok(sentence.includes(atom), `envelope states ${atom}`); + } +}); + +test("only Grok gains the prefix rule: every other engine's envelope stays byte-identical", () => { + const unchanged = `Your mounted tools are exactly: ${mounted.join(", ")}. Call them by these names; your instructions may spell them differently. No other tool reaches the newsroom.`; + for (const kind of ["codex", "agy"] as const) assert.equal(envelopeToolSentence(kind), unchanged); + assert.notEqual(envelopeToolSentence("grok"), unchanged); + // An unmounted agent gets no tool sentence at all, on every engine. + for (const kind of ["codex", "agy", "grok"] as const) assert.doesNotMatch(identityEnvelope(rootConfig("/private/org", kind)), /Your mounted tools/u); +}); From 964dda31f76f40604445c8805c6af3813e1ec90d Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 00:14:12 +0200 Subject: [PATCH 087/124] fix: prefix the inbox prompt's oversized-payload branch for Grok too --- src/runtime/attentionDispatcher.test.ts | 22 +++++++++++++++++++++- src/runtime/attentionDispatcher.ts | 4 ++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/runtime/attentionDispatcher.test.ts b/src/runtime/attentionDispatcher.test.ts index 3e61593..6e1c0b6 100644 --- a/src/runtime/attentionDispatcher.test.ts +++ b/src/runtime/attentionDispatcher.test.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { AttentionDispatcher } from "./attentionDispatcher.js"; +import { AttentionDispatcher, inboxPrompt } from "./attentionDispatcher.js"; import { DAIMON_GROK_TOOL_PREFIX, grokDaimonToolName } from "../contracts/grokWorkerContract.js"; import { createOrganizationRuntimeHost } from "./organizationRuntimeHost.js"; import { WakeAcceptanceStore, WakeExecutionClaimLostError } from "./wakeAcceptanceStore.js"; @@ -243,3 +243,23 @@ test("every other engine's inbox turn keeps the bare tool names", async () => { assert.equal(text.includes(DAIMON_GROK_TOOL_PREFIX), false); } finally { await f.cleanup(); } }); + +/** + * Every branch of the inbox prompt, not only the one a small delivery takes: + * the oversized-payload fallback is the branch a busy agent meets, and it names + * `daimon_inbox` too. + */ +test("every branch of the inbox prompt names its tools the way the engine can call them", () => { + const delivery = { acceptance_id: "a-1", delivery_id: "d-1", kind: "message", text: "Do the thing", occurred_at: "2026-09-11T00:00:00.000Z" }; + const oversized = { ...delivery, text: "x".repeat(2_000) }; + for (const [messages, budget] of [[[delivery], 12_000], [[oversized], 64], [[oversized], 8]] as const) { + const grok = inboxPrompt(messages, "grok", budget), codex = inboxPrompt(messages, "codex", budget); + // Mutation guard: an unprefixed branch leaves a bare name in the Grok text. + assert.equal(grok.split("daimon_inbox").length - 1, grok.split(DAIMON_GROK_TOOL_PREFIX).length - 1, grok); + assert.ok(grok.includes(grokDaimonToolName("daimon_inbox")), grok); + assert.equal(codex.includes(DAIMON_GROK_TOOL_PREFIX), false, codex); + assert.ok(codex.includes("daimon_inbox"), codex); + } + // The smallest budget is the fallback that only points at the tool. + assert.match(inboxPrompt([oversized], "grok", 8), new RegExp(`exceeds the prompt budget; read it with ${grokDaimonToolName("daimon_inbox")}\\.$`, "u")); +}); diff --git a/src/runtime/attentionDispatcher.ts b/src/runtime/attentionDispatcher.ts index 5d842b5..ad204fc 100644 --- a/src/runtime/attentionDispatcher.ts +++ b/src/runtime/attentionDispatcher.ts @@ -202,7 +202,7 @@ function deliveryBlock(message: unknown, index: number): string | undefined { * cannot mark its work complete — and an unmarked, finished wake is recorded as * deferred. */ -function inboxPrompt(messages: readonly unknown[], engine: OrganizationRuntimeAgentConfig["engine"]["kind"], maxBytes = 12000): string { +export function inboxPrompt(messages: readonly unknown[], engine: OrganizationRuntimeAgentConfig["engine"]["kind"], maxBytes = 12000): string { const body = JSON.stringify(messages); const blocks = messages.map(deliveryBlock).filter((block): block is string => block !== undefined); const tool = (name: string): string => engine === "grok" ? grokDaimonToolName(name) : name; @@ -221,7 +221,7 @@ function inboxPrompt(messages: readonly unknown[], engine: OrganizationRuntimeAg const prefix = `Handle this inbox turn. Use ${tool("daimon_inbox")} for deliveries and remaining allowances. Explicitly call ${tool("daimon_inbox_disposition")} for each handled delivery (complete) or unfinished delivery (defer). Reading or ending this turn never completes a delivery. Deferred work waits for a later external wake.\n`; const prompt = prefix + body; if (Buffer.byteLength(body) > maxBytes || !fits(prompt)) { - return prefix + "The selected payload exceeds the prompt budget; read it with daimon_inbox."; + return prefix + `The selected payload exceeds the prompt budget; read it with ${tool("daimon_inbox")}.`; } return prompt; } From 23b28e97556c58fb8f1b552c19fad7d86cf7fe15 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 00:16:57 +0200 Subject: [PATCH 088/124] fix: forward the MCP session and protocol headers and the GET/DELETE routes through the broker facade --- src/runtime/engineBrokerMcpFacade.test.ts | 231 +++++++++++++++++++++- src/runtime/engineBrokerMcpFacade.ts | 202 ++++++++++++++++++- 2 files changed, 424 insertions(+), 9 deletions(-) diff --git a/src/runtime/engineBrokerMcpFacade.test.ts b/src/runtime/engineBrokerMcpFacade.test.ts index 5244b4f..c19272a 100644 --- a/src/runtime/engineBrokerMcpFacade.test.ts +++ b/src/runtime/engineBrokerMcpFacade.test.ts @@ -1,10 +1,235 @@ import assert from "node:assert/strict"; -import { createServer } from "node:http"; +import { createServer, type Server as HttpServer, type IncomingMessage } from "node:http"; +import { randomUUID } from "node:crypto"; import test from "node:test"; -import { startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; + +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; +import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; +import { ToolListChangedNotificationSchema } from "@modelcontextprotocol/sdk/types.js"; +import { Type } from "@earendil-works/pi-ai"; +import { defineTool } from "@earendil-works/pi-coding-agent"; + +import { createPiToolMcpServer } from "../mcp/toolServer.js"; +import { ENGINE_BROKER_MCP_FACADE_PORT, startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; + +const FACADE_URL = `http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}/mcp`; test("MCP facade routes only valid active capabilities to the registered mount", async () => { let calls=0;const target=createServer((_request,response)=>{calls++;response.writeHead(200,{"content-type":"application/json"});response.end('{"ok":true}');});await new Promise((resolve)=>target.listen(0,"127.0.0.1",resolve));const address=target.address();if(address===null||typeof address==="string")throw new Error(); - const facade=await startEngineBrokerMcpFacade();const token=facade.register("agent","turn",`http://127.0.0.1:${address.port}/mcp`);const call=(value:string)=>fetch("http://127.0.0.1:43124/mcp",{method:"POST",headers:{authorization:`Bearer ${value}`,"content-type":"application/json"},body:"{}"}); + const facade=await startEngineBrokerMcpFacade();const token=facade.register("agent","turn",`http://127.0.0.1:${address.port}/mcp`);const call=(value:string)=>fetch(FACADE_URL,{method:"POST",headers:{authorization:`Bearer ${value}`,"content-type":"application/json"},body:"{}"}); try{assert.equal((await call("wrong-token-abcdefghijklmnopqrstuvwxyz0123456789")).status,403);assert.equal((await call(token)).status,200);assert.equal(calls,1);facade.revoke("turn");assert.equal((await call(token)).status,403);assert.equal(calls,1);}finally{await facade.close();await new Promise((resolve)=>target.close(()=>resolve()));} }); + +/** + * The brokered worker's real route: a real Daimon MCP mount behind a real + * Streamable HTTP transport, reached by a real MCP client through the facade. + * Asserting that a header is copied would pass while the route stayed broken, + * so every case below drives the transport end to end. + */ +type Rig = Readonly<{ + facade: Awaited>; + mount: HttpServer; + transport: StreamableHTTPServerTransport; + server: ReturnType; + capability: string; + observed: IncomingMessage[]; + close: () => Promise; +}>; + +const echoTool = defineTool({ + name: "moltnet_read", + label: "Read a scoped Moltnet surface", + description: "Reads the fixture room.", + parameters: Type.Object({ target: Type.String() }, { additionalProperties: false }), + async execute(_toolCallId: string, params: { target: string }) { + return { content: [{ type: "text" as const, text: `read ${params.target}` }], details: { target: params.target } }; + } +}); + +const startRig = async (): Promise => { + const server = createPiToolMcpServer([echoTool], {}); + const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: () => randomUUID() }); + await server.connect(transport); + const observed: IncomingMessage[] = []; + const mount = createServer((request, response) => { + observed.push(request); + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.on("end", () => { + const raw = Buffer.concat(chunks); + let parsed: unknown; + try { parsed = raw.length === 0 ? undefined : JSON.parse(raw.toString("utf8")); } catch { parsed = undefined; } + void transport.handleRequest(request, response, parsed); + }); + }); + await new Promise((resolve) => mount.listen(0, "127.0.0.1", resolve)); + const address = mount.address(); + if (address === null || typeof address === "string") throw new Error("mount address unavailable"); + const facade = await startEngineBrokerMcpFacade(); + const capability = facade.register("alpha", "turn-1", `http://127.0.0.1:${address.port}/mcp`); + return { + facade, mount, transport, server, capability, observed, + close: async () => { + facade.revoke("turn-1"); + await facade.close(); + await new Promise((resolve) => mount.close(() => resolve())); + await transport.close().catch(() => undefined); + await server.close().catch(() => undefined); + } + }; +}; + +const connectClient = async (capability: string): Promise<{ client: Client; transport: StreamableHTTPClientTransport }> => { + const transport = new StreamableHTTPClientTransport(new URL(FACADE_URL), { + requestInit: { headers: { authorization: `Bearer ${capability}` } } + }); + const client = new Client({ name: "daimon-facade-test-client", version: "0.1.0" }); + await client.connect(transport); + return { client, transport }; +}; + +test("a brokered worker completes the whole MCP handshake through the facade and calls a mounted tool", async () => { + const rig = await startRig(); + try { + // connect() is initialize + notifications/initialized: before the session + // header was forwarded the notification came back HTTP 400. + const { client, transport } = await connectClient(rig.capability); + try { + assert.equal(typeof transport.sessionId, "string", "the mount's session id must reach the client"); + const listed = await client.listTools(); + assert.deepEqual(listed.tools.map((tool) => tool.name), ["moltnet_read"]); + const called = await client.callTool({ name: "moltnet_read", arguments: { target: "room:desk" } }); + assert.deepEqual(called.structuredContent, { target: "room:desk" }); + assert.equal(called.isError, undefined); + } finally { + await client.close(); + } + } finally { + await rig.close(); + } +}); + +test("the facade carries the mount's server-initiated SSE stream, which only the GET route provides", async () => { + const rig = await startRig(); + try { + const { client, transport } = await connectClient(rig.capability); + const notified = new Promise((resolve) => client.setNotificationHandler(ToolListChangedNotificationSchema, () => resolve())); + try { + // The standalone GET stream is the only route a server notification can + // take; a POST-only facade answers it 403 and this never arrives. + await new Promise((resolve) => setTimeout(resolve, 150)); + rig.server.sendToolListChanged(); + await Promise.race([ + notified, + new Promise((_resolve, reject) => setTimeout(() => reject(new Error("no server notification reached the client")), 4_000)) + ]); + assert.ok(rig.observed.some((request) => request.method === "GET"), "the mount must have seen the GET stream"); + assert.equal(typeof transport.sessionId, "string"); + } finally { + await client.close(); + } + } finally { + await rig.close(); + } +}); + +test("the facade carries the session-closing DELETE, and the mount then refuses the stale session", async () => { + const rig = await startRig(); + try { + const { client, transport } = await connectClient(rig.capability); + const sessionId = transport.sessionId; + assert.equal(typeof sessionId, "string"); + await transport.terminateSession(); + assert.equal(transport.sessionId, undefined, "DELETE must be accepted, not 403ed"); + assert.ok(rig.observed.some((request) => request.method === "DELETE"), "the mount must have seen the DELETE"); + const stale = await fetch(FACADE_URL, { + method: "POST", + headers: { + authorization: `Bearer ${rig.capability}`, + "content-type": "application/json", + accept: "application/json, text/event-stream", + "mcp-session-id": sessionId! + }, + body: JSON.stringify({ jsonrpc: "2.0", id: 9, method: "tools/list", params: {} }) + }); + assert.ok(stale.status >= 400, `a terminated session must not still route (got ${stale.status})`); + await stale.body?.cancel(); + await client.close().catch(() => undefined); + } finally { + await rig.close(); + } +}); + +test("the facade forwards a closed header allowlist and never the worker's bearer", async () => { + const rig = await startRig(); + try { + const { client } = await connectClient(rig.capability); + await client.listTools(); + await client.close(); + const forwarded = rig.observed.flatMap((request) => Object.keys(request.headers)); + assert.equal(forwarded.includes("authorization"), false, "the turn capability must never reach the mount"); + assert.equal(forwarded.includes("cookie"), false); + const allowed = new Set(["host", "connection", "content-length", "transfer-encoding", "accept-encoding", "accept-language", "user-agent", "content-type", "accept", "mcp-session-id", "mcp-protocol-version", "last-event-id", + // undici's own outbound headers, set by the facade's fetch rather than forwarded from the worker. + "sec-fetch-mode"]); + const unexpected = [...new Set(forwarded)].filter((name) => !allowed.has(name)); + assert.deepEqual(unexpected, [], `unexpected headers reached the mount: ${unexpected.join(",")}`); + } finally { + await rig.close(); + } +}); + +test("the facade withholds a mount response header that is not on the allowlist", async () => { + const target = createServer((_request, response) => { + response.writeHead(200, { + "content-type": "application/json", + "mcp-session-id": "session-from-mount", + "set-cookie": "leak=1", + "www-authenticate": "Bearer realm=\"mount\"", + "x-mount-internal": "private" + }); + response.end('{"ok":true}'); + }); + await new Promise((resolve) => target.listen(0, "127.0.0.1", resolve)); + const address = target.address(); + if (address === null || typeof address === "string") throw new Error("target address unavailable"); + const facade = await startEngineBrokerMcpFacade(); + const capability = facade.register("alpha", "turn-1", `http://127.0.0.1:${address.port}/mcp`); + try { + const answered = await fetch(FACADE_URL, { + method: "POST", + headers: { authorization: `Bearer ${capability}`, "content-type": "application/json" }, + body: "{}" + }); + assert.equal(answered.status, 200); + assert.equal(answered.headers.get("mcp-session-id"), "session-from-mount"); + assert.equal(answered.headers.get("cache-control"), "no-store"); + assert.equal(answered.headers.get("set-cookie"), null); + assert.equal(answered.headers.get("www-authenticate"), null); + assert.equal(answered.headers.get("x-mount-internal"), null); + await answered.body?.cancel(); + } finally { + facade.revoke("turn-1"); + await facade.close(); + await new Promise((resolve) => target.close(() => resolve())); + } +}); + +test("the facade still refuses every route and method outside the MCP surface", async () => { + const facade = await startEngineBrokerMcpFacade(); + const capability = facade.register("alpha", "turn-1", "http://127.0.0.1:1/mcp"); + const call = (method: string, path: string) => fetch(`http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}${path}`, { + method, headers: { authorization: `Bearer ${capability}` } + }); + try { + assert.equal((await call("GET", "/")).status, 403); + assert.equal((await call("GET", "/mcp?probe=1")).status, 403); + assert.equal((await call("PUT", "/mcp")).status, 403); + assert.equal((await call("PATCH", "/mcp")).status, 403); + assert.equal((await fetch(`http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}/mcp`, { method: "GET" })).status, 403); + } finally { + facade.revoke("turn-1"); + await facade.close(); + } +}); diff --git a/src/runtime/engineBrokerMcpFacade.ts b/src/runtime/engineBrokerMcpFacade.ts index 156506b..ee1defd 100644 --- a/src/runtime/engineBrokerMcpFacade.ts +++ b/src/runtime/engineBrokerMcpFacade.ts @@ -1,10 +1,200 @@ -import { createServer } from "node:http"; +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import { Readable } from "node:stream"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; +/** + * The brokered worker's only route to its own per-wake Daimon MCP mount. The + * worker holds a turn capability and nothing else: it never learns the mount's + * address, and the mount never learns the capability. Every header crossing + * either way is rebuilt from a closed allowlist, so this stays a boundary and + * not a transparent proxy — a blanket passthrough would hand the mount the + * worker's bearer and hand the worker whatever the mount chose to say. + * + * The allowlists are exactly the Streamable HTTP transport's own routing + * headers. Anything outside them (authorization above all, cookies, auth + * challenges, forwarding and tracing headers) is dropped in both directions. + * + * Client -> mount: + * - `content-type`: the JSON-RPC body's media type; the mount refuses a POST + * without it. + * - `accept`: the transport negotiates `application/json, text/event-stream` + * per request and the mount answers 406 when a POST does not accept both. + * - `mcp-session-id`: the opaque session the mount issued on `initialize`. + * Dropping it made the mount answer every later request with HTTP 400 + * `Mcp-Session-Id header is required`, so no tool was ever reachable. It is + * a routing value, not a secret — and not a value to log either. + * - `mcp-protocol-version`: the version the handshake settled on. The mount + * validates it and otherwise assumes a default that can disagree with what + * the client negotiated. + * - `last-event-id`: SSE resumability. A reconnecting stream replays from the + * last event it saw; without it the mount cannot tell where to resume. + * + * Mount -> client: + * - `content-type`: tells the client whether it got JSON or an SSE stream. + * - `mcp-session-id`: the id minted on `initialize`. The client must learn it + * or it can never make a second request. + * - `mcp-protocol-version`: the version the mount confirms for the session. + * - `cache-control: no-store` is the facade's own, not the mount's. + * + * Methods are the three the transport uses: POST for JSON-RPC, GET for the + * server-to-client SSE stream (notifications and progress arrive only there), + * and DELETE to end a session. POST alone left the GET stream answering 403. + */ +const FORWARDED_REQUEST_HEADERS = ["content-type", "accept", "mcp-session-id", "mcp-protocol-version", "last-event-id"] as const; +const FORWARDED_RESPONSE_HEADERS = ["content-type", "mcp-session-id", "mcp-protocol-version"] as const; +const FORWARDED_METHODS = new Set(["POST", "GET", "DELETE"]); +const MAX_REQUEST_BYTES = 1024 * 1024; +export const ENGINE_BROKER_MCP_FACADE_PORT = 43_124; + +class FacadeRefusal extends Error {} + export async function startEngineBrokerMcpFacade() { - const capabilities=new EngineBrokerCapabilities();const targets=new Map(); - const server=createServer((request,response)=>{void(async()=>{try{if(request.url!=="/mcp"||request.method!=="POST")throw new Error();const match=request.headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u);if(!match)throw new Error();const scope=capabilities.authorizeToken(match[1]!);if(!scope)throw new Error();const target=targets.get(scope.turnId);if(!target)throw new Error();const body=await bounded(request);const payload=body.buffer.slice(body.byteOffset,body.byteOffset+body.byteLength) as ArrayBuffer;const upstream=await fetch(target,{method:"POST",headers:{"content-type":request.headers["content-type"]??"application/json","accept":request.headers.accept??"application/json, text/event-stream"},body:payload});response.writeHead(upstream.status,{"content-type":upstream.headers.get("content-type")??"application/json","cache-control":"no-store"});response.end(Buffer.from(await upstream.arrayBuffer()));}catch{response.writeHead(403,{"content-type":"application/json","cache-control":"no-store"});response.end('{"error":"forbidden"}');}})();}); - await new Promise((resolve,reject)=>{server.once("error",reject);server.listen(43_124,"127.0.0.1",()=>{server.off("error",reject);resolve();});}); - return {register(agentId:string,turnId:string,endpoint:string){const url=new URL(endpoint);if(url.protocol!=="http:"||url.hostname!=="127.0.0.1"||url.pathname!=="/mcp")throw new TypeError("invalid scoped MCP mount");if(targets.has(turnId))throw new Error("MCP turn already registered");targets.set(turnId,url.href);return capabilities.issue(agentId,turnId,15*60_000,128);},revoke(turnId:string){targets.delete(turnId);capabilities.revoke(turnId);},close:()=>new Promise((resolve,reject)=>server.close((error)=>error?reject(error):resolve()))}; + const capabilities = new EngineBrokerCapabilities(); + const targets = new Map(); + /** In-flight upstream calls per turn, so a revoke or a close tears down any open SSE tunnel. */ + const inflight = new Map>(); + + const server = createServer((request, response) => { + void route(request, response).catch((error: unknown) => { + if (response.headersSent || response.destroyed) { response.destroy(); return; } + const status = error instanceof FacadeRefusal ? 403 : 502; + const body = error instanceof FacadeRefusal ? '{"error":"forbidden"}' : '{"error":"bad_gateway"}'; + response.writeHead(status, { "content-type": "application/json", "cache-control": "no-store" }); + response.end(body); + }); + }); + + async function route(request: IncomingMessage, response: ServerResponse): Promise { + const method = request.method ?? ""; + if (request.url !== "/mcp" || !FORWARDED_METHODS.has(method)) throw new FacadeRefusal(); + const match = request.headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u); + if (!match) throw new FacadeRefusal(); + const scope = capabilities.authorizeToken(match[1]!); + if (!scope) throw new FacadeRefusal(); + const target = targets.get(scope.turnId); + if (target === undefined) throw new FacadeRefusal(); + + // Only POST carries a JSON-RPC body; drain anything else so the socket + // never stalls waiting for a body the facade will not forward. + const body = method === "POST" ? await bounded(request) : (request.resume(), undefined); + const payload = body === undefined ? undefined : (body.buffer.slice(body.byteOffset, body.byteOffset + body.byteLength) as ArrayBuffer); + + const controller = new AbortController(); + const open = inflight.get(scope.turnId) ?? new Set(); + open.add(controller); + inflight.set(scope.turnId, open); + const abort = (): void => controller.abort(); + response.on("close", abort); + try { + await forward(target, method, headersFor(method, request), payload, controller.signal, response); + } finally { + response.off("close", abort); + open.delete(controller); + if (open.size === 0) inflight.delete(scope.turnId); + } + } + + function endTurnStreams(turnId: string): void { + for (const controller of inflight.get(turnId) ?? []) controller.abort(); + inflight.delete(turnId); + } + + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(ENGINE_BROKER_MCP_FACADE_PORT, "127.0.0.1", () => { server.off("error", reject); resolve(); }); + }); + + return { + register(agentId: string, turnId: string, endpoint: string): string { + const url = new URL(endpoint); + if (url.protocol !== "http:" || url.hostname !== "127.0.0.1" || url.pathname !== "/mcp") throw new TypeError("invalid scoped MCP mount"); + if (targets.has(turnId)) throw new Error("MCP turn already registered"); + targets.set(turnId, url.href); + return capabilities.issue(agentId, turnId, 15 * 60_000, 128); + }, + revoke(turnId: string): void { + targets.delete(turnId); + capabilities.revoke(turnId); + endTurnStreams(turnId); + }, + close: async (): Promise => { + for (const turnId of [...inflight.keys()]) endTurnStreams(turnId); + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + // A GET SSE tunnel keeps its socket open indefinitely, and `close` + // only stops accepting; without this a shutdown would hang on it. + server.closeAllConnections(); + }); + } + }; +} + +/** The client -> mount allowlist, with the two defaults the mount requires of a POST. */ +function headersFor(method: string, request: IncomingMessage): Record { + const headers: Record = {}; + for (const name of FORWARDED_REQUEST_HEADERS) { + const value = request.headers[name]; + if (typeof value === "string" && value.length > 0) headers[name] = value; + } + if (method === "POST") { + headers["content-type"] ??= "application/json"; + headers["accept"] ??= "application/json, text/event-stream"; + } else { + delete headers["content-type"]; + } + return headers; +} + +/** + * Streams the exchange rather than buffering it: a GET stream stays open for + * the whole session, and a buffered POST would withhold progress + * notifications until the call had already finished. + */ +async function forward( + target: string, + method: string, + headers: Record, + body: ArrayBuffer | undefined, + signal: AbortSignal, + response: ServerResponse +): Promise { + const upstream = await fetch(target, { method, headers, body, signal, redirect: "manual" }); + // MCP never redirects, and following one would let the mount aim the facade + // at a host the capability was never scoped to. `manual` also reports an + // opaque redirect as status 0, which is not a status to relay at all. + const relayable = (upstream.status >= 200 && upstream.status < 300) || (upstream.status >= 400 && upstream.status <= 599); + if (!relayable) throw new Error("unexpected MCP mount status"); + const outbound: Record = { "cache-control": "no-store" }; + for (const name of FORWARDED_RESPONSE_HEADERS) { + const value = upstream.headers.get(name); + if (value !== null && value.length > 0) outbound[name] = value; + } + outbound["content-type"] ??= "application/json"; + response.writeHead(upstream.status, outbound); + if (upstream.body === null) { response.end(); return; } + const stream = Readable.fromWeb(upstream.body as Parameters[0]); + try { + for await (const chunk of stream) { + if (!response.write(chunk as Uint8Array)) await new Promise((resolve) => response.once("drain", resolve)); + } + response.end(); + } catch { + // The client hung up or the mount's stream broke: tear the tunnel down + // rather than leaving a half-written response open. + response.destroy(); + } finally { + stream.destroy(); + } +} + +async function bounded(request: AsyncIterable): Promise { + const chunks: Buffer[] = []; + let bytes = 0; + for await (const chunk of request) { + const value = Buffer.from(chunk as Uint8Array); + bytes += value.length; + if (bytes > MAX_REQUEST_BYTES) throw new FacadeRefusal(); + chunks.push(value); + } + return Buffer.concat(chunks); } -async function bounded(request:AsyncIterable):Promise{const chunks:Buffer[]=[];let bytes=0;for await(const chunk of request){const value=Buffer.from(chunk as Uint8Array);bytes+=value.length;if(bytes>1024*1024)throw new Error();chunks.push(value);}return Buffer.concat(chunks);} From 8d5c26c2cee1e486fca02f2df326282c67373ebf Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 00:16:57 +0200 Subject: [PATCH 089/124] docs: record the broker MCP facade's closed header allowlist and supported methods --- src/runtime/AGENTS.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 1860ba4..6276db1 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -179,6 +179,26 @@ model declares it, so the declared effort is the model's single `reasoning_efforts` entry. HTTP MCP needs CA certificates in the image even for a loopback `http://` URL ("Failed to build HTTP client"). +`engineBrokerMcpFacade.ts` is the worker's only route to its per-wake MCP mount +and rebuilds every header from a closed allowlist in both directions, so the +worker's bearer never reaches the mount and no mount header reaches the worker +uninvited. That allowlist must include the Streamable HTTP transport's own +routing headers or the route does not exist: forwarding only +`content-type`/`accept` destroyed `Mcp-Session-Id`, so `initialize` returned 200 +while every request after it — `notifications/initialized`, `tools/list`, +`tools/call` — came back HTTP 400 `Mcp-Session-Id header is required`, and the +model saw `search_tool` answer `{"results":[],"total_hidden_tools":0,"status": +"partial"}`. Client to mount: `content-type`, `accept`, `mcp-session-id`, +`mcp-protocol-version`, `last-event-id`. Mount to client: `content-type`, +`mcp-session-id`, `mcp-protocol-version`, plus the facade's own +`cache-control: no-store`. The session id is an opaque routing value and is +never logged or ledgered. The facade also carries the three methods the +transport uses — POST, the standalone `GET` SSE stream that is the only route a +server notification or progress frame can take, and the `DELETE` that ends a +session — and streams each body rather than buffering it, because a GET tunnel +stays open for the whole session. Never widen it into a transparent proxy: the +whole point of the boundary is that the allowlist is closed. + Worker `GROK_HOME` layout the deployment must provision (attested before every turn by `grokWorkerHomeAttestation.ts`, recorded in `GROK_ENGINE_BROKER.worker.home`): `$GROK_HOME` and `$GROK_HOME/sessions` `root: 1771`; `config.toml`, From a43d62c1b977b016f633d0aa6821c84271f28498 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 00:18:03 +0200 Subject: [PATCH 090/124] fix: name the transport send tool the way Grok can call it --- src/runtime/engineDispatcher.test.ts | 2 +- src/runtime/engineDispatcher.ts | 18 ++++++++++-- src/runtime/engineDispatcherIdentity.test.ts | 29 ++++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/runtime/engineDispatcher.test.ts b/src/runtime/engineDispatcher.test.ts index 0dae9a8..ca1b5e1 100644 --- a/src/runtime/engineDispatcher.test.ts +++ b/src/runtime/engineDispatcher.test.ts @@ -290,7 +290,7 @@ test("Daimon frames one escaped identity envelope for every production engine", const envelope = JSON.stringify({ id: config.id, name: identity.name, instructions: identity.instructions }); assert.equal(result.text.split(envelope).length - 1, 1); assert.match(result.text, //); - assert.match(result.text, /Colleagues only hear you when you call moltnet_send/u); + assert.ok(result.text.includes(`Colleagues only hear you when you call ${kind === "grok" ? "daimon__moltnet_send" : "moltnet_send"};`), `${kind} must name the send tool the way it can call it`); assert.match(result.text, /Do not seek transport credentials or invoke a transport CLI/u); assert.match(result.text, /payload/); await handle.stop(); diff --git a/src/runtime/engineDispatcher.ts b/src/runtime/engineDispatcher.ts index 40b06f1..9b72149 100644 --- a/src/runtime/engineDispatcher.ts +++ b/src/runtime/engineDispatcher.ts @@ -1,5 +1,5 @@ import { attentionTools, type AttentionRegistry } from "./attention.js"; -import { grokMountedToolNamingRule } from "../contracts/grokWorkerContract.js"; +import { grokDaimonToolName, grokMountedToolNamingRule } from "../contracts/grokWorkerContract.js"; import path from "node:path"; import type { AgentHandle } from "../core/types.js"; @@ -207,12 +207,26 @@ export function identityEnvelope(agent: OrganizationRuntimeAgentConfig, mountedT + "your instructions may spell them differently.") + " No other tool reaches the newsroom." ]), - "Colleagues only hear you when you call moltnet_send; your terminal response is a private note to the runtime, not a message to anyone — keep it to one line or leave it empty. " + // The one tool that reaches colleagues is named the way this engine can + // call it. Meaning and prohibition are unchanged; only the spelling is. + `Colleagues only hear you when you call ${engineToolName(agent, "moltnet_send")}; your terminal response is a private note to the runtime, not a message to anyone — keep it to one line or leave it empty. ` + "Do not seek transport credentials or invoke a transport CLI unless the caller explicitly mounted an authenticated transport tool.", "The following is the current wake event." ].join("\n") + "\n"; } +/** + * One Daimon tool name, spelled the way this agent's engine accepts it. + * + * On Grok the bare form is refused as an invalid MCP tool name, so any + * engine-facing sentence that *names* a tool renders it through the contract's + * `grokDaimonToolName`; every other engine keeps the bare name byte for byte. + * Grok's own native tools (`read_file`, `search_tool`, `use_tool`) are not + * Daimon tools and never take the prefix. + */ +const engineToolName = (agent: OrganizationRuntimeAgentConfig, tool: string): string => + agent.engine.kind === "grok" ? grokDaimonToolName(tool) : tool; + function cliHarness( agent: OrganizationRuntimeAgentConfig, sessionFactory: ReturnType, diff --git a/src/runtime/engineDispatcherIdentity.test.ts b/src/runtime/engineDispatcherIdentity.test.ts index b82dafa..13b1607 100644 --- a/src/runtime/engineDispatcherIdentity.test.ts +++ b/src/runtime/engineDispatcherIdentity.test.ts @@ -56,10 +56,39 @@ test("the Grok envelope and the pinned worker system prompt state the same route } }); +/** + * The transport sentence names the one tool an agent needs to reach its + * colleagues. A bare `moltnet_send` one line under "a bare name is not a valid + * MCP tool name and reaches nothing" is the same self-contradiction, on the + * tool that matters most. + */ +test("the transport sentence names the send tool the way Grok can call it, prohibition unchanged", () => { + const grok = identityEnvelope(rootConfig("/private/org", "grok"), mounted); + const transport = grok.split("\n").find((line) => line.startsWith("Colleagues only hear you"))!; + // Mutation guard: un-prefixing this name leaves the bare form the line above declares invalid. + assert.ok(transport.includes(`you call ${grokDaimonToolName("moltnet_send")};`), transport); + assert.equal(new RegExp(`(? { const unchanged = `Your mounted tools are exactly: ${mounted.join(", ")}. Call them by these names; your instructions may spell them differently. No other tool reaches the newsroom.`; for (const kind of ["codex", "agy"] as const) assert.equal(envelopeToolSentence(kind), unchanged); assert.notEqual(envelopeToolSentence("grok"), unchanged); // An unmounted agent gets no tool sentence at all, on every engine. for (const kind of ["codex", "agy", "grok"] as const) assert.doesNotMatch(identityEnvelope(rootConfig("/private/org", kind)), /Your mounted tools/u); + // The transport sentence keeps its bare spelling on every other engine. + const transport = "Colleagues only hear you when you call moltnet_send; your terminal response is a private note to the runtime, not a message to anyone — keep it to one line or leave it empty. Do not seek transport credentials or invoke a transport CLI unless the caller explicitly mounted an authenticated transport tool."; + for (const kind of ["codex", "agy"] as const) { + assert.ok(identityEnvelope(rootConfig("/private/org", kind), mounted).includes(`\n${transport}\n`), kind); + assert.equal(identityEnvelope(rootConfig("/private/org", kind), mounted).includes(DAIMON_GROK_TOOL_PREFIX), false, kind); + } }); From 9d773aede5b51c6559af29fe8d42c01f8f61aab0 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 00:38:11 +0200 Subject: [PATCH 091/124] test: share one facade across the broker MCP facade tests and cover revoke and shutdown teardown --- src/runtime/engineBrokerMcpFacade.test.ts | 136 +++++++++++++++++----- 1 file changed, 107 insertions(+), 29 deletions(-) diff --git a/src/runtime/engineBrokerMcpFacade.test.ts b/src/runtime/engineBrokerMcpFacade.test.ts index c19272a..95d0b65 100644 --- a/src/runtime/engineBrokerMcpFacade.test.ts +++ b/src/runtime/engineBrokerMcpFacade.test.ts @@ -14,11 +14,40 @@ import { createPiToolMcpServer } from "../mcp/toolServer.js"; import { ENGINE_BROKER_MCP_FACADE_PORT, startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; const FACADE_URL = `http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}/mcp`; +type Facade = Awaited>; + +/** + * One facade serves every turn of a broker, so the tests share one too. It + * also keeps the fixed port free: a facade per test would leave the HTTP + * client pooling a socket onto a server that no longer exists. + */ +let shared: Facade | undefined; +const sharedFacade = async (): Promise => (shared ??= await startFacade()); +const releaseShared = async (): Promise => { const facade = shared; shared = undefined; if (facade) await facade.close(); }; +test.after(releaseShared); + +/** + * Closing a facade destroys its sockets, and the port is fixed, so the HTTP + * client can still hold a pooled connection to the server that just went away. + * That is a test-harness artifact — one facade outlives a whole broker — so a + * fresh facade is probed until a refusal proves the route is live again. + */ +const startFacade = async (): Promise => { + const facade = await startEngineBrokerMcpFacade(); + for (let attempt = 0; attempt < 20; attempt += 1) { + try { + const probe = await fetch(FACADE_URL, { method: "PUT" }); + await probe.body?.cancel(); + if (probe.status === 403) return facade; + } catch { /* a pooled socket onto the previous facade: try the next one */ } + } + throw new Error("facade did not answer after starting"); +}; test("MCP facade routes only valid active capabilities to the registered mount", async () => { let calls=0;const target=createServer((_request,response)=>{calls++;response.writeHead(200,{"content-type":"application/json"});response.end('{"ok":true}');});await new Promise((resolve)=>target.listen(0,"127.0.0.1",resolve));const address=target.address();if(address===null||typeof address==="string")throw new Error(); - const facade=await startEngineBrokerMcpFacade();const token=facade.register("agent","turn",`http://127.0.0.1:${address.port}/mcp`);const call=(value:string)=>fetch(FACADE_URL,{method:"POST",headers:{authorization:`Bearer ${value}`,"content-type":"application/json"},body:"{}"}); - try{assert.equal((await call("wrong-token-abcdefghijklmnopqrstuvwxyz0123456789")).status,403);assert.equal((await call(token)).status,200);assert.equal(calls,1);facade.revoke("turn");assert.equal((await call(token)).status,403);assert.equal(calls,1);}finally{await facade.close();await new Promise((resolve)=>target.close(()=>resolve()));} + const facade=await sharedFacade();const token=facade.register("agent","turn-capabilities",`http://127.0.0.1:${address.port}/mcp`);const call=(value:string)=>fetch(FACADE_URL,{method:"POST",headers:{authorization:`Bearer ${value}`,"content-type":"application/json"},body:"{}"}); + try{assert.equal((await call("wrong-token-abcdefghijklmnopqrstuvwxyz0123456789")).status,403);assert.equal((await call(token)).status,200);assert.equal(calls,1);facade.revoke("turn-capabilities");assert.equal((await call(token)).status,403);assert.equal(calls,1);}finally{facade.revoke("turn-capabilities");await new Promise((resolve)=>target.close(()=>resolve()));} }); /** @@ -28,12 +57,13 @@ test("MCP facade routes only valid active capabilities to the registered mount", * so every case below drives the transport end to end. */ type Rig = Readonly<{ - facade: Awaited>; - mount: HttpServer; - transport: StreamableHTTPServerTransport; + facade: Facade; + turnId: string; server: ReturnType; capability: string; observed: IncomingMessage[]; + /** Resolves when the mount's standalone GET stream is torn down. */ + getStreamClosed: Promise; close: () => Promise; }>; @@ -47,13 +77,20 @@ const echoTool = defineTool({ } }); -const startRig = async (): Promise => { +let turns = 0; + +const startRig = async (facade?: Facade): Promise => { + const host = facade ?? await sharedFacade(); + const turnId = `turn-${++turns}`; const server = createPiToolMcpServer([echoTool], {}); const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: () => randomUUID() }); await server.connect(transport); const observed: IncomingMessage[] = []; + let noteGetStreamClosed = (): void => undefined; + const getStreamClosed = new Promise((resolve) => { noteGetStreamClosed = resolve; }); const mount = createServer((request, response) => { observed.push(request); + if (request.method === "GET") response.on("close", () => noteGetStreamClosed()); const chunks: Buffer[] = []; request.on("data", (chunk: Buffer) => chunks.push(chunk)); request.on("end", () => { @@ -66,20 +103,23 @@ const startRig = async (): Promise => { await new Promise((resolve) => mount.listen(0, "127.0.0.1", resolve)); const address = mount.address(); if (address === null || typeof address === "string") throw new Error("mount address unavailable"); - const facade = await startEngineBrokerMcpFacade(); - const capability = facade.register("alpha", "turn-1", `http://127.0.0.1:${address.port}/mcp`); + const capability = host.register("alpha", turnId, `http://127.0.0.1:${address.port}/mcp`); return { - facade, mount, transport, server, capability, observed, + facade: host, turnId, server, capability, observed, getStreamClosed, close: async () => { - facade.revoke("turn-1"); - await facade.close(); - await new Promise((resolve) => mount.close(() => resolve())); - await transport.close().catch(() => undefined); - await server.close().catch(() => undefined); + host.revoke(turnId); + await closeMount(mount, transport, server); } }; }; +const closeMount = async (mount: HttpServer, transport: StreamableHTTPServerTransport, server: ReturnType): Promise => { + mount.closeAllConnections(); + await new Promise((resolve) => mount.close(() => resolve())); + await transport.close().catch(() => undefined); + await server.close().catch(() => undefined); +}; + const connectClient = async (capability: string): Promise<{ client: Client; transport: StreamableHTTPClientTransport }> => { const transport = new StreamableHTTPClientTransport(new URL(FACADE_URL), { requestInit: { headers: { authorization: `Bearer ${capability}` } } @@ -89,6 +129,15 @@ const connectClient = async (capability: string): Promise<{ client: Client; tran return { client, transport }; }; +const withDeadline = async (work: Promise, ms: number, reason: string): Promise => { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([work, new Promise((_resolve, reject) => { timer = setTimeout(() => reject(new Error(reason)), ms); })]); + } finally { + if (timer) clearTimeout(timer); + } +}; + test("a brokered worker completes the whole MCP handshake through the facade and calls a mounted tool", async () => { const rig = await startRig(); try { @@ -120,10 +169,7 @@ test("the facade carries the mount's server-initiated SSE stream, which only the // take; a POST-only facade answers it 403 and this never arrives. await new Promise((resolve) => setTimeout(resolve, 150)); rig.server.sendToolListChanged(); - await Promise.race([ - notified, - new Promise((_resolve, reject) => setTimeout(() => reject(new Error("no server notification reached the client")), 4_000)) - ]); + await withDeadline(notified, 4_000, "no server notification reached the client"); assert.ok(rig.observed.some((request) => request.method === "GET"), "the mount must have seen the GET stream"); assert.equal(typeof transport.sessionId, "string"); } finally { @@ -170,9 +216,12 @@ test("the facade forwards a closed header allowlist and never the worker's beare const forwarded = rig.observed.flatMap((request) => Object.keys(request.headers)); assert.equal(forwarded.includes("authorization"), false, "the turn capability must never reach the mount"); assert.equal(forwarded.includes("cookie"), false); - const allowed = new Set(["host", "connection", "content-length", "transfer-encoding", "accept-encoding", "accept-language", "user-agent", "content-type", "accept", "mcp-session-id", "mcp-protocol-version", "last-event-id", - // undici's own outbound headers, set by the facade's fetch rather than forwarded from the worker. - "sec-fetch-mode"]); + const allowed = new Set([ + "host", "connection", "content-length", "transfer-encoding", "accept-encoding", "accept-language", "user-agent", + "content-type", "accept", "mcp-session-id", "mcp-protocol-version", "last-event-id", + // undici's own outbound header, set by the facade's fetch rather than forwarded from the worker. + "sec-fetch-mode" + ]); const unexpected = [...new Set(forwarded)].filter((name) => !allowed.has(name)); assert.deepEqual(unexpected, [], `unexpected headers reached the mount: ${unexpected.join(",")}`); } finally { @@ -194,8 +243,8 @@ test("the facade withholds a mount response header that is not on the allowlist" await new Promise((resolve) => target.listen(0, "127.0.0.1", resolve)); const address = target.address(); if (address === null || typeof address === "string") throw new Error("target address unavailable"); - const facade = await startEngineBrokerMcpFacade(); - const capability = facade.register("alpha", "turn-1", `http://127.0.0.1:${address.port}/mcp`); + const facade = await sharedFacade(); + const capability = facade.register("alpha", "turn-response-headers", `http://127.0.0.1:${address.port}/mcp`); try { const answered = await fetch(FACADE_URL, { method: "POST", @@ -210,15 +259,14 @@ test("the facade withholds a mount response header that is not on the allowlist" assert.equal(answered.headers.get("x-mount-internal"), null); await answered.body?.cancel(); } finally { - facade.revoke("turn-1"); - await facade.close(); + facade.revoke("turn-response-headers"); await new Promise((resolve) => target.close(() => resolve())); } }); test("the facade still refuses every route and method outside the MCP surface", async () => { - const facade = await startEngineBrokerMcpFacade(); - const capability = facade.register("alpha", "turn-1", "http://127.0.0.1:1/mcp"); + const facade = await sharedFacade(); + const capability = facade.register("alpha", "turn-refusals", "http://127.0.0.1:1/mcp"); const call = (method: string, path: string) => fetch(`http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}${path}`, { method, headers: { authorization: `Bearer ${capability}` } }); @@ -229,7 +277,37 @@ test("the facade still refuses every route and method outside the MCP surface", assert.equal((await call("PATCH", "/mcp")).status, 403); assert.equal((await fetch(`http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}/mcp`, { method: "GET" })).status, 403); } finally { - facade.revoke("turn-1"); - await facade.close(); + facade.revoke("turn-refusals"); } }); + +/** + * Last, because both cases take the fixed port for themselves: an open GET + * tunnel must not survive its own capability, and must not stall shutdown + * either — a server-to-client stream stays open for the whole session, so + * before it existed nothing could hold the listener open. + */ +test("revoking a turn tears down its open server-to-client stream, and closing never stalls on one", async () => { + await releaseShared(); + const facade = await startFacade(); + const rig = await startRig(facade); + const { client } = await connectClient(rig.capability); + await new Promise((resolve) => setTimeout(resolve, 150)); + assert.ok(rig.observed.some((request) => request.method === "GET"), "the GET stream must be open before revoking"); + + // The client's stream survives its own capability unless the facade ends the + // tunnel: the mount's GET response is the side that has to close. + facade.revoke(rig.turnId); + await withDeadline(rig.getStreamClosed, 4_000, "a revoked capability left its SSE tunnel open"); + + // A second turn's tunnel, deliberately left open, is what shutdown must not wait on. + const second = await startRig(facade); + const held = await connectClient(second.capability); + await new Promise((resolve) => setTimeout(resolve, 150)); + await withDeadline(facade.close(), 3_000, "closing the facade stalled on an open SSE tunnel"); + + await held.client.close().catch(() => undefined); + await client.close().catch(() => undefined); + await second.close().catch(() => undefined); + await rig.close().catch(() => undefined); +}); From 73cc6b6e3b8a5509e71d6fe17f2ac00e4133f63a Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 01:00:33 +0200 Subject: [PATCH 092/124] fix: name the underlying fault beside the proxy's broker_unavailable log line --- src/runtime/AGENTS.md | 14 ++++++++- src/runtime/grokBrokerProxy.test.ts | 25 ++++++++++++++++ src/runtime/grokBrokerProxy.ts | 45 ++++++++++++++++++++++++++--- 3 files changed, 79 insertions(+), 5 deletions(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 6276db1..fb1e433 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -171,7 +171,19 @@ any capability lookup, isolation guard, credential read or upstream call, and Grok falls back to the truncated prompt as the title. That refusal and a bare unauthenticated `GET /` probe are the two requests a healthy turn always makes and the proxy never forwards; neither prints a `refused:` line, because for as -long as they did, every healthy turn read as broken. The sink keeps its 503 +long as they did, every healthy turn read as broken. Every *other* refused +request does name itself on the broker's stderr, and a fault that is not a +`GrokBrokerProxyRefusal` names its own class and message beside +`broker_unavailable` — `[grok-proxy] refused: broker_unavailable (TypeError: +…)` — because the bare word carries no diagnostic content and is answered 503, +which Grok blind-retries: one live turn emitted it fifteen times over five +minutes, spent $0, and died with no account of why. That cause is the error's +class and message only (never a body, bearer, capability, session id or +header), redacted through `redactCredentialText` with that request's own +capabilities as exact secrets and the `CLI_ENGINE_MAX_DIAGNOSTIC_BYTES` bound, +flattened to one line, exactly as the failed CLI child and the launcher's +worker diagnostic are. It is a log line only: the 503 is unchanged, because a +genuinely transient fault is still transient. The sink keeps its 503 shape because every live capture was taken with it: forcing 400 and 503 there were both observed to end the turn `exit=0, result: success`, so a hard 4xx on that request does *not* end Grok's session. And effort is only sent when the diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index 05caadf..8ca502d 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -112,3 +112,28 @@ test("the two requests every healthy turn makes are not named as refusals", asyn assert.deepEqual(lines, ["[grok-proxy] refused: unknown_capability\n"], "a genuine policy miss is still named with its reason code"); } finally { process.stderr.write = original; await proxy.close(); } }); + +test("a non-refusal fault names its own class and message on one bounded line, credentials withheld", async () => { + const lines: string[] = []; const original = process.stderr.write.bind(process.stderr); + process.stderr.write = ((chunk: string | Uint8Array) => { lines.push(String(chunk)); return true; }) as typeof process.stderr.write; + const provider = "provider-vqmxdfhlzptgnbwc"; let capability = ""; + // The fault's own words carry both credentials verbatim — the worker's turn + // capability and the broker's provider bearer, neither in a shape any generic + // pattern recognises — plus a newline, a control character, and far more text + // than the bound admits. + const proxy = await startGrokBrokerProxy( + { accessToken: async () => provider, markRejected: async () => undefined }, + async () => { throw new RangeError(`socket hang up forwarding ${capability}\nwith ${provider} ${"pad ".repeat(400)}`); }); + try { + capability = proxy.capabilities.issue("agent", "turn"); arm(proxy, async () => undefined); + assert.equal(await post(proxy.port, capability, leanBody()), 503, "a genuine transient fault keeps its 503"); + assert.equal(lines.length, 1, "one line per fault"); + const line = lines[0]!; + assert.match(line, /^\[grok-proxy\] refused: broker_unavailable \(RangeError: socket hang up forwarding /u, "the fault names its own class and message"); + assert.ok(!line.includes(capability), `the worker's own capability is withheld: ${line}`); + assert.ok(!line.includes(provider), `the broker's provider bearer is withheld: ${line}`); + assert.match(line, /\[REDACTED\]/u, "the withheld values are marked, not silently dropped"); + assert.match(line, /^[^\n]+\n$/u, "one line: newlines and control characters are flattened"); + assert.ok(Buffer.byteLength(line, "utf8") <= 900, `the line stays bounded: ${Buffer.byteLength(line, "utf8")} bytes`); + } finally { process.stderr.write = original; await proxy.close(); } +}); diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index c848d91..3955938 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -1,6 +1,8 @@ import { createHash } from "node:crypto"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import type { AddressInfo } from "node:net"; +import { redactCredentialText } from "../core/credentialRedaction.js"; +import { CLI_ENGINE_MAX_DIAGNOSTIC_BYTES } from "../pi/cliChildOutput.js"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; import { DEFAULT_GROK_BROKER_MODEL_POLICY, parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; import { authorizeGrokBrokerProxyRequest } from "./grokBrokerProxyRequest.js"; @@ -51,14 +53,19 @@ export class GrokBrokerProxyRefusal extends Error { async function serve(request: IncomingMessage, response: ServerResponse, authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream, capabilities: EngineBrokerCapabilities,guards:MapPromise>,turns:Map,fallback:GrokBrokerModelPolicy,grants?:GrokInferenceGrants): Promise { let settle:((usage:ReturnType,toolCalls?:readonly string[])=>void)|undefined; let titleSink = false; + // Every credential this request holds, kept only for this request and only so + // that a fault's own words can be redacted against them exactly as the CLI + // child and launcher diagnostics are. Nothing reads them but {@link brokerFaultCause}. + const secrets: string[] = []; try { const body = await readBody(request); const headers = Object.fromEntries(Object.entries(request.headers).map(([key, value]) => [key, Array.isArray(value) ? value[0] : value])); titleSink = headers.authorization === `Bearer ${GROK_SESSION_TITLE_SINK_KEY}`; const match=headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u); + if(match)secrets.push(match[1]!); if(match&&match[1]!.startsWith(GROK_ENGINE_BROKER.inferenceGrants.tokenPrefix)){if(!grants)throw new GrokBrokerProxyRefusal("inference_grants_unavailable");return await serveGrokInferenceGrant({method:request.method??"",pathname:new URL(request.url??"/","http://127.0.0.1").pathname,headers,body,token:match[1]!},response,grants,authority,upstream);} const scope=match?capabilities.inspectToken(match[1]!):undefined;if(!scope)throw new GrokBrokerProxyRefusal("unknown_capability");const guard=guards.get(scope.turnId),turn=turns.get(scope.turnId);if(!guard||!turn)throw new GrokBrokerProxyRefusal("no_active_turn"); try { await guard(); } catch (error) { throw new GrokBrokerProxyRefusal("worker_isolation_unverified"); } - let token = await authority.accessToken(false);const rejectedDigest=createHash("sha256").update(token).digest("hex"); let prepared = authorizeRequestOrRefuse({ method: request.method ?? "", pathname: new URL(request.url ?? "/", "http://127.0.0.1").pathname, headers, body }, capabilities, token, turn.policy ?? fallback); token = ""; + let token = await authority.accessToken(false);secrets.push(token);const rejectedDigest=createHash("sha256").update(token).digest("hex"); let prepared = authorizeRequestOrRefuse({ method: request.method ?? "", pathname: new URL(request.url ?? "/", "http://127.0.0.1").pathname, headers, body }, capabilities, token, turn.policy ?? fallback); token = ""; // The spend gate runs after the body is proven a real lean worker request // (a refused session-title body never counts) and before any upstream call. const admission=turn.meter.admit(); @@ -66,7 +73,7 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori if("busy" in admission){response.writeHead(429,{"content-type":"application/json","cache-control":"no-store"});response.end('{"error":"turn request in flight"}');return;} settle=(usage,toolCalls)=>{turn.meter.settle(admission.index,usage,body.byteLength,toolCalls);settle=undefined;}; let result = await upstream(prepared,admission.signal); - if (result.status === 401) { token = authority.refreshAfterRejection?await authority.refreshAfterRejection(rejectedDigest):await authority.accessToken(true);const refreshedDigest=createHash("sha256").update(token).digest("hex"); prepared = { ...prepared, headers: { ...prepared.headers, authorization: `Bearer ${token}` } }; token = ""; result = await upstream(prepared,admission.signal);if(result.status===401)await authority.markRejected(refreshedDigest); } + if (result.status === 401) { token = authority.refreshAfterRejection?await authority.refreshAfterRejection(rejectedDigest):await authority.accessToken(true);secrets.push(token);const refreshedDigest=createHash("sha256").update(token).digest("hex"); prepared = { ...prepared, headers: { ...prepared.headers, authorization: `Bearer ${token}` } }; token = ""; result = await upstream(prepared,admission.signal);if(result.status===401)await authority.markRejected(refreshedDigest); } // Names only, bounded, and never a reason to fail the request: the response // is already buffered here for its usage block, so what the model tried to // call is in hand. A decoder fault records no attempt rather than a false @@ -79,7 +86,11 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori // or a token) so a failing turn is diagnosable without a stub harness — // except for the two requests every healthy turn makes anyway. const refusal = error instanceof GrokBrokerProxyRefusal ? error.reason : "broker_unavailable"; - if (!titleSink && !expectedWorkerProbe(request)) process.stderr.write(`[grok-proxy] refused: ${refusal}\n`); + // A named refusal is its own account; anything else used to reach the log as + // the bare word `broker_unavailable`, which names nothing — so it carries the + // fault's own class and message, and nothing else, beside it. + const named = error instanceof GrokBrokerProxyRefusal ? refusal : `${refusal} (${brokerFaultCause(error, secrets)})`; + if (!titleSink && !expectedWorkerProbe(request)) process.stderr.write(`[grok-proxy] refused: ${named}\n`); // Grok's own session-title call is refused by design, and keeps the transient // 503 shape it has always had. Forcing 400 and 503 on it were both observed // to end the turn `exit=0, result: success`, so the shape is kept because it @@ -97,9 +108,35 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori } response.writeHead(503, { "content-type": "application/json", "cache-control": "no-store" }); response.end('{"error":"broker unavailable"}'); - } + } finally { secrets.length = 0; } } +/** + * A non-refusal fault, named on one bounded line. + * + * `broker_unavailable` on its own carries no diagnostic content at all, and it + * is answered 503, which Grok blind-retries: one live turn emitted it fifteen + * times over five minutes, spent $0 — so no upstream call ever succeeded — and + * died with no account of why. The error's own class and message are the whole + * of what is logged: never a request body, bearer, capability, session id or + * header. It is redacted exactly as the failed CLI child and the launcher's + * worker diagnostic are — `redactCredentialText` with this request's own + * capabilities as exact secrets and the same `CLI_ENGINE_MAX_DIAGNOSTIC_BYTES` + * bound — and flattened to one line, because it travels on a log line. Naming + * a fault must never be able to fail the response that reports it, so a value + * that cannot even be described degrades to a marker. + */ +const brokerFaultCause = (error: unknown, secrets: readonly string[]): string => { + try { + const described = error instanceof Error + ? `${error.constructor?.name ?? error.name}: ${error.message}` + : `${typeof error}: ${String(error)}`; + const flattened = described.replace(/[-]+/gu, " ").replace(/\s+/gu, " ").trim(); + const named = redactCredentialText(flattened, secrets, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); + return named.length === 0 ? "unnamed" : named; + } catch { return "unnameable"; } +}; + /** * The unauthenticated connectivity probe Grok sends before its own requests: a * bare `GET /` with no Authorization header, which has no capability to look up From 16c12241f3cbf3bbfb1dc9071c165c935cb66a6d Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 01:07:58 +0200 Subject: [PATCH 093/124] fix: name one level of a broker fault's own cause so a failed provider fetch says which fault it was --- src/runtime/AGENTS.md | 8 ++++++-- src/runtime/grokBrokerProxy.test.ts | 13 +++++++++++++ src/runtime/grokBrokerProxy.ts | 18 +++++++++++++++--- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index fb1e433..c0ce68e 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -178,8 +178,12 @@ request does name itself on the broker's stderr, and a fault that is not a …)` — because the bare word carries no diagnostic content and is answered 503, which Grok blind-retries: one live turn emitted it fifteen times over five minutes, spent $0, and died with no account of why. That cause is the error's -class and message only (never a body, bearer, capability, session id or -header), redacted through `redactCredentialText` with that request's own +class and message, plus one level of its own `cause` — every failed provider +`fetch` is `TypeError: fetch failed` and names nothing without it, so the line +reads `broker_unavailable (TypeError: fetch failed <- Error: ENOTFOUND)`, an +errno cause with no message named by its `code`. Nothing else: never a body, +bearer, capability, session id or +header. It is redacted through `redactCredentialText` with that request's own capabilities as exact secrets and the `CLI_ENGINE_MAX_DIAGNOSTIC_BYTES` bound, flattened to one line, exactly as the failed CLI child and the launcher's worker diagnostic are. It is a log line only: the 503 is unchanged, because a diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index 8ca502d..be3b2f4 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -137,3 +137,16 @@ test("a non-refusal fault names its own class and message on one bounded line, c assert.ok(Buffer.byteLength(line, "utf8") <= 900, `the line stays bounded: ${Buffer.byteLength(line, "utf8")} bytes`); } finally { process.stderr.write = original; await proxy.close(); } }); + +test("a fault's own cause is named too, because `fetch failed` on its own names nothing", async () => { + const lines: string[] = []; const original = process.stderr.write.bind(process.stderr); + process.stderr.write = ((chunk: string | Uint8Array) => { lines.push(String(chunk)); return true; }) as typeof process.stderr.write; + // Exactly the shape undici throws when the provider is unreachable. + const fault = new TypeError("fetch failed"); (fault as { cause?: unknown }).cause = Object.assign(new Error(""), { code: "ENOTFOUND" }); + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { throw fault; }); + try { + const token = proxy.capabilities.issue("agent", "turn"); arm(proxy, async () => undefined); + assert.equal(await post(proxy.port, token, leanBody()), 503); + assert.deepEqual(lines, ["[grok-proxy] refused: broker_unavailable (TypeError: fetch failed <- Error: ENOTFOUND)\n"]); + } finally { process.stderr.write = original; await proxy.close(); } +}); diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index 3955938..a92b61c 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -125,18 +125,30 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori * bound — and flattened to one line, because it travels on a log line. Naming * a fault must never be able to fail the response that reports it, so a value * that cannot even be described degrades to a marker. + * + * One level of `cause` is named too, because the fault this exists for names + * nothing without it: every failed `fetch` to the provider is `TypeError: + * fetch failed`, and which fault it was — `ENOTFOUND`, `ECONNREFUSED`, a TLS + * refusal, an abort — is only in the cause. An errno error whose message is + * empty is named by its `code`. */ const brokerFaultCause = (error: unknown, secrets: readonly string[]): string => { try { - const described = error instanceof Error - ? `${error.constructor?.name ?? error.name}: ${error.message}` - : `${typeof error}: ${String(error)}`; + const described = `${describeFault(error)}${error instanceof Error && error.cause !== undefined && error.cause !== null ? ` <- ${describeFault(error.cause)}` : ""}`; const flattened = described.replace(/[-]+/gu, " ").replace(/\s+/gu, " ").trim(); const named = redactCredentialText(flattened, secrets, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); return named.length === 0 ? "unnamed" : named; } catch { return "unnameable"; } }; +/** One value's class and words: an error's own, or the type of whatever else was thrown. */ +const describeFault = (error: unknown): string => { + if (!(error instanceof Error)) return `${typeof error}: ${String(error)}`; + const code = (error as NodeJS.ErrnoException).code; + const words = error.message.length > 0 ? error.message : typeof code === "string" ? code : "(no message)"; + return `${error.constructor?.name ?? error.name}: ${words}`; +}; + /** * The unauthenticated connectivity probe Grok sends before its own requests: a * bare `GET /` with no Authorization header, which has no capability to look up From f3d4b2bc9b4bec41aacffc0e1758c1ae9eb6e8d8 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 01:09:40 +0200 Subject: [PATCH 094/124] fix: escape the flatten ranges and the test's control byte so no source carries a raw control byte --- src/runtime/grokBrokerProxy.test.ts | 2 +- src/runtime/grokBrokerProxy.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index be3b2f4..ee9270b 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -123,7 +123,7 @@ test("a non-refusal fault names its own class and message on one bounded line, c // than the bound admits. const proxy = await startGrokBrokerProxy( { accessToken: async () => provider, markRejected: async () => undefined }, - async () => { throw new RangeError(`socket hang up forwarding ${capability}\nwith ${provider} ${"pad ".repeat(400)}`); }); + async () => { throw new RangeError(`socket hang up forwarding ${capability}\nwith ${provider}\u0007 ${"pad ".repeat(400)}`); }); try { capability = proxy.capabilities.issue("agent", "turn"); arm(proxy, async () => undefined); assert.equal(await post(proxy.port, capability, leanBody()), 503, "a genuine transient fault keeps its 503"); diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index a92b61c..47be53c 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -135,7 +135,7 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori const brokerFaultCause = (error: unknown, secrets: readonly string[]): string => { try { const described = `${describeFault(error)}${error instanceof Error && error.cause !== undefined && error.cause !== null ? ` <- ${describeFault(error.cause)}` : ""}`; - const flattened = described.replace(/[-]+/gu, " ").replace(/\s+/gu, " ").trim(); + const flattened = described.replace(/[\u0000-\u001f\u007f]+/gu, " ").replace(/\s+/gu, " ").trim(); const named = redactCredentialText(flattened, secrets, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); return named.length === 0 ? "unnamed" : named; } catch { return "unnameable"; } From 6a4fa63da47fdcf5478a0b6468f2644051edd99a Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 01:15:52 +0200 Subject: [PATCH 095/124] fix: wake the MCP facade's backpressure await on a hang-up or abort so a stalled tunnel cannot park a turn --- src/runtime/engineBrokerMcpFacade.test.ts | 50 ++++++++++++++++++++++- src/runtime/engineBrokerMcpFacade.ts | 35 +++++++++++++++- 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/src/runtime/engineBrokerMcpFacade.test.ts b/src/runtime/engineBrokerMcpFacade.test.ts index 95d0b65..057e798 100644 --- a/src/runtime/engineBrokerMcpFacade.test.ts +++ b/src/runtime/engineBrokerMcpFacade.test.ts @@ -1,5 +1,6 @@ import assert from "node:assert/strict"; import { createServer, type Server as HttpServer, type IncomingMessage } from "node:http"; +import { connect, type Socket } from "node:net"; import { randomUUID } from "node:crypto"; import test from "node:test"; @@ -11,7 +12,7 @@ import { Type } from "@earendil-works/pi-ai"; import { defineTool } from "@earendil-works/pi-coding-agent"; import { createPiToolMcpServer } from "../mcp/toolServer.js"; -import { ENGINE_BROKER_MCP_FACADE_PORT, startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; +import { awaitMcpTunnelDrain, ENGINE_BROKER_MCP_FACADE_PORT, startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; const FACADE_URL = `http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}/mcp`; type Facade = Awaited>; @@ -311,3 +312,50 @@ test("revoking a turn tears down its open server-to-client stream, and closing n await second.close().catch(() => undefined); await rig.close().catch(() => undefined); }); + +/** + * The relay parks on this await whenever a tunnel is backpressured, and a + * parked await is invisible from outside: no status, no refusal, no line. So + * the assertion is that it settles at all — on a client that hung up + * mid-write, on the turn's abort, and on a genuine drain — with a deadline + * standing in for the hang. + */ +test("a backpressured MCP tunnel always settles: on a hang-up, on the turn's abort, and on a real drain", async () => { + const parked = new Map>(); + // One controller per phase: the turn whose abort is under test must not be + // the turn that is still relaying. + const controllers = new Map(); + const server = createServer((request, response) => { + const phase = request.url ?? ""; + const controller = new AbortController(); controllers.set(phase, controller); + response.writeHead(200, { "content-type": "text/event-stream" }); + // A paused client cannot absorb this, so `write` reports backpressure and + // the relay would park exactly here. + response.write("data: open\n\n"); + if (phase === "/drain") assert.equal(response.write(Buffer.alloc(16 * 1024 * 1024, 0x61)), false, "a paused client must backpressure the tunnel"); + parked.set(phase, awaitMcpTunnelDrain(response, controller.signal).then(() => "drained", (error: Error) => error.message)); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); if (address === null || typeof address === "string") throw new Error(); + const open = (phase: string): Promise => new Promise((resolve) => { + const socket = connect(address.port, "127.0.0.1", () => socket.write(`GET ${phase} HTTP/1.1\r\nhost: facade\r\n\r\n`)); + socket.once("data", () => { socket.pause(); resolve(socket); }); + }); + const settled = (phase: string): Promise => withDeadline(parked.get(phase)!, 3_000, `the ${phase} await never settled: the relay is parked`); + const sockets: Socket[] = []; + try { + sockets.push(await open("/hangup")); + sockets[0]!.destroy(); + assert.equal(await settled("/hangup"), "MCP tunnel closed", "a client that hung up mid-write wakes the await"); + sockets.push(await open("/abort")); + controllers.get("/abort")!.abort(); + assert.equal(await settled("/abort"), "MCP tunnel aborted", "the turn's own abort wakes the await"); + const draining = await open("/drain"); + sockets.push(draining); + draining.resume(); + assert.equal(await settled("/drain"), "drained", "a client that resumes reading resolves the await"); + } finally { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve) => server.close(() => resolve())); + } +}); diff --git a/src/runtime/engineBrokerMcpFacade.ts b/src/runtime/engineBrokerMcpFacade.ts index ee1defd..03eee1f 100644 --- a/src/runtime/engineBrokerMcpFacade.ts +++ b/src/runtime/engineBrokerMcpFacade.ts @@ -175,7 +175,8 @@ async function forward( const stream = Readable.fromWeb(upstream.body as Parameters[0]); try { for await (const chunk of stream) { - if (!response.write(chunk as Uint8Array)) await new Promise((resolve) => response.once("drain", resolve)); + if (response.destroyed || response.writableEnded) throw new Error("MCP tunnel closed"); + if (!response.write(chunk as Uint8Array)) await awaitMcpTunnelDrain(response, signal); } response.end(); } catch { @@ -187,6 +188,38 @@ async function forward( } } +/** + * Waits for a backpressured tunnel to drain, or for the tunnel to end — + * whichever happens first, but always one of them. + * + * A bare `once("drain")` never settles for a client that hung up mid-write: + * `drain` cannot fire on a socket nobody is reading, and neither the + * response's own `close` nor the turn's abort woke that await. The GET SSE + * tunnel stays open for a whole session, so the handler — and the upstream + * call it was relaying — leaked for the life of the broker process, with no + * refusal, no status and no line anywhere to read. Every outcome settles this + * now, and every outcome but an actual drain *rejects*, so the caller tears + * the tunnel down instead of writing into a socket that is gone. + * + * Exported for its own test: a hang is only observable from inside. + */ +export function awaitMcpTunnelDrain(response: ServerResponse, signal: AbortSignal): Promise { + if (response.destroyed || response.writableEnded) return Promise.reject(new Error("MCP tunnel closed")); + if (signal.aborted) return Promise.reject(new Error("MCP tunnel aborted")); + return new Promise((resolve, reject) => { + const settle = (finish: () => void) => (): void => { + response.off("drain", onDrain); response.off("close", onClosed); response.off("error", onClosed); + signal.removeEventListener("abort", onAborted); + finish(); + }; + const onDrain = settle(resolve); + const onClosed = settle(() => reject(new Error("MCP tunnel closed"))); + const onAborted = settle(() => reject(new Error("MCP tunnel aborted"))); + response.once("drain", onDrain); response.once("close", onClosed); response.once("error", onClosed); + signal.addEventListener("abort", onAborted, { once: true }); + }); +} + async function bounded(request: AsyncIterable): Promise { const chunks: Buffer[] = []; let bytes = 0; From 64a30bb5d19fdc9499c008b8b06e9fb563a98032 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 01:17:52 +0200 Subject: [PATCH 096/124] fix: refuse a fenced credential realm as a named non-retryable auth_stale on the turn path --- src/runtime/AGENTS.md | 22 +++++++++++-- src/runtime/engineBrokerProtocol.ts | 9 +++++- src/runtime/grokBrokerProxy.test.ts | 50 +++++++++++++++++++++++++++++ src/runtime/grokBrokerProxy.ts | 17 ++++++++-- 4 files changed, 92 insertions(+), 6 deletions(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index c0ce68e..947682e 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -187,7 +187,18 @@ header. It is redacted through `redactCredentialText` with that request's own capabilities as exact secrets and the `CLI_ENGINE_MAX_DIAGNOSTIC_BYTES` bound, flattened to one line, exactly as the failed CLI child and the launcher's worker diagnostic are. It is a log line only: the 503 is unchanged, because a -genuinely transient fault is still transient. The sink keeps its 503 +genuinely transient fault is still transient. + +One fault is *not* transient and no longer wears that shape: a fenced +credential realm. `isStale()` is checked on the turn path before the +credential read, and the request that discovers the fence (the authority's own +generic error) is promoted to the same refusal, so a stale realm is a named +400 `auth_stale` instead of one 503 plus fourteen blind retries — the training +login expired at 22:28Z and the 22:48Z run spent five minutes and $0 learning +nothing. `ENGINE_BROKER_AUTH_STALE` (`engineBrokerProtocol.ts`) is the single +name behind the turn failure code, this refusal reason and the grant path's +401 `GROK_INFERENCE_AUTH_STALE_BODY`; the grant path keeps its own 401 shape, +and the title sink keeps its 503 on a fenced realm like everywhere else. The sink keeps its 503 shape because every live capture was taken with it: forcing 400 and 503 there were both observed to end the turn `exit=0, result: success`, so a hard 4xx on that request does *not* end Grok's session. And effort is only sent when the @@ -212,7 +223,14 @@ never logged or ledgered. The facade also carries the three methods the transport uses — POST, the standalone `GET` SSE stream that is the only route a server notification or progress frame can take, and the `DELETE` that ends a session — and streams each body rather than buffering it, because a GET tunnel -stays open for the whole session. Never widen it into a transparent proxy: the +stays open for the whole session. Streaming means backpressure, and a +backpressured tunnel must never park: `awaitMcpTunnelDrain` races the client's +`drain` against its `close`/`error` and the turn's abort, because a bare +`once("drain")` cannot fire for a client that hung up mid-write and left the +handler — and the upstream call it was relaying — awaiting for the life of the +process, with no status, no refusal and no line anywhere to read. Every +outcome but a real drain rejects, so the relay tears the tunnel down instead +of writing into a socket that is gone. Never widen it into a transparent proxy: the whole point of the boundary is that the allowlist is closed. Worker `GROK_HOME` layout the deployment must provision (attested before every diff --git a/src/runtime/engineBrokerProtocol.ts b/src/runtime/engineBrokerProtocol.ts index eedf428..224ca44 100644 --- a/src/runtime/engineBrokerProtocol.ts +++ b/src/runtime/engineBrokerProtocol.ts @@ -36,7 +36,14 @@ export type EngineBrokerResponse = | (Readonly<{ version: typeof VERSION; kind: "completed"; requestId: string; turnId: string; text: string; workerPid: number; workerUid: number; workerStartTime: string }> & EngineBrokerTurnAccounting) | (Readonly<{ version: typeof VERSION; kind: "failed"; requestId: string; turnId: string; code: EngineBrokerFailureCode; diagnostic?: EngineBrokerFailureDiagnostic }> & EngineBrokerTurnAccounting) | EngineBrokerInferenceResponse; -export const ENGINE_BROKER_FAILURE_CODES = ["auth_stale", "cancelled", "engine_failed", "invalid_request", "limit_exceeded", "turn_conflict", "unavailable"] as const; +/** + * The one name for a fenced credential realm. The turn's failure code, the + * proxy's refusal reason on a worker request, and the grant path's 401 body + * (`GROK_INFERENCE_AUTH_STALE_BODY`) all say this same word, so an operator + * greps one string across every surface instead of three spellings of it. + */ +export const ENGINE_BROKER_AUTH_STALE = "auth_stale" as const; +export const ENGINE_BROKER_FAILURE_CODES = [ENGINE_BROKER_AUTH_STALE, "cancelled", "engine_failed", "invalid_request", "limit_exceeded", "turn_conflict", "unavailable"] as const; export type EngineBrokerFailureCode = (typeof ENGINE_BROKER_FAILURE_CODES)[number]; export type EngineBrokerTerminalResponse = Extract; type V1Completed = Readonly<{ version: typeof V1; kind: "completed"; requestId: string; turnId: string; text: string; workerPid: number; workerUid: number; workerStartTime: string }>; diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index ee9270b..6c9cfd9 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -4,6 +4,8 @@ import test from "node:test"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; import { GROK_SESSION_TITLE_SINK_KEY } from "./grokBrokerWorkerConfig.js"; import { GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; +import { ENGINE_BROKER_AUTH_STALE } from "./engineBrokerProtocol.js"; +import { GROK_INFERENCE_AUTH_STALE_BODY } from "./grokInferenceProxy.js"; type Proxy = Awaited>; const arm = (proxy: Proxy, guard: () => Promise, meter = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 })): GrokBrokerTurnMeter => { @@ -150,3 +152,51 @@ test("a fault's own cause is named too, because `fetch failed` on its own names assert.deepEqual(lines, ["[grok-proxy] refused: broker_unavailable (TypeError: fetch failed <- Error: ENOTFOUND)\n"]); } finally { process.stderr.write = original; await proxy.close(); } }); + +/** + * A fenced realm is the one fault this proxy answered worst: the credential is + * gone until an operator logs in again, and 503 made Grok retry it fifteen + * times over five minutes for nothing. It is a named, non-retryable refusal + * now — and it wears the same name as the turn failure code and the grant + * path's 401 body, so one word finds it on every surface. + */ +test("a fenced credential realm is a named 400 auth_stale, before any credential read or upstream call", async () => { + const lines: string[] = []; const original = process.stderr.write.bind(process.stderr); + process.stderr.write = ((chunk: string | Uint8Array) => { lines.push(String(chunk)); return true; }) as typeof process.stderr.write; + let reads = 0, calls = 0, stale = true; + const proxy = await startGrokBrokerProxy( + { accessToken: async () => { reads += 1; return "provider-token"; }, markRejected: async () => undefined, isStale: () => stale }, + async () => { calls += 1; return { status: 200, headers: { "content-type": "application/json" }, body: Buffer.from("{}") }; }); + try { + const token = proxy.capabilities.issue("agent", "turn"); arm(proxy, async () => undefined); + assert.equal(await post(proxy.port, token, leanBody()), 400, "a stale realm is not transient, so it must not be retryable"); + assert.deepEqual({ reads, calls }, { reads: 0, calls: 0 }, "no credential is read and nothing is forwarded for a fenced realm"); + assert.deepEqual(lines, [`[grok-proxy] refused: ${ENGINE_BROKER_AUTH_STALE}\n`]); + // The title sink keeps the 503 it has always had, fenced realm or not. + assert.equal(await post(proxy.port, GROK_SESSION_TITLE_SINK_KEY, leanBody()), 503); + // And the same capability serves the real request once the realm is healthy. + stale = false; lines.length = 0; + assert.equal(await post(proxy.port, token, leanBody()), 200); + assert.deepEqual({ reads, calls, lines }, { reads: 1, calls: 1, lines: [] }); + } finally { process.stderr.write = original; await proxy.close(); } +}); + +test("the request that discovers the fence is named auth_stale too, not one transient fault", async () => { + const lines: string[] = []; const original = process.stderr.write.bind(process.stderr); + process.stderr.write = ((chunk: string | Uint8Array) => { lines.push(String(chunk)); return true; }) as typeof process.stderr.write; + // The live shape: `accessToken` fences the realm and throws the authority's + // own generic error, which on its own reads as a transient fault. + let stale = false; + const proxy = await startGrokBrokerProxy( + { accessToken: async () => { stale = true; throw new Error("Grok broker credential authority unavailable"); }, markRejected: async () => undefined, isStale: () => stale }, + async () => ({ status: 200, headers: { "content-type": "application/json" }, body: Buffer.from("{}") })); + try { + const token = proxy.capabilities.issue("agent", "turn"); arm(proxy, async () => undefined); + assert.equal(await post(proxy.port, token, leanBody()), 400); + assert.deepEqual(lines, [`[grok-proxy] refused: ${ENGINE_BROKER_AUTH_STALE}\n`]); + } finally { process.stderr.write = original; await proxy.close(); } +}); + +test("the turn path and the grant path name a fenced realm the same way", () => { + assert.ok(GROK_INFERENCE_AUTH_STALE_BODY.includes(ENGINE_BROKER_AUTH_STALE), "one name, not three spellings"); +}); diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index 47be53c..55f335a 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -4,6 +4,7 @@ import type { AddressInfo } from "node:net"; import { redactCredentialText } from "../core/credentialRedaction.js"; import { CLI_ENGINE_MAX_DIAGNOSTIC_BYTES } from "../pi/cliChildOutput.js"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; +import { ENGINE_BROKER_AUTH_STALE } from "./engineBrokerProtocol.js"; import { DEFAULT_GROK_BROKER_MODEL_POLICY, parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; import { authorizeGrokBrokerProxyRequest } from "./grokBrokerProxyRequest.js"; import { GROK_SESSION_TITLE_SINK_KEY } from "./grokBrokerWorkerConfig.js"; @@ -65,6 +66,11 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori if(match&&match[1]!.startsWith(GROK_ENGINE_BROKER.inferenceGrants.tokenPrefix)){if(!grants)throw new GrokBrokerProxyRefusal("inference_grants_unavailable");return await serveGrokInferenceGrant({method:request.method??"",pathname:new URL(request.url??"/","http://127.0.0.1").pathname,headers,body,token:match[1]!},response,grants,authority,upstream);} const scope=match?capabilities.inspectToken(match[1]!):undefined;if(!scope)throw new GrokBrokerProxyRefusal("unknown_capability");const guard=guards.get(scope.turnId),turn=turns.get(scope.turnId);if(!guard||!turn)throw new GrokBrokerProxyRefusal("no_active_turn"); try { await guard(); } catch (error) { throw new GrokBrokerProxyRefusal("worker_isolation_unverified"); } + // A fenced realm is not a transient fault: the credential is gone until an + // operator re-logs in, and 503 made Grok blind-retry it (observed: fifteen + // retries over five minutes, $0 spent, nothing learned). Named, 400, and + // checked before the credential read, so the miss costs one round trip. + if (authority.isStale?.() === true) throw new GrokBrokerProxyRefusal(ENGINE_BROKER_AUTH_STALE); let token = await authority.accessToken(false);secrets.push(token);const rejectedDigest=createHash("sha256").update(token).digest("hex"); let prepared = authorizeRequestOrRefuse({ method: request.method ?? "", pathname: new URL(request.url ?? "/", "http://127.0.0.1").pathname, headers, body }, capabilities, token, turn.policy ?? fallback); token = ""; // The spend gate runs after the body is proven a real lean worker request // (a refused session-title body never counts) and before any upstream call. @@ -85,11 +91,16 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori // Name the refusal on the broker's own stderr (reason code only, never a body // or a token) so a failing turn is diagnosable without a stub harness — // except for the two requests every healthy turn makes anyway. - const refusal = error instanceof GrokBrokerProxyRefusal ? error.reason : "broker_unavailable"; + // The request that *discovers* the fence throws an ordinary error from the + // credential authority, so it is promoted to the same named refusal: one + // stale realm must not read as one transient fault plus fourteen retries. + const fenced = !(error instanceof GrokBrokerProxyRefusal) && authority.isStale?.() === true; + const refused = error instanceof GrokBrokerProxyRefusal || fenced; + const refusal = error instanceof GrokBrokerProxyRefusal ? error.reason : fenced ? ENGINE_BROKER_AUTH_STALE : "broker_unavailable"; // A named refusal is its own account; anything else used to reach the log as // the bare word `broker_unavailable`, which names nothing — so it carries the // fault's own class and message, and nothing else, beside it. - const named = error instanceof GrokBrokerProxyRefusal ? refusal : `${refusal} (${brokerFaultCause(error, secrets)})`; + const named = refused ? refusal : `${refusal} (${brokerFaultCause(error, secrets)})`; if (!titleSink && !expectedWorkerProbe(request)) process.stderr.write(`[grok-proxy] refused: ${named}\n`); // Grok's own session-title call is refused by design, and keeps the transient // 503 shape it has always had. Forcing 400 and 503 on it were both observed @@ -101,7 +112,7 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori response.end('{"error":"broker unavailable"}'); return; } - if (error instanceof GrokBrokerProxyRefusal) { + if (refused) { response.writeHead(400, { "content-type": "application/json", "cache-control": "no-store" }); response.end(JSON.stringify({ error: "broker refused this request", reason: refusal })); return; From 8ac45f49bf0025a2cec3ac8ad5465e0d595e5dc4 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 01:18:02 +0200 Subject: [PATCH 097/124] docs: keep the session-title sink's own paragraph intact in the runtime guide --- src/runtime/AGENTS.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 947682e..baa9983 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -198,7 +198,9 @@ login expired at 22:28Z and the 22:48Z run spent five minutes and $0 learning nothing. `ENGINE_BROKER_AUTH_STALE` (`engineBrokerProtocol.ts`) is the single name behind the turn failure code, this refusal reason and the grant path's 401 `GROK_INFERENCE_AUTH_STALE_BODY`; the grant path keeps its own 401 shape, -and the title sink keeps its 503 on a fenced realm like everywhere else. The sink keeps its 503 +and the title sink keeps its 503 on a fenced realm like everywhere else. + +The sink keeps that 503 shape because every live capture was taken with it: forcing 400 and 503 there were both observed to end the turn `exit=0, result: success`, so a hard 4xx on that request does *not* end Grok's session. And effort is only sent when the From a509277d215c786650a5e1345f811255d9107791 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 01:20:17 +0200 Subject: [PATCH 098/124] fix: let a usage decoder fault fall through to the estimate instead of failing a paid request --- src/runtime/AGENTS.md | 8 +++++++- src/runtime/grokBrokerProxy.test.ts | 29 ++++++++++++++++++++++++++++- src/runtime/grokBrokerProxy.ts | 17 ++++++++++++++++- 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index baa9983..147dc5c 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -424,7 +424,13 @@ decoded records *no field at all*, because a fabricated empty list is byte-identical to a measured one. On the stream row path the names are attached only when the proxy's timings and the worker's stream requests are aligned request-for-request, since an unaligned index would credit one request's attempt -to another. It is an additive field inside the unchanged +to another. The *usage* decode beside it is wrapped the same way, and for a +sharper reason: the upstream call has already succeeded, so a decoder fault +that failed the request would throw away a response the broker paid for and +have Grok buy it again. A fault there falls through to the documented estimate +(`ceil(bodyBytes/2) + 4096`, `usage_source: "estimated"`, counted in +`estimated_requests`) — never to zero and never to absence, because the +ceiling must still count what was spent. It is an additive field inside the unchanged `noopolis.daimon.turn-requests.v1` row and deliberately not a version bump: Spawnfile's reader pins `v` and ignores fields it does not know, and Paideia only relocates this stream's path. The whole path is advisory — the parse is diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index 6c9cfd9..526bd78 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -3,7 +3,7 @@ import { request as httpRequest } from "node:http"; import test from "node:test"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; import { GROK_SESSION_TITLE_SINK_KEY } from "./grokBrokerWorkerConfig.js"; -import { GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; +import { estimateGrokRequestUsage, GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; import { ENGINE_BROKER_AUTH_STALE } from "./engineBrokerProtocol.js"; import { GROK_INFERENCE_AUTH_STALE_BODY } from "./grokInferenceProxy.js"; @@ -200,3 +200,30 @@ test("the request that discovers the fence is named auth_stale too, not one tran test("the turn path and the grant path name a fenced realm the same way", () => { assert.ok(GROK_INFERENCE_AUTH_STALE_BODY.includes(ENGINE_BROKER_AUTH_STALE), "one name, not three spellings"); }); + +/** + * A response the usage decoder cannot read is the worst case to get wrong: the + * upstream call already succeeded, so the money is spent whatever happens + * next. It must reach the worker anyway, and it must be charged — a fabricated + * zero is byte-identical to a measured one, so the documented estimate is the + * only honest landing place. (The fault is synthetic: the real one is a + * response body past the maximum string length, which is not a thing to + * allocate in a test.) + */ +test("a response whose usage cannot be decoded is still delivered, and charged the documented estimate", async () => { + const contentType = { toString: () => "application/json" } as unknown as string; + const proxy = await startGrokBrokerProxy( + { accessToken: async () => "provider-token", markRejected: async () => undefined }, + async () => ({ status: 200, headers: { "content-type": contentType }, body: Buffer.from('{"usage":{"prompt_tokens":11,"completion_tokens":3}}') })); + try { + const token = proxy.capabilities.issue("agent", "turn"); + const meter = arm(proxy, async () => undefined); + const payload = leanBody(); + assert.equal(await post(proxy.port, token, payload), 200, "a paid response must not be thrown away over its own instrumentation"); + const snapshot = meter.snapshot(); + assert.equal(snapshot.requests, 1); + assert.equal(snapshot.estimatedRequests, 1, "the row says the charge was estimated, not measured"); + assert.deepEqual(snapshot.usage, estimateGrokRequestUsage(Buffer.byteLength(payload)), "charged the documented conservative estimate, never zero and never nothing"); + assert.equal(snapshot.timings[0]?.toolCalls, undefined, "an undecodable response records no tool-call attempt either"); + } finally { await proxy.close(); } +}); diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index 55f335a..9857cff 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -84,7 +84,7 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori // is already buffered here for its usage block, so what the model tried to // call is in hand. A decoder fault records no attempt rather than a false // empty one, and never disturbs the turn. - settle?.(parseGrokUpstreamUsage(result.body,result.headers["content-type"]),toolCallsOrNothing(result.body,result.headers["content-type"])); + settle?.(usageOrEstimate(result.body,result.headers["content-type"]),toolCallsOrNothing(result.body,result.headers["content-type"])); response.writeHead(result.status, { "content-type": result.headers["content-type"] ?? "application/json", "cache-control": "no-store" }); response.end(result.body); } catch (error) { settle?.(undefined); @@ -176,6 +176,21 @@ const expectedWorkerProbe = (request: IncomingMessage): boolean => request.headers.authorization === undefined && (request.method ?? "") === "GET" && new URL(request.url ?? "/", "http://127.0.0.1").pathname === "/"; +/** + * Upstream-reported usage, or the documented conservative estimate. + * + * The decoder faulting must not fail the request that already cost real + * money: the upstream call succeeded, the worker is owed its answer, and a + * 503 here would throw away a paid response and have Grok buy it again. The + * `undefined` this returns is not "no usage" — `GrokBrokerTurnMeter.settle` + * charges it `ceil(bodyBytes/2) + 4096` and marks the row + * `usage_source: "estimated"`, so the token ceiling still counts it and no + * fabricated zero ever reaches a ledger. + */ +const usageOrEstimate = (body: Uint8Array, contentType: string | undefined): ReturnType => { + try { return parseGrokUpstreamUsage(body, contentType); } catch { return undefined; } +}; + /** Instrumentation must never fail a turn: a throwing decoder records nothing, exactly as an undecodable response does. */ const toolCallsOrNothing = (body: Uint8Array, contentType: string | undefined): readonly string[] | undefined => { try { return parseGrokResponseToolNames(body, contentType); } catch { return undefined; } From 20ae57befb33baa5958dacb0aabe28a5e7b19548 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 01:39:02 +0200 Subject: [PATCH 099/124] fix: decode a failed worker's last words as text and keep both ends of the diagnostic window --- src/pi/cliChildOutput.ts | 82 ++++++++++++++++---- src/pi/cliSessionOutput.test.ts | 47 ++++++++++- src/runtime/AGENTS.md | 18 ++++- src/runtime/engineBrokerNativeClient.test.ts | 60 +++++++++++++- src/runtime/engineBrokerNativeClient.ts | 38 +++++++-- src/runtime/native/AGENTS.md | 16 ++++ src/runtime/toolResultSpill.ts | 6 +- 7 files changed, 237 insertions(+), 30 deletions(-) diff --git a/src/pi/cliChildOutput.ts b/src/pi/cliChildOutput.ts index 19b5ca2..78a87d4 100644 --- a/src/pi/cliChildOutput.ts +++ b/src/pi/cliChildOutput.ts @@ -1,13 +1,14 @@ import type { ChildProcess } from "node:child_process"; import { redactCredentialText } from "../core/credentialRedaction.js"; +import { headUtf8, tailUtf8 } from "../runtime/toolResultSpill.js"; import type { TurnUsageFailureReason } from "../runtime/turnUsageLedger.js"; import { decodeCodexTurnUsage, type CodexTurnUsage } from "./codexHeadlessResult.js"; import { terminateChild, trackCliChild } from "./cliProcess.js"; /** Maximum assistant reply bytes retained from stdout. */ export const CLI_ENGINE_MAX_OUTPUT_BYTES = 64 * 1024; -/** Tail bytes retained from stderr only for a failed-child diagnostic. */ +/** Diagnostic bytes retained from stderr for a failed child: its head and its tail together. */ export const CLI_ENGINE_MAX_DIAGNOSTIC_BYTES = 768; const CLI_ENGINE_FAILURE_SCAN_CHARS = 256; @@ -39,18 +40,36 @@ const redactChildOutput = (value: string, secretValues: readonly string[]): stri return redactCredentialText(value, secretValues, CLI_ENGINE_MAX_OUTPUT_BYTES); }; -const utf8Tail = (value: string, maxBytes: number): string => { - const bytes = Buffer.from(value, "utf8"); - if (bytes.length <= maxBytes) return value; - let result = bytes.subarray(bytes.length - maxBytes).toString("utf8"); - while (result.startsWith("\uFFFD")) result = result.slice(1); - return result; +/** + * One bounded window over a failed child's own output: its head AND its tail, + * with an explicit marker naming the bytes elided between them. + * + * A pure tail is the wrong end for the process this exists for. A worker that + * dies early prints its error first and then echoes its own input, so the tail + * is the echo: one live brokered turn reported 512 bytes of its own prompt + * read back, with the actual error already off the front and discarded. Both + * ends cost the same window, and the marker is the one oversized tool results + * already use (`toolResultSpill.ts`), so a reader meets one shape everywhere. + * + * The marker is paid for out of the same budget — it is sized against the + * largest count it could carry — so the result never exceeds `maxBytes`, and + * output that fits is returned byte-identical with no marker at all. + */ +export const boundedDiagnosticWindow = (value: string, maxBytes: number): string => { + const total = Buffer.byteLength(value, "utf8"); + if (total <= maxBytes) return value; + const budget = Math.max(0, maxBytes - Buffer.byteLength(elisionMarker(total), "utf8")); + const head = headUtf8(value, Math.floor(budget / 2)); + const tail = tailUtf8(value, budget - Buffer.byteLength(head, "utf8")); + const elided = total - Buffer.byteLength(head, "utf8") - Buffer.byteLength(tail, "utf8"); + return `${head}${elisionMarker(elided)}${tail}`; }; +const elisionMarker = (elided: number): string => `[… ${elided} bytes elided …]`; const childDiagnostic = (stdout: string, stderr: string, secretValues: readonly string[]): string => { const output = stderr.trim().length > 0 ? stderr : stdout; const redacted = redactCredentialText(output, secretValues, Number.MAX_SAFE_INTEGER).trim(); - const bounded = utf8Tail(redacted, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); + const bounded = boundedDiagnosticWindow(redacted, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); return bounded.length > 0 ? `: ${bounded}` : ""; }; @@ -104,7 +123,13 @@ export const readChild = ( const stdout: Buffer[] = []; let stdoutTail = Buffer.alloc(0); let droppingStdoutLine = false; + // Both ends of stderr, retained as it streams: the head frozen once it is + // full, the tail sliding. A single sliding tail dropped the head at capture + // time, which is where the cause of an early death lives — no later window + // can recover what was never kept. + let stderrHead = Buffer.alloc(0); let stderrTail = Buffer.alloc(0); + let stderrBytes = 0; let stdoutBytes = 0; let stdoutRemainder = Buffer.alloc(0); let droppingNdjsonLine = false; @@ -113,8 +138,18 @@ export const readChild = ( let cleanupStarted = false; let classifiedFailure: Error | undefined; let failureScanTail = ""; - const stderrRetentionBytes = CLI_ENGINE_MAX_DIAGNOSTIC_BYTES - + Math.max(0, ...secretValues.map((secret) => Buffer.byteLength(secret, "utf8"))); + /** + * Each retained end carries its reported share PLUS one whole secret, which + * is the invariant that keeps exact redaction exact: a secret that reaches + * the reported window can extend at most its own length past that window's + * cut, so retaining that much more on each side means the redactor always + * sees the secret whole. Halving one shared budget instead broke it — a + * 2000-byte secret was cut in the middle and its tail fragment + * (`…qqq-secret-end`) reached the diagnostic verbatim. + */ + const stderrSecretAllowance = Math.max(0, ...secretValues.map((secret) => Buffer.byteLength(secret, "utf8"))); + const stderrHeadBytes = Math.floor(CLI_ENGINE_MAX_DIAGNOSTIC_BYTES / 2) + stderrSecretAllowance; + const stderrTailBytes = CLI_ENGINE_MAX_DIAGNOSTIC_BYTES - Math.floor(CLI_ENGINE_MAX_DIAGNOSTIC_BYTES / 2) + stderrSecretAllowance; const settle = (action: () => void): void => { if (settled) return; settled = true; @@ -221,18 +256,33 @@ export const readChild = ( } stdout.push(value); }; - const retainStderrTail = (chunk: Buffer): void => { + const retainStderrWindow = (chunk: Buffer): void => { const value = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); classifyFailure(value); - if (value.length >= stderrRetentionBytes) { - stderrTail = Buffer.from(value.subarray(value.length - stderrRetentionBytes)); + stderrBytes += value.length; + if (stderrHead.length < stderrHeadBytes) stderrHead = Buffer.concat([stderrHead, value.subarray(0, stderrHeadBytes - stderrHead.length)]); + if (value.length >= stderrTailBytes) { + stderrTail = Buffer.from(value.subarray(value.length - stderrTailBytes)); return; } - const overflow = stderrTail.length + value.length - stderrRetentionBytes; + const overflow = stderrTail.length + value.length - stderrTailBytes; stderrTail = Buffer.concat([overflow > 0 ? stderrTail.subarray(overflow) : stderrTail, value]); }; + /** + * The retained stderr, reassembled exactly. + * + * Nothing was elided while the whole output fitted the budget, and then the + * two ends overlap: they tile the stream, so dropping the overlap from the + * tail rebuilds it byte-identically. Above the budget the ends are joined by + * the marker, which names how many bytes never reached this process at all. + */ + const retainedStderr = (): string => { + const elided = stderrBytes - stderrHead.length - stderrTail.length; + if (elided <= 0) return Buffer.concat([stderrHead, stderrTail.subarray(stderrHead.length + stderrTail.length - stderrBytes)]).toString("utf8"); + return `${stderrHead.toString("utf8")}${elisionMarker(elided)}${stderrTail.toString("utf8")}`; + }; child.stdout?.on("data", retainStdout); - child.stderr?.on("data", retainStderrTail); + child.stderr?.on("data", retainStderrWindow); const timer = timeoutMs === undefined ? undefined : setTimeout(() => abort(tagCliChildFailure(new Error(options.timeoutErrorMessage ?? "CLI engine timed out"), "wake_timeout")), timeoutMs); child.once("error", abort); child.once("close", (code, signal) => { @@ -244,7 +294,7 @@ export const readChild = ( return; } settle(() => reject(tagCliChildFailure(classifiedFailure ?? new Error(`CLI engine exited ${code ?? signal}${childDiagnostic( - retainedStdout.toString("utf8"), stderrTail.toString("utf8"), secretValues + retainedStdout.toString("utf8"), retainedStderr(), secretValues )}`), "engine_exit"))); }); }); diff --git a/src/pi/cliSessionOutput.test.ts b/src/pi/cliSessionOutput.test.ts index 3e6ec4a..65d0b2c 100644 --- a/src/pi/cliSessionOutput.test.ts +++ b/src/pi/cliSessionOutput.test.ts @@ -34,11 +34,15 @@ test("verbose progress stderr is drained without invalidating a bounded successf } }); -test("failed verbose stderr retains only a redacted bounded diagnostic tail", async () => { +test("failed verbose stderr retains a redacted bounded window of its head AND its tail", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-cli-stderr-diagnostic-")); const engine = path.join(root, "verbose-failure.mjs"); const secret = "bounded-diagnostic-secret-value"; await writeFile(engine, [ + // The real shape of an early death: the cause first, then a flood, then + // the last thing the child happened to print. A pure tail kept only the + // last of the three. + `process.stderr.write(${JSON.stringify("first-cause: profile refused\n")});`, `process.stderr.write("p".repeat(${CLI_ENGINE_MAX_OUTPUT_BYTES * 8}));`, `process.stderr.write(${JSON.stringify(` final-error ${secret}`)});`, "process.exitCode = 7;" @@ -54,7 +58,9 @@ test("failed verbose stderr retains only a redacted bounded diagnostic tail", as await assert.rejects(readChild(child, 10_000, [secret]), (error: unknown) => { assert.ok(error instanceof Error); assert.match(error.message, /CLI engine exited 7/); + assert.match(error.message, /first-cause: profile refused/, "the head of the output is the error, and must survive the bound"); assert.match(error.message, /final-error \[REDACTED\]/); + assert.match(error.message, /\[… \d+ bytes elided …\]/, "and what was dropped between the two ends is named"); assert.equal(error.message.includes(secret), false); assert.ok(Buffer.byteLength(error.message) <= CLI_ENGINE_MAX_DIAGNOSTIC_BYTES + 80); return true; @@ -64,6 +70,45 @@ test("failed verbose stderr retains only a redacted bounded diagnostic tail", as } }); +/** + * The mirror of the long-secret case, at the other cut. Retaining both ends + * only stays safe while each end carries a whole secret's worth beyond its + * reported share: sizing the two ends by halving one shared budget let a + * 2000-byte secret straddle the cut and leak its own tail verbatim. + */ +test("a secret straddling the head's own retention edge is still redacted whole", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-cli-head-edge-secret-")); + const engine = path.join(root, "head-edge-secret.mjs"); + // The first ten bytes are a distinctive token, so any surviving prefix of + // this credential is visible to the assertion rather than merely shorter. + const secret = `CREDENTIAL-${"z".repeat(200)}-END`; + // Land it across the edge of the head's REPORTED share + // (`CLI_ENGINE_MAX_DIAGNOSTIC_BYTES / 2`): 80 bytes inside the window, the + // rest past it. Only the extra secret-length the head retains beyond that + // share lets the redactor match it whole; without it those 80 bytes are a + // verbatim credential prefix in the diagnostic. + const pad = Math.floor(CLI_ENGINE_MAX_DIAGNOSTIC_BYTES / 2) - 80; + await writeFile(engine, [ + `process.stderr.write("h".repeat(${pad}));`, + `process.stderr.write(${JSON.stringify(secret)});`, + `process.stderr.write("p".repeat(${CLI_ENGINE_MAX_OUTPUT_BYTES * 8}));`, + `process.stderr.write(${JSON.stringify(" terminal-detail")});`, + "process.exitCode = 9;" + ].join("\n")); + try { + const child = spawnEngine({ + command: process.execPath, commandArgs: [engine], engine: "agy", + maxToolTurns: 1, timeoutMs: 10_000 + }, "verbose", { cwd: root }, undefined); + await assert.rejects(readChild(child, 10_000, [secret]), (error: unknown) => { + assert.ok(error instanceof Error); + assert.match(error.message, /terminal-detail/u, "the tail still reports the last words"); + assert.doesNotMatch(error.message, /CREDENTIAL|z{32}/u, "no fragment of the secret may survive the cut"); + return true; + }); + } finally { await rm(root, { recursive: true, force: true }); } +}); + test("redacts a 2000-byte exact secret before retaining a failed stderr tail", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-cli-long-secret-")); const engine = path.join(root, "long-secret-failure.mjs"); diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 147dc5c..de8319d 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -89,7 +89,23 @@ prelaunch failure ran nothing. `engineBrokerNativeClient.ts` redacts that tail exactly as the CLI child path redacts a failed engine child (`redactCredentialText` with the turn's own provider/MCP capabilities as exact secrets, the same `CLI_ENGINE_MAX_DIAGNOSTIC_BYTES` bound) and flattens it to -one line as `diagnostic.reason`. It is an optional, control-character-free +one line as `diagnostic.reason`. + +Two rules that live capture taught, both cheap and both load bearing. The +bytes are **decoded**, never stringified: `Uint8Array.prototype.toString("utf8")` +ignores its argument and renders bytes as comma-separated decimals, and a +worker's last words reached an operator as +`reason=108,111,110,101,46,32,87,104,101,110,...` — a string, control-character +free, inside the bound, and passing every check on the way out. So the frame is +normalized to a `Buffer` once on entry and the diagnostic goes through an +explicit `TextDecoder`, which also replaces rather than throws on the +multi-byte sequence a byte-counted window can cut in half. And the window +keeps **both ends** (`boundedDiagnosticWindow`): a worker that dies early +prints its error before it echoes its input, so a pure tail is the echo. The +marker is paid out of the same budget, and output that fits is returned +byte-identical. The launcher's own 512-byte window is still tail-only — see +`native/AGENTS.md`, it needs an artifact rebuild — so the head of a large blob +is still lost before Daimon sees it. It is an optional, control-character-free member of the sealed terminal response's closed diagnostic — admitted by `engineBrokerProtocol.ts` only for the statuses where a worker ran and spoke — so it replays with the sealed record and reaches the operator through diff --git a/src/runtime/engineBrokerNativeClient.test.ts b/src/runtime/engineBrokerNativeClient.test.ts index f960d63..4b0d8db 100644 --- a/src/runtime/engineBrokerNativeClient.test.ts +++ b/src/runtime/engineBrokerNativeClient.test.ts @@ -1,10 +1,10 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { CLI_ENGINE_MAX_DIAGNOSTIC_BYTES } from "../pi/cliChildOutput.js"; +import { boundedDiagnosticWindow, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES } from "../pi/cliChildOutput.js"; import { decodeNativeBrokerResult,encodeNativeBrokerTurn,ENGINE_BROKER_NATIVE_DIAGNOSTIC_BYTES,ENGINE_BROKER_NATIVE_RESULT_BYTES,NativeBrokerTurnFailure } from "./engineBrokerNativeClient.js"; const turnId="turn-1"; -function frame(values:Readonly<{status?:number;uid?:number;pid?:number;exit?:number;signal?:number;ticks?:bigint;stage?:number;failure?:number;profile?:number;diagnosticLength?:number;diagnostic?:string;text?:string}>={}):Buffer{ +function frame(values:Readonly<{status?:number;uid?:number;pid?:number;exit?:number;signal?:number;ticks?:bigint;stage?:number;failure?:number;profile?:number;diagnosticLength?:number;diagnostic?:string|Buffer;text?:string}>={}):Buffer{ const text=Buffer.from(values.text??""),diagnostic=Buffer.from(values.diagnostic??"");const out=Buffer.alloc(ENGINE_BROKER_NATIVE_RESULT_BYTES+text.length+diagnostic.length);out.writeUInt32LE(2,0);out.writeUInt32LE(values.status??0,4);out.writeUInt32LE(values.uid??2200,8);out.writeUInt32LE(text.length,12);out.writeInt32LE(values.pid??42,16);out.writeInt32LE(values.exit??0,20);out.writeInt32LE(values.signal??0,24);out.writeBigUInt64LE(values.ticks??123n,32);out.write(turnId,40);out.writeUInt32LE(values.stage??7,108);out.writeUInt32LE(values.failure??0,112);out.writeUInt32LE(values.profile??0,116);out.writeUInt32LE(values.diagnosticLength??diagnostic.length,120);text.copy(out,ENGINE_BROKER_NATIVE_RESULT_BYTES);diagnostic.copy(out,ENGINE_BROKER_NATIVE_RESULT_BYTES+text.length);return out; } @@ -41,3 +41,59 @@ test("a failed worker's own last words cross as a redacted, bounded reason",()=> }); assert.throws(()=>decodeNativeBrokerResult(frame({status:2,stage:6,failure:5,exit:1}),turnId,[]),(error:unknown)=>error instanceof NativeBrokerTurnFailure&&error.diagnostic.reason===undefined,"a worker that said nothing reports no reason rather than an empty one"); }); + +/** + * The assertion is the SENTENCE. A live turn reported its worker's last words + * as `reason=108,111,110,101,46,32,87,104,101,110,...` — the bytes of "lone. + * When ..." rendered as decimals, because a byte view that is not a Node + * `Buffer` answers `toString("utf8")` with a comma-separated list and every + * check the diagnostic passed on the way out (a string, no control bytes, + * under the bound) is satisfied by digits. Nothing weaker than the decoded + * text can catch that. + */ +const words = "lone. When assigned, read `room:assignment`, open my row in the desk index."; + +test("a worker's last words cross as decoded text, not as the decimals of their bytes", () => { + for (const [shape, view] of [["a Buffer", (bytes: Buffer): Uint8Array => bytes], ["a plain Uint8Array", (bytes: Buffer): Uint8Array => new Uint8Array(bytes)]] as const) { + assert.throws(() => decodeNativeBrokerResult(view(frame({ status: 2, stage: 6, failure: 5, exit: 1, diagnostic: words })), turnId, []), (error: unknown) => { + assert.ok(error instanceof NativeBrokerTurnFailure); + assert.equal(error.diagnostic.reason, words, `${shape}: the reason must be the worker's sentence, byte-identical`); + assert.doesNotMatch(error.diagnostic.reason ?? "", /^[0-9,]+$/u, `${shape}: a decimal byte list is what this regressed to before`); + return true; + }); + } +}); + +test("a window that cut a multi-byte sequence in half decodes with replacement instead of throwing", () => { + // The launcher's window is a byte count: these first two bytes are the tail + // of a three-byte sequence whose leading byte the window already dropped. + const cut = Buffer.concat([Buffer.from([0x9c, 0xa8]), Buffer.from(" grok: exiting 1")]); + assert.throws(() => decodeNativeBrokerResult(frame({ status: 2, stage: 6, failure: 5, exit: 1, diagnostic: cut }), turnId, []), (error: unknown) => { + assert.ok(error instanceof NativeBrokerTurnFailure); + const reason = error.diagnostic.reason ?? ""; + assert.match(reason, /grok: exiting 1$/u, "the legible remainder survives the cut sequence"); + assert.match(reason, /\uFFFD/u, "the cut sequence is replaced, not thrown on"); + return true; + }); +}); + +/** + * The window keeps both ends. A pure tail is what turned the one diagnostic + * this project has ever got out of a failed worker into 512 bytes of the + * worker's own prompt echoed back, with the error itself off the front. + */ +test("the bounded diagnostic window keeps the head, the tail, and a marker naming what it dropped", () => { + const blob = `START-OF-ERROR ${"m".repeat(40_000)} END-OF-ECHO`; + const window = boundedDiagnosticWindow(blob, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES); + assert.ok(window.startsWith("START-OF-ERROR "), `the head must survive: ${window.slice(0, 40)}`); + assert.ok(window.endsWith(" END-OF-ECHO"), "the tail must survive too"); + const marker = /\[… (\d+) bytes elided …\]/u.exec(window); + assert.ok(marker !== null, "the elision is named, not silent"); + assert.equal(Number(marker[1]) + Buffer.byteLength(window.replace(marker[0], ""), "utf8"), Buffer.byteLength(blob, "utf8"), "the marker's count is exactly what was dropped"); + assert.ok(Buffer.byteLength(window, "utf8") <= CLI_ENGINE_MAX_DIAGNOSTIC_BYTES, `the marker is paid for out of the same budget: ${Buffer.byteLength(window, "utf8")} bytes`); +}); + +test("output that fits the window is returned byte-identical, with no marker", () => { + for (const value of ["", "grok: exiting 1", `${"m".repeat(CLI_ENGINE_MAX_DIAGNOSTIC_BYTES - 4)}tail`]) + assert.equal(boundedDiagnosticWindow(value, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES), value, "a short diagnostic must not be reshaped at all"); +}); diff --git a/src/runtime/engineBrokerNativeClient.ts b/src/runtime/engineBrokerNativeClient.ts index 18fff46..cac8711 100644 --- a/src/runtime/engineBrokerNativeClient.ts +++ b/src/runtime/engineBrokerNativeClient.ts @@ -1,6 +1,6 @@ import { spawn } from "node:child_process"; import { redactCredentialText } from "../core/credentialRedaction.js"; -import { CLI_ENGINE_MAX_DIAGNOSTIC_BYTES } from "../pi/cliChildOutput.js"; +import { boundedDiagnosticWindow, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES } from "../pi/cliChildOutput.js"; import { terminateChild, trackCliChild } from "../pi/cliProcess.js"; export const ENGINE_BROKER_NATIVE_REQUEST_BYTES = 396; @@ -24,12 +24,23 @@ export async function runNativeBrokerTurn(executable:string,input:NativeBrokerTu } export function encodeNativeBrokerTurn(input:NativeBrokerTurn):Buffer{if(!Number.isInteger(input.slot)||input.slot<0)throw new TypeError("invalid engine broker turn");const p=Buffer.from(input.prompt),provider=Buffer.from(input.providerCapability),mcp=Buffer.from(input.mcpCapability);if(p.length<1||p.length>MAX_PROMPT||provider.length<1||provider.length>MAX_CAPABILITY||mcp.length<1||mcp.length>MAX_CAPABILITY||provider.equals(mcp))throw new TypeError("invalid engine broker turn");const c=Buffer.alloc(4+provider.length+mcp.length);c.writeUInt16LE(provider.length,0);provider.copy(c,2);c.writeUInt16LE(mcp.length,2+provider.length);mcp.copy(c,4+provider.length);const frame=Buffer.alloc(ENGINE_BROKER_NATIVE_REQUEST_BYTES+8+p.length+c.length);frame.writeUInt32LE(2,0);frame.writeUInt32LE(input.slot,4);field(frame,8,65,input.requestId);field(frame,73,65,input.turnId);field(frame,138,129,input.agentId);field(frame,267,129,input.wakeId);let o=ENGINE_BROKER_NATIVE_REQUEST_BYTES;frame.writeUInt32LE(p.length,o);o+=4;p.copy(frame,o);o+=p.length;frame.writeUInt32LE(c.length,o);o+=4;c.copy(frame,o);p.fill(0);provider.fill(0);mcp.fill(0);c.fill(0);return frame;} -export function decodeNativeBrokerResult(output:Buffer,turnId:string,secrets:readonly string[]=[]):NativeBrokerTurnResult{ +/** + * `output` is accepted as any byte view, and normalized to a `Buffer` once + * here: `Uint8Array.prototype.toString("utf8")` ignores its argument and + * renders the bytes as comma-separated decimals, which is how a live worker's + * last words reached an operator as `reason=108,111,110,101,...` instead of a + * sentence. Decoding is explicit from here on, never a stringification. + */ +export function decodeNativeBrokerResult(input:Uint8Array,turnId:string,secrets:readonly string[]=[]):NativeBrokerTurnResult{ + const output=Buffer.isBuffer(input)?input:Buffer.from(input.buffer,input.byteOffset,input.byteLength); if(output.lengthbytes.every((byte)=>byte===0)); if(output.length!==ENGINE_BROKER_NATIVE_RESULT_BYTES+length+diagnosticLength||output.readUInt32LE(0)!==2||status>=statuses.length||stage>=stages.length||failure>=failures.length||profile>1||!paddingZero||observed!==turnId||length>MAX_OUTPUT||diagnosticLength>ENGINE_BROKER_NATIVE_DIAGNOSTIC_BYTES)throw new Error("engine broker turn failed"); - const reason=workerReason(output.subarray(ENGINE_BROKER_NATIVE_RESULT_BYTES+length),secrets); + // Decoded from the caller's own view, not from the normalized frame: the + // decode is what must be correct for any byte view, and it is the step the + // live `reason=108,111,110,...` failure came from. + const reason=workerReason(input.subarray(ENGINE_BROKER_NATIVE_RESULT_BYTES+length),secrets); const diagnostic:NativeBrokerDiagnostic={status:statuses[status]!,stage:stages[stage]!,failureClass:failures[failure]!,profileApplied:profile===1,...(reason===undefined?{}:{reason}),exitCode,termSignal,workerPid:pid,workerUid:uid,startTicks:ticks.toString()}; const success=status===0&&stage===7&&failure===0&&profile===0&&pid>0&&uid>=2200&&ticks>0n&&exitCode===0&&termSignal===0&&diagnosticLength===0; const prelaunch=status===1&&stage>=1&&stage<=5&&failure>=1&&failure<=5&&profile===0&&pid===0&&uid===0&&ticks===0n&&diagnosticLength===0; @@ -44,16 +55,27 @@ export function decodeNativeBrokerResult(output:Buffer,turnId:string,secrets:rea * * A failed brokered turn otherwise reports nothing but `exit=1`: the launcher * merges the worker's stdout and stderr into one pipe and publishes no output - * for a failure, so this bounded tail is the only account of why it failed. + * for a failure, so this bounded window is the only account of why it failed. + * It keeps both ends of what it is given (`boundedDiagnosticWindow`), because + * a worker that dies early prints its error before it echoes anything. * It is worker-controlled text, so it is redacted exactly as the CLI child * path redacts a failed engine child (`redactCredentialText` with the turn's * own capabilities as exact secrets, the same diagnostic bound) and flattened * to one line, because it travels inside a failure message. */ -function workerReason(tail:Buffer,secrets:readonly string[]):string|undefined{ - if(tail.length===0)return undefined; - const flattened=tail.toString("utf8").replace(/[\u0000-\u001f\u007f]+/gu," ").replace(/\s+/gu," ").trim(); - const reason=redactCredentialText(flattened,secrets,CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); +function workerReason(tail:Uint8Array,secrets:readonly string[]):string|undefined{ + if(tail.byteLength===0)return undefined; + // Decoded explicitly, and with replacement rather than a throw: the window + // is a byte count, so it can cut a multi-byte sequence in half at either + // end, and a worker's last words must not be lost to its own encoding. + const flattened=UTF8.decode(tail).replace(/[\u0000-\u001f\u007f]+/gu," ").replace(/\s+/gu," ").trim(); + // Redact first, unbounded, then window: redaction can lengthen the text + // ([REDACTED] is longer than a short secret), so bounding before it could + // hand back more bytes than the boundary admits. + const redacted=redactCredentialText(flattened,secrets,Number.MAX_SAFE_INTEGER).trim(); + const reason=boundedDiagnosticWindow(redacted,CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); return reason.length===0?undefined:reason; } +/** Non-fatal by construction: a cut multi-byte sequence becomes U+FFFD, never an exception. */ +const UTF8=new TextDecoder("utf-8"); function field(target:Buffer,offset:number,length:number,value:string):void{if(!/^[A-Za-z0-9][A-Za-z0-9._:-]*$/u.test(value)||Buffer.byteLength(value)>=length)throw new TypeError("invalid engine broker turn");target.write(value,offset,"utf8");} diff --git a/src/runtime/native/AGENTS.md b/src/runtime/native/AGENTS.md index 5e8f4f9..16c3b2d 100644 --- a/src/runtime/native/AGENTS.md +++ b/src/runtime/native/AGENTS.md @@ -46,3 +46,19 @@ bytes of the worker's merged stdout/stderr and sends them after the fixed frame, while `output_length` stays 0 as before. Every other failure sends none, and `closed_result` refuses a frame that mixes the two. The bytes are the worker's own, so the broker redacts them before they cross any boundary. + +**Known gap: that window is the wrong end.** A worker that dies early prints +its error first and then echoes its own input, so a pure tail keeps the echo: +the one live capture this has ever produced was 512 bytes of the agent's own +prompt read back, with the error already off the front and erased here. The +broker side now keeps both ends of whatever it is handed +(`boundedDiagnosticWindow` in `../../pi/cliChildOutput.ts`, the same +head-plus-marker-plus-tail shape as an oversized tool result), but it cannot +recover a head this supervisor never sent. The fix belongs in +`engineBrokerLauncherServer.inc`, where the full `used` bytes are still in +hand at the point of the `memmove`: keep the first `DBL_MAX_DIAGNOSTIC / 2` +bytes, then a marker naming the elided count, then the last +`DBL_MAX_DIAGNOSTIC / 2`. It is a source change to a *pinned* artifact, so it +lands only together with `node --import tsx src/runtime/native/build.ts` and a +re-pin of `artifacts.sourceSha256`/`x64Sha256`/`arm64Sha256`; +`artifactsManifest.test.ts` fails by design until the binaries are rebuilt. diff --git a/src/runtime/toolResultSpill.ts b/src/runtime/toolResultSpill.ts index f557c38..5955577 100644 --- a/src/runtime/toolResultSpill.ts +++ b/src/runtime/toolResultSpill.ts @@ -116,7 +116,8 @@ const measure = (content: readonly unknown[], details: Record): Buffer.byteLength(safeStringify({ content, structuredContent: details }), "utf8"); /** Cut on a code-point boundary, from the front. */ -const headUtf8 = (value: string, maxBytes: number): string => { +/** Exported for the bounded diagnostic window (`cliChildOutput.ts`): one boundary-safe implementation, not two. */ +export const headUtf8 = (value: string, maxBytes: number): string => { const bytes = Buffer.from(value, "utf8"); if (bytes.byteLength <= maxBytes) return value; let end = Math.max(0, maxBytes); @@ -125,7 +126,8 @@ const headUtf8 = (value: string, maxBytes: number): string => { }; /** Cut on a code-point boundary, from the back. */ -const tailUtf8 = (value: string, maxBytes: number): string => { +/** Exported for the bounded diagnostic window (`cliChildOutput.ts`): one boundary-safe implementation, not two. */ +export const tailUtf8 = (value: string, maxBytes: number): string => { const bytes = Buffer.from(value, "utf8"); if (bytes.byteLength <= maxBytes) return value; let start = Math.max(0, bytes.byteLength - maxBytes); From 3f063c6f2ec913e55b7b1fa3da108d558ee4e335 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 01:48:03 +0200 Subject: [PATCH 100/124] fix: keep both ends of the launcher's diagnostic window and scrub credentials split by its cut --- src/contracts/runtimeContractManifest.ts | 6 +- src/runtime/AGENTS.md | 11 +++- src/runtime/engineBrokerNativeClient.test.ts | 33 ++++++++++ src/runtime/engineBrokerNativeClient.ts | 39 +++++++++++- src/runtime/native/AGENTS.md | 43 ++++++++----- .../artifacts/daimon-engine-broker-arm64 | Bin 67456 -> 67456 bytes ...daimon-engine-broker-arm64.provenance.json | 2 +- .../native/artifacts/daimon-engine-broker-x64 | Bin 55392 -> 55392 bytes .../daimon-engine-broker-x64.provenance.json | 2 +- src/runtime/native/engineBrokerLauncher.h | 5 ++ ...ngineBrokerLauncherIntegrationLauncher.inc | 43 +++++++++++++ .../native/engineBrokerLauncherServer.inc | 59 +++++++++++++++--- src/runtime/native/fixtureWorker.c | 2 +- 13 files changed, 210 insertions(+), 35 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 41baa05..548340c 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -133,9 +133,9 @@ export const GROK_ENGINE_BROKER = { projectionVersion: "noopolis.daimon.grok-broker-projection.v1", slotPreflightVersion: "noopolis.daimon.grok-slot-preflight.v2", artifacts: { - sourceSha256: "356a8e56e44dca9ab4784ac343587c0fc4d57e23f961124d8491cf6f504f8e98", - x64Sha256: "a21efd6a47059de7c5fe6add8b23799946728dfb44845ea9b6602402e5cfad02", - arm64Sha256: "a8bf311ca82ed004dd4efd69d1b7ae9edc1264876ca9bbb94c77226d40c75381" + sourceSha256: "d6bc575ab239f3cf3140b5ec296f72f9890617a6c38ca4328a7b77d6fd9f2c61", + x64Sha256: "3fd834c512a926002e215540568f75b8f1e9d9a5362ad8a9cb661ffa771884d5", + arm64Sha256: "ffa965b506160f432839e69ffe36518c1dc8c013d651bd4c8a200d4c44df2277" } } as const; export const AGY_SUBSCRIPTION_REALM = { diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index de8319d..e87dc52 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -103,9 +103,14 @@ multi-byte sequence a byte-counted window can cut in half. And the window keeps **both ends** (`boundedDiagnosticWindow`): a worker that dies early prints its error before it echoes its input, so a pure tail is the echo. The marker is paid out of the same budget, and output that fits is returned -byte-identical. The launcher's own 512-byte window is still tail-only — see -`native/AGENTS.md`, it needs an artifact rebuild — so the head of a large blob -is still lost before Daimon sees it. It is an optional, control-character-free +byte-identical. The launcher's own 512-byte window keeps both ends too +(`diagnostic_window`, `native/AGENTS.md`), so the head of a large blob now +survives the one place it used to be erased. Its elision is a cut, and a cut +can split a capability in half into a fragment exact redaction cannot match, so +`scrubCutFragments` matches that fragment here, where the turn's capabilities +are known — on both sides of every marker and at the window's outer ends. A +margin reserved in the launcher could not do this: there, what is kept is +exactly what is sent. It is an optional, control-character-free member of the sealed terminal response's closed diagnostic — admitted by `engineBrokerProtocol.ts` only for the statuses where a worker ran and spoke — so it replays with the sealed record and reaches the operator through diff --git a/src/runtime/engineBrokerNativeClient.test.ts b/src/runtime/engineBrokerNativeClient.test.ts index 4b0d8db..104150d 100644 --- a/src/runtime/engineBrokerNativeClient.test.ts +++ b/src/runtime/engineBrokerNativeClient.test.ts @@ -97,3 +97,36 @@ test("output that fits the window is returned byte-identical, with no marker", ( for (const value of ["", "grok: exiting 1", `${"m".repeat(CLI_ENGINE_MAX_DIAGNOSTIC_BYTES - 4)}tail`]) assert.equal(boundedDiagnosticWindow(value, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES), value, "a short diagnostic must not be reshaped at all"); }); + +/** + * The cut the launcher makes is the one place exact redaction cannot reach on + * its own, and it is the cut this test straddles: the capability begins inside + * the retained head and ends inside the elided middle, so the redactor never + * sees it whole and, without the scrub, its first characters travel verbatim. + * The mirror case is the tail's leading edge, where the capability's last + * characters survive instead. + */ +test("a capability the launcher's window cut in half never crosses as a fragment", () => { + const provider = `provider-${"A".repeat(34)}`, mcp = `mcp-${"B".repeat(39)}`; + const elision = "[… 9000 bytes elided …]"; + const cut = `grok: refused ${provider.slice(0, 20)}${elision}${mcp.slice(mcp.length - 20)} exiting 1`; + assert.throws(() => decodeNativeBrokerResult(frame({ status: 2, stage: 6, failure: 5, exit: 1, diagnostic: cut }), turnId, [provider, mcp]), (error: unknown) => { + assert.ok(error instanceof NativeBrokerTurnFailure); + const reason = error.diagnostic.reason ?? ""; + assert.doesNotMatch(reason, /A{12}|B{12}|provider-A|BBB-?mcp/u, `no credential fragment may cross: ${reason}`); + assert.match(reason, /grok: refused \[REDACTED\]/u, "the head's cut fragment is marked where it was"); + assert.match(reason, /\[REDACTED\] exiting 1$/u, "and so is the tail's"); + assert.ok(reason.includes(elision), "the launcher's own elision marker is left alone"); + return true; + }); +}); + +test("ordinary words at a cut are not eaten by the fragment scrub", () => { + const provider = `provider-${"A".repeat(34)}`; + const words = "grok: profile refused, exiting 1"; + assert.throws(() => decodeNativeBrokerResult(frame({ status: 2, stage: 6, failure: 5, exit: 1, diagnostic: words }), turnId, [provider]), (error: unknown) => { + assert.ok(error instanceof NativeBrokerTurnFailure); + assert.equal(error.diagnostic.reason, words, "text that is not a credential fragment is untouched"); + return true; + }); +}); diff --git a/src/runtime/engineBrokerNativeClient.ts b/src/runtime/engineBrokerNativeClient.ts index cac8711..de6406a 100644 --- a/src/runtime/engineBrokerNativeClient.ts +++ b/src/runtime/engineBrokerNativeClient.ts @@ -72,10 +72,47 @@ function workerReason(tail:Uint8Array,secrets:readonly string[]):string|undefine // Redact first, unbounded, then window: redaction can lengthen the text // ([REDACTED] is longer than a short secret), so bounding before it could // hand back more bytes than the boundary admits. - const redacted=redactCredentialText(flattened,secrets,Number.MAX_SAFE_INTEGER).trim(); + const redacted=redactCredentialText(scrubCutFragments(flattened,secrets),secrets,Number.MAX_SAFE_INTEGER).trim(); const reason=boundedDiagnosticWindow(redacted,CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); return reason.length===0?undefined:reason; } /** Non-fatal by construction: a cut multi-byte sequence becomes U+FFFD, never an exception. */ const UTF8=new TextDecoder("utf-8"); + +/** + * A credential the launcher's window cut in half, at either side of a cut. + * + * Exact redaction matches a secret whole, so a secret a cut split survives as + * a fragment it can never match: the piece before a cut can end with a + * secret's prefix, and the piece after it can begin with a secret's suffix. + * The trick that answers this where Daimon owns both ends — retain one whole + * secret more than is reported (`cliChildOutput.ts`) — cannot work at this + * boundary, because the launcher's window *is* what it sends: a margin + * reserved there would be reported along with everything else. So the fragment + * is matched here, where the turn's own capabilities are known, and every cut + * the window can make is covered: the two sides of each elision marker, and + * the outer ends, where the launcher's capture itself stopped reading. + * + * Only a fragment long enough to be a credential is scrubbed. Below + * {@link MIN_CREDENTIAL_FRAGMENT} characters a piece of a random token is + * indistinguishable from ordinary words and carries nothing usable, and + * scrubbing it would eat real text. + */ +const MIN_CREDENTIAL_FRAGMENT=12; +const ELISION_MARKER=/(\[… \d+ bytes elided …\])/u; +const scrubCutFragments=(value:string,secrets:readonly string[]):string=> + value.split(ELISION_MARKER).map((part)=>ELISION_MARKER.test(part)?part:scrubEnds(part,secrets)).join(""); +function scrubEnds(part:string,secrets:readonly string[]):string{ + let result=part; + for(const secret of secrets){ + if(secret.length<=MIN_CREDENTIAL_FRAGMENT)continue; + for(let length=Math.min(secret.length-1,result.length);length>=MIN_CREDENTIAL_FRAGMENT;length-=1){ + if(result.endsWith(secret.slice(0,length))){result=`${result.slice(0,result.length-length)}[REDACTED]`;break;} + } + for(let length=Math.min(secret.length-1,result.length);length>=MIN_CREDENTIAL_FRAGMENT;length-=1){ + if(result.startsWith(secret.slice(secret.length-length))){result=`[REDACTED]${result.slice(length)}`;break;} + } + } + return result; +} function field(target:Buffer,offset:number,length:number,value:string):void{if(!/^[A-Za-z0-9][A-Za-z0-9._:-]*$/u.test(value)||Buffer.byteLength(value)>=length)throw new TypeError("invalid engine broker turn");target.write(value,offset,"utf8");} diff --git a/src/runtime/native/AGENTS.md b/src/runtime/native/AGENTS.md index 16c3b2d..94627d9 100644 --- a/src/runtime/native/AGENTS.md +++ b/src/runtime/native/AGENTS.md @@ -47,18 +47,31 @@ frame, while `output_length` stays 0 as before. Every other failure sends none, and `closed_result` refuses a frame that mixes the two. The bytes are the worker's own, so the broker redacts them before they cross any boundary. -**Known gap: that window is the wrong end.** A worker that dies early prints -its error first and then echoes its own input, so a pure tail keeps the echo: -the one live capture this has ever produced was 512 bytes of the agent's own -prompt read back, with the error already off the front and erased here. The -broker side now keeps both ends of whatever it is handed -(`boundedDiagnosticWindow` in `../../pi/cliChildOutput.ts`, the same -head-plus-marker-plus-tail shape as an oversized tool result), but it cannot -recover a head this supervisor never sent. The fix belongs in -`engineBrokerLauncherServer.inc`, where the full `used` bytes are still in -hand at the point of the `memmove`: keep the first `DBL_MAX_DIAGNOSTIC / 2` -bytes, then a marker naming the elided count, then the last -`DBL_MAX_DIAGNOSTIC / 2`. It is a source change to a *pinned* artifact, so it -lands only together with `node --import tsx src/runtime/native/build.ts` and a -re-pin of `artifacts.sourceSha256`/`x64Sha256`/`arm64Sha256`; -`artifactsManifest.test.ts` fails by design until the binaries are rebuilt. +**The window keeps both ends.** A worker that dies early prints its error +first and then echoes its own input, so a pure tail kept the echo: the one live +capture this had ever produced was 512 bytes of the agent's own prompt read +back, with the error already off the front and erased here. `diagnostic_window` +keeps the first `DBL_MAX_DIAGNOSTIC / 2`, then `DBL_DIAGNOSTIC_ELISION` naming +the bytes dropped, then the last `DBL_MAX_DIAGNOSTIC / 2`, all inside the same +bound — the marker is sized against `used`, the largest count it can carry, so +the budget holds for every input, and a `snprintf` that will not fit falls back +to the tail. Output that already fits is left in place, byte-identical, with no +marker. The marker text is byte-identical to the TypeScript window's +(`boundedDiagnosticWindow`), so one grep finds an elision on either side of the +boundary. + +That elision is a *cut*, and a cut can split a turn capability in half, leaving +a fragment exact redaction can never match. The answer used where Daimon owns +both ends — retain one whole secret more than is reported — cannot work here, +because what this window keeps is exactly what it sends: a margin reserved here +would be sent too. So the fragment is scrubbed where the capabilities are +known, in `engineBrokerNativeClient.ts` (`scrubCutFragments`), on both sides of +every marker and at the window's outer ends. + +Changing any of the six pinned launcher sources means rebuilding: `node +--import tsx src/runtime/native/build.ts`, then re-pin +`artifacts.sourceSha256`/`x64Sha256`/`arm64Sha256` in the contract manifest and +re-emit it. `artifactsManifest.test.ts` fails by design until that is done. The +adversarial suite is `docker build -f Dockerfile.integration -t .` in this +folder and `docker run --rm --privileged `; `worker_flood_case` is the +head-and-tail cover and fails first if the window regresses to a tail. diff --git a/src/runtime/native/artifacts/daimon-engine-broker-arm64 b/src/runtime/native/artifacts/daimon-engine-broker-arm64 index 8f89d08f2297daaa569b69b164f914dd6d52455a..3ce7ade6e2817f7cc88e9b1fc5c33df78c69292f 100755 GIT binary patch delta 3187 zcmYjU3sjWH6`uM3W!dGS;PMhgb|HYQh>x%!?|%XDm6I6coV=Q55sgXl5d{>5W`B%L z+L9cZP7G0_S=-ozt!Xf8QupKqF|p>vL|jd3(wuZbTTk1h2ah6a5U1ZDHvP}Jd~@%e z|IVFn?%aRTcNz3uhOU8#S?y%vnbpp|et(*)`Ih!0`ngEgKFDEyUcsq?Cg(He;e=Car(edkf9VSvoN|&5grR7f9t4ld_zdS zQ?41F_P;eF35gpCBMw)h=64ZbisrY1cu7lI$T_ZIz6<%ej~j2e71QNUMnv6cgTj(P zO;W=`c0skcy+dk}mzwlIP)+n!Aen36w@ERbk#=d@_F+Ni`6o4gL7hG^uK;t>NJ672 zjMP~)JhpLQs;v-PI)tgevGZ%0LUlakOmq$YR=5l>f= zI=_IeQ*M^Hk?B?Exc+~c?woLtF3F~3(v2OY-EfXKO}({~;8uC~DPAQ;58QJ@bn{_x zRN976ezGW*!RAJc4xB^~x`HHSqVHnS4X{V`{xTpMazBlJZAW07I~Fc?$6!n8WJiEX zei0=K`t4{zHQ8QyQ*`S^#1TAB8Ks`ueE13U03u4d=DWDv)osxZm8aGQE`yy&@;IFM5SIR#?)<)W1MB81l#5XaCI$+cn+LUOsS z*kZQG{mdthJ-n59T8mga zKUi!@{VTj<@IIxcV2xzEqUg2#x?X!>Bk`Z(XE5V5W>jN_y!OGhn*a8Ui;cv8lGBp3 zdxMyv<%9}@h*6KVzk%L2OX|t?4(R8le?m{P%RD97ec95VWOqUzkbbEn`a$TY_@)v| z^sAFr$IrYWNbg-C(V{C+63Kp{y z))E$mu>o?(Rl;`mm9T%$p4u}Tx6}@4>^HJy{o(AN88JOM&}Tt%sEL_uCUz+$wkI2U z79sa{UD8bKgl=Mgw@mBFgz*l@p`eNFBxWY7hC(V`Ma-rbv9TH>3;oW>RM;ft4WyJ2 zt&|OgPwA0!(XYEI`%0P3W8`m?&Qx4xe!4V1^Do!1$HRtAk0}J5B9f(Jnrs4o6uVfZunSMA6C{O^9krF7 zQogn#Rk?7DKV9*(*(q~C_7yx{t5}*ixffX)ag*bEjAP{(naUEGGHLI?I1=OWw)cCe zRS!&jT_VZ#E6DRb-+nEF(mvkrOrpd5YvYiT%-ui_6(M>H)GAKNU-Y-r>0 z(EB2w+ZAR*hm4hesplCc^l0ffL&xei;t5>EO)ElQu)_AjO&9MVJ3)JbJ)hI+>KS?z z8y|~&{ShtcMvP2rE%6g$sBlnC_V)7n>MFXCAF8e}KXX<3G{2dDUtOhyuJQ$o7A*40 zPBAX>2ayTxrZ*A+^c%wQvVlw;sxmQ3x+b~-rJl`1i)M0>_Ynw(Y}dxb#3DcCJ&PWp z3EW&$5`PvxT}2lL54X!@1Am|B_|qcvfqzGW=)0y)~VF z#P_%EqbGUMwl?MPLH_o(?bOBRZ@0w1^tljs$=vX)fVAU8&hrPhuZ+KV`bRP(0T3n{hh35JewKcf;lCc#^^LpRju@fex z`QGkT2AUVqyk5RVNb~ycy#`tk+cl}*#?#${20Az5$Nw3+lf!61L~F~f*IHZt^k5jx GQvM6q%iP=m delta 3047 zcmYjT4|J2&6@Tyh(ll)a3T;VgDNTxPZ3=}-(|^$ROH-gC(^_*jQO_ZzJE9g*{hwG3_V@0e zci(;Y-Fs6{kIvJh>ph+Ntc#3&&$`&(-Kf8|ymy`9f1?4xgmt`@FIo>sxi;W?oaOaITba9qs{nh5D7 z{tE7%y^dN#S7xuK)E`Oj%`!EE^qDm^z}Q7>Os^rWpUyuNs41g?7gf-Zi_%niIn3QHX`caq0_f;CH!jD(x(PR8< z<;-b=@C%y(r$V|jB7FDxbz0cns12m>$yLQk^3+|tuFB~C@S1S6a?Ia z$9XJKaW@uQxq{p=khhRxmnBLPP?60 z`zOkxS1d8mQog@7BV`^mq?}m8{M}k(-ePdhxx{LQHyua?DWHhX)2i`-TH`c5EM$(H za}rfDd?H)KU&&tO=J^?UBRG8o+NafAoRMQu6i;iP$J2WLaS|Tn_rT&muxNsXY{A${ zE&QFug$@$F#J%(L?;$eQl0;PuKH6~hb?k`TCixtT7kmP|HpXXLWSlbDkz%ROw%EX5 zmU_v<`+o3W^MUyp6aF+Vm3cbHCofp+UJXgUn$tniJ!!Z_DyYL&#g=%hSPIq#P)WRs z8O+rz8EZZ0qO+Qv^jEVataYG+v1+zet6^W2P70Kw!Y!Z zwWqNHn8na9jU<_!S962jLQnMHe0$w=gEN^)<_rF2-QCu2&`PqkM7uStAWgCf;;2A{ zdZK&(i%JFWM01-|ZP=7Vj>S}JN$~ReJ16hKfwG2W(NL7%BYbE5eDmAzmvZPC)IPO5 z1!=s!q}vR?L$c!G>jVB}{d%20nQi244VgNVfvx468k#Ko4a`7O*m%ZzFZxs1$yf@z zU{~1rr`53}C6T9E8%?$RVnd#y4spHqOiG)K0nsn#FKG>TRE^(7mPW<+x((}cxkjY2 zM9RkHUaU*89(BDJpe9drY|^KZ;VX#qvrwLGCZ#cMYs{v8et+X#)WQRe#dYU~g>OBP z^_OY+@G6|Tw6LZmDm%wyQ!VdHnHw_jf5ckoNjZ||z&lv__yl`C4fEOdm9&_@WarCj z;7yY=yofS0#)TDpBo(|xVdivCv|Q>XU#1)5XG=YPOXTTh;*SoZ)0SZ_m>_%khLd~A zR-}EZ#lgKzP18v#8y!h|bGep%eXa~^B?%LKqAITD1W)sLlb!bP!i5bfzZ{f0EnLDM zTWD9Zuks@cZ(sO-(^o~sfO8sj|B$dfSK8GVb zvPByaV~0G+OBOAsm3;f6`B_(>^Qf4@$isK#@;v`^Q9kYG!;9=$$DkkXXbOz1TNwEK zhA_$;9_IHfo=3a*w#7f8asJuj$LTiyqvi(up6zTdrXBoP^NuP1h}6l}d|JIZu)VPe zzxR&%DHm>vRZKDyp4odT!e75*Ctc6Wmn^3ba<-(K*6_SLSJIx)wma>V)`$MQ)McXU z_`=6N3RSPJ(7Uxag!mer-*Q-pEtm)-B*ZKhrhKLYH zfu=qou1onJgm@5lfB%#ahk=m~h4>ig{78r=D0`_q;sB`k2>(TA8E@${(2x1D z&Y4P6h(Fm`L=W+UomKQQ|EzP4;yB7DcCF26d0Pk*w!npHk8ZDeE0)ud%KP5tySs|$ z0p8b@K?nKSuEomIcleBri|VH%xy{HOcK9jCtF!HSQQOP4M*f2KBGc?I^=dEoXm;Q8 z+OD0N{pUNh-QAk~mFKiqw`fvWLXJozeS0R?l*zmQ0w%|n)e0cM{S-R8TCqpLS0yKlX&6PpF<{Mn4^a;M- zWt_Ha0OcSzkEcgkI`CS4eg>AOLSMMr-CWz^=glK6%HXQd?vaxQWw6D!LPyI|2Up3R vaJ$O4TSqG=r$2U+{|iA9zWlp^o%H(l=XsGFNZD{hP@(FyG9iM5#2k4W9s4#ve+1WX&`&_Rk zJouG{It{p0k$)|mHu?d5przin8D z>E)G%{)kkPtp>gF!grLe3F)zd=*L<@hl+77C~zFf8yh{$3U}gX7DLKjUUV|U1a$woTa6Ff%p>@%$Ac=FD;I7eY&viq{=S}q`fKtz(QrR3K5ZVt(QEs-tJeM>b$LW&C` zxPnf_h0&boYE?<(5Sw}KWsa*c2O|?mmf4Rnk1cjs%NrXEkFngiCU@(XZ4)&4P{5ndM{V(SC8& zVYn=1PNX&YhH&d#zjZ*fmFcI-XQZ>ulRaa3i9c3oevz9q>uKkjON<5x=iGYz)7n+59Xf&;5-UJ9X>(>k>tNWp|&rr9?r zO?Ok#RRAg5Hg(pEN(%Vq7r;&Sj2Kg<#ws_OtJsb5CbqCy2bmaWA(a#CjybLBv2dk@>$}B8{fL zViabu$T!t3QLZ+U2y-8ypplF=rwI*>44~BOM0=qpc8|v4k-Bp8T6K=c1-l9AI@S8x~-ffK}qIQJQS@8&4SP;MQNoI;^rN z;4L2-Reiw(G%8sxu!hNyW0J9K{!HkUIl>{R?0|h$9N#=hss$o-*^*~m$0e%eSy)BLifp=2TaS zvf&i^wKi4f4%`qVN$oJINtG&|8W6W&RK?~rYq6AGBu6I>kvhSrx%e2LJo>kSOOhw# zy~S9;i66z+LX^!o*`5Pxx6hHp+~bgLgS3)0Ijj70j%crDd@cC*7n7$fJqGWA2}awY z#UxF*KY)AGPI4$!#_ zdXP#Gc{4da^v6YNseW;R98d1qsSzV9%H!h;q$PRAfXrK3>7%+17=|Un=-mq{lT^lz zj8^MwPi6n;gnqzoSN*A~{#H5X;99F*4oXns=V zBg!eT#r$`UY}h()7q1 z3$TJZu^ak3+o}d+{q_Q4lwyTT3rL1!88?5vmX@6xv>jq;ak3z4nQyrvC+ESdLA%?) zBbWFk6F-RXsZU1i{8fut@qVeLMcMNunUrdDWI+%(SvQrAXxyw_sHZ`>`vXMmNU&^N z$3PV!{hV^KZKR&k+P0CJp3eiL=6JZyX?snWUdeV?lJ6_GzhF6*id*@YUy*!2r^fSw zvXJylixaDm9#%~3{YJT6PpoNuC;kZyTW)lJRT!cytye=B=B60T!9c3MuMfzkYqAll zECmB|F{SxB)-IArN!kQHn*1*Kn;jYY*XIBk7i|AF)|$qOKVA$z5)^ zkM>bn#Fn`-AmJd98_^+oPR=q!E16fxqVy-aBwke;iR%Owif88L`D9CaY^*ou;VRa+ z3_oEp=$#R0d2gI2wdvi2E9Z%lK2G?)j`W+7EI8{(?v&@BrQc}2it#$ip6ohS$}8$r z(w(c4Ars*lL3F~&9oK)(!&%_kc8)dk8rDJ0P`1i<{^p*#1L4XP z*fp16#1d?-B_G+Q@sDkox>WF-Cd;Ny5z^0)>ZwEd$4JZ6qMkkG;$32rJr+}lT{`B7 zOGCPY5xYzZH`2p7Bx73di2GO#RP+y3?#>}cr{xMWPx}(5PvH4zUqR-Sj>7t4-_hr` zO&)P0iR0Y<{LXkD*1<`j-p$V-F&QE{cIGqFrQ++lnviNJVgop(p^$9eJyKYmL{9H^ z$MB%>s8^PKLMe{%wf#7|=-oq_N0B9a)(MS|`RE>V#|>UXkLSL1?$*cK?FITdujJ2r zNk7lN@WpvA>cP(P_3>WfiK!&U*L#0%P#66wO6!X#b>er>LFuz(XGsP-J{;%!p=3Mn za4w*<6~XB3l(yqc-{HIZtLSoBwyp_b4_2V~^^s+%hE2F3II&z{cfuBL{mSsdL zD=F>8yjVKx`vX`31d~~Z21QD*QkudfxFr18*1O2ILurEY8o7FCV26DUoVH*RQ{JaT zQ&Dh`pC^1pB$-hj7IM9q(tc!Nc^ZF;l$6H__BY5E<+Fvs8_CFu$$T~`s>m3q-$dzn z)j9G3bYjD;?YL}k8^INBBKIohL|)#Ecj0#GVXPj=vdXUf*Th|!uw>cwPh!8yTQ!@9Xip+UN!M*c8vSx9>!%Y2j47jVxm#+8Hfl3rC^=luc+ zUMJLxNiIV&1C^NBj$_AX7r6F%VW>#%ZB$-6Jw3RrcAN#W5#WxKtyPn{n?6OVaB$dV z!fzcU_5+kwka7M#B-S6xD`d2PBEO5g>VHc}`i{g@kLUN2nbi{x e6n+OOs!s@g@F9ql){hFcm1-(B-`)E8!v6tLVD3l& delta 4489 zcmZ8l4OmoF8ou`eGMI4(1x5Uu^lFMfFr|fxIHLnxaX|dZb$3zOG(;VZ3R>*324u$N zE|SzS)9eS^TIqRwiX2Ko7=|RtCpPg1r7RuGt&7y6ZJB24+4tN#qvU;_;l1Dcedqhm z`ObH~d%6DF5Pxk*t%2|VPC2Jt@062){FDgOKwUI>=J5%9RLMbIyp0}37&3sLMfOGx z8Jb?JD-WI1(VzpDM@~Rn7SMijm-Y?9i$jKb1KO^;w28zl4tb!aPIsl=rM(syb>%Kz zt@Au1?lubZwt8YrYeV_1#3?l;?A^w3vT{M~`i)YzPjnrJQ@L^)3D7M8x?QKYk@aKW z6t+u599Pc2?b$W%vcP{$$|iL4)nr5bv-|_3C%%=x*Hb%jZ#eHEiyn0GpLxzcxRU3W zl82|Q8+lI&^WG?x^BiYWmPiI!aY?xl~o}S zWwSIM)07pvWKviEo6?G~Tbsz3WZQteDdH%XmWjM7Qji>*=zu%%piN4v;YBwKWIk3C zE56iXi@12rCS`#<2r^aq-Wt&^x+4X0D0yq{^V%HXT>P+Au|}u5e-K?SGOj)xzS&Q% zut}yiOK1qYS7b-3`3Jjl!>(M$wCoeQ<;wliLT$EF?@b@kpn&CG%XIbfz%Z~6z;d6RV(N(#$l`Z+t%dI)q<<=F}l}E%;B?!o=tG=eREGblU zZM{XQw!Ai*Gzp$KYa{!M#kz{2s$hw#ER<%}&jH(T$3TsWg9|MKMRBl-^+qti!!J?V zy_!6fGKODF{**E-HU>_#wVJP4m3C_r^B`BuGLp)a__!y7N+mnC`!NV*o25xy;Uu}4 zoJbk$EOuC%IDz9br}t)xX}#UcV1XqU#cK}Bile!rer3h6T;Vji0=dSflczbZ%^Hd^ zNUGJ*(;QShsVhPQithfw$mWxo%Kw7dRfN8=Sjv@~r6e!M88~&=H#&U~w3-FMs(g&YQT_#F z(cK-!ad4cZ+AdOR_Gx)!d&;TbfPkHQ6jG~x%XUn~wnRZPUo%s+4h&OsOntwf(lEI) zrtopH!fJ|o4x(M@)#96d>g=z{cI!B?4XjP+l@*(0RA2FvR_hpHter>5Gaoabgj-)T>tbqLz}sqAG>-!lu<){6U=6>*FM=eq3E5!N;50(v z?T0;86W~3N)TvNu%a$u=?b)D>EY+4x8*bZU{SrLh_==_Dm*4S5@E5@&lIf1Zs*(1*shm3&R$ukHPbv<-1rH!W@x>v zN7$8L@Y~AumFk}j3ae_+kv2BsZ-|CU+Io%@r;Y6Y6eK&!W5qdA zowjU3=#P5oqq^#(#Az6`=$ytRHL#;2SJX*WsbCw)ed!~G5$A|CJw0*cT+SZtTMNVL&siTw zpO=+xvk#kk1gzJDBc-}OC!6HOF*2hNZI_HlFL(CU>7?sLUqYQh<9c8P8Z zVKfA}<)$g18>H)|YPx<9tYwrI8d!U{PKxZac^~<}o+GS3Lk4BOCA{P(^_jzjVm~>P zX_~rJZ@OCFx6iNnGx-(Ac4S%CXhV|rOVjR~WCv{f)qj#dfwed7--fgT6T48Kgt|@Km z`qSi_1?z>BlO#KP`q(iib!R8BGQFm$y8Q&Au|Z8B*$W?eH+!(K>=Zeg9nWtgSF>Y; zyc5K*P#$5&)cop6#(jCJQLyQW~6CV$&2O~zM8_o1c9uXIhj^3l2nC_VDUPauCHwr_KdwyO# zo9DYc(;mqR7Z$oa?wr>$roKC!f!b#wj%=!p2(<`DyFL`9KT=u=8EvyuV z*a1%6wTUd<_mCh@C(rM5$F(G}=FqD?@gAi(w><}NSkbH}zmr5J*Hj74NKbQ(HJlX0 z_+G4)JeFEN&vTv;2P#6MlD1I#Ln)=Z@au3=x{=g>u)t&d@KxSfolog?1hY6OHKB+~ zfY3z(HEz&d| z!1iAv&mNu_lMhWA)8LZvV>|yAX*xV#FgQv7#`u2iPD;;)kQt3*`c;>PhWL4MUsDVz zZX6JnQ%30!qBPFuCy~y^Q9^qqxz@Ns*ha|GrVPG>>}^^QU$TSJ6wNtS2Q0yU@9)Fq zg4+SEYX^z*J{n`&iFeyR<^xzckQcmz_+;|7H(|qsV<`B3g zeK>Y{LQ$N3?FOJAbHU~Ju`_{d?8Dihn+k3`sc%jnT3w4&vGaq=f?pLh?H)>-$a3Ep zlI$DK%jA!~xqK*j&9_I`b&1SqN#PA-Q_F1UvTrFJ&Bozklax_1h2SDJM(1+C72L%Y zfOFi%xxnRv8`5X&u7JP3+Kf1J_VWhebr%sjr}VF`(XFd%$c)Z3R9I diff --git a/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json b/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json index 2769a2c..74d77d1 100644 --- a/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json +++ b/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json @@ -1 +1 @@ -{"version":"daimon.engine-broker-native-build.v1","architecture":"x64","target":"linux/amd64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:356a8e56e44dca9ab4784ac343587c0fc4d57e23f961124d8491cf6f504f8e98","binary_sha256":"sha256:a21efd6a47059de7c5fe6add8b23799946728dfb44845ea9b6602402e5cfad02","install_path":"/opt/daimon/bin/daimon-engine-broker"} +{"version":"daimon.engine-broker-native-build.v1","architecture":"x64","target":"linux/amd64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:d6bc575ab239f3cf3140b5ec296f72f9890617a6c38ca4328a7b77d6fd9f2c61","binary_sha256":"sha256:3fd834c512a926002e215540568f75b8f1e9d9a5362ad8a9cb661ffa771884d5","install_path":"/opt/daimon/bin/daimon-engine-broker"} diff --git a/src/runtime/native/engineBrokerLauncher.h b/src/runtime/native/engineBrokerLauncher.h index 7d902a6..636e484 100644 --- a/src/runtime/native/engineBrokerLauncher.h +++ b/src/runtime/native/engineBrokerLauncher.h @@ -14,6 +14,11 @@ reason reaches the host instead of `exit=1`. It is a diagnostic, never the turn's output: `output_length` stays 0 on every failure. */ #define DBL_MAX_DIAGNOSTIC 512u +/* The marker that joins the two ends of an elided diagnostic, byte-identical + to the TypeScript window's (`boundedDiagnosticWindow` in + `src/pi/cliChildOutput.ts`), so one grep finds every elision on either side + of the boundary. Its own bytes are paid for out of DBL_MAX_DIAGNOSTIC. */ +#define DBL_DIAGNOSTIC_ELISION "[\xe2\x80\xa6 %llu bytes elided \xe2\x80\xa6]" #ifndef DBL_REGISTRY #define DBL_REGISTRY "/etc/daimon-engine-broker/registrations.bin" #endif diff --git a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc index 243d777..a1f8367 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc @@ -111,6 +111,48 @@ static void worker_failure_case(void) { close(p); close(c); } +/* A worker that dies after printing far more than the window: the error is at + the START of what it printed and the echo at the end, which is the shape a + pure tail got wrong. The distinctive head run STRADDLES the head's own cut — + its first bytes are inside the retained head and its later bytes are elided — + so a tail-only window loses it entirely and this case fails. */ +static void worker_flood_case(void) { + int s = connect_socket(), p = sealed("prompt"), + c = sealed_bundle("stderr-flood", "mcp.Flood-2"); + struct dbl_request q = request(); + struct dbl_result r; + size_t printed = strlen("HEAD-OF-ERROR grok: profile refused ") + 4096u + + strlen(" TAIL-OF-ECHO"); + unsigned long long elided = 0; + char *reason, *marker, *shown; + send_request(s, &q, p, c); + check(read_all(s, &r, sizeof(r)) && r.status == DBL_STATUS_WORKER_FAILED && + r.exit_code == 1 && r.output_length == 0 && + r.diagnostic_length > 0 && + r.diagnostic_length <= DBL_MAX_DIAGNOSTIC, + "worker flood diagnostic length"); + reason = calloc(1, r.diagnostic_length + 1); + check(read_all(s, reason, r.diagnostic_length), "worker flood diagnostic"); + check(!strncmp(reason, "HEAD-OF-ERROR grok: profile refused", + strlen("HEAD-OF-ERROR grok: profile refused")), + "worker flood head"); + shown = strstr(reason, " TAIL-OF-ECHO"); + check(shown && shown[strlen(" TAIL-OF-ECHO")] == 0, "worker flood tail"); + marker = strstr(reason, "[\xe2\x80\xa6 "); + check(marker && sscanf(marker, "[\xe2\x80\xa6 %llu bytes elided", &elided) == 1, + "worker flood marker"); + /* The count is exactly what was dropped: everything printed, less the two + ends that survived (the whole window less the marker itself). */ + size_t marker_length = (size_t)(strchr(marker, ']') - marker) + 1u; + check((size_t)elided + (size_t)r.diagnostic_length - marker_length == printed, + "worker flood elided count"); + char extra; + check(read(s, &extra, 1) == 0, "worker flood EOF"); + free(reason); + close(s); + close(p); + close(c); +} static void org_cases(void) { check(!setgid(DBL_BROKER_UID) && !setuid(DBL_BROKER_UID), "drop broker uid"); client_case(); @@ -119,6 +161,7 @@ static void org_cases(void) { output_boundary_case("exact-output", 0); output_boundary_case("overflow-output", 1); worker_failure_case(); + worker_flood_case(); int s = connect_socket(), p = sealed("prompt"), c = sealed_bundle("provider.Token-1", "mcp.Token-2"); struct dbl_request q = request(); diff --git a/src/runtime/native/engineBrokerLauncherServer.inc b/src/runtime/native/engineBrokerLauncherServer.inc index 1d4cd76..72e0293 100644 --- a/src/runtime/native/engineBrokerLauncherServer.inc +++ b/src/runtime/native/engineBrokerLauncherServer.inc @@ -148,6 +148,45 @@ static pid_t launch(const struct dbl_registration *r, int executable, launch_fail(status_fd, 6); } +/* Both ends of a failed worker's own output, inside DBL_MAX_DIAGNOSTIC. + A pure tail was the wrong end for the process this exists for: a worker that + dies early prints its error first and then echoes its input, so the tail is + the echo. One live turn reported 512 bytes of the agent's own prompt read + back, with the error already off the front and erased here — the head is + only recoverable where it still exists, which is here. + The marker is paid for out of the same budget and sized against `used`, the + largest count it can ever carry, so the result never exceeds the bound for + any input; output that already fits is left exactly as it is, in place and + byte-identical, with no marker at all. A snprintf that will not fit falls + back to the tail this replaced. */ +static size_t diagnostic_window(unsigned char *bytes, size_t used) { + char marker[64]; + int width, final; + size_t budget, head, tail; + if (used <= DBL_MAX_DIAGNOSTIC) + return used; + width = snprintf(marker, sizeof(marker), DBL_DIAGNOSTIC_ELISION, + (unsigned long long)used); + budget = (width > 0 && (size_t)width + 2u <= DBL_MAX_DIAGNOSTIC) + ? DBL_MAX_DIAGNOSTIC - (size_t)width + : 0u; + if (!budget) { + memmove(bytes, bytes + (used - DBL_MAX_DIAGNOSTIC), DBL_MAX_DIAGNOSTIC); + return DBL_MAX_DIAGNOSTIC; + } + head = budget / 2u; + tail = budget - head; + final = snprintf(marker, sizeof(marker), DBL_DIAGNOSTIC_ELISION, + (unsigned long long)(used - head - tail)); + if (final <= 0 || (size_t)final > (size_t)width) { + memmove(bytes, bytes + (used - DBL_MAX_DIAGNOSTIC), DBL_MAX_DIAGNOSTIC); + return DBL_MAX_DIAGNOSTIC; + } + /* The head stays where it is; the tail moves up behind the marker. */ + memmove(bytes + head + (size_t)final, bytes + (used - tail), tail); + memcpy(bytes + head, marker, (size_t)final); + return head + (size_t)final + tail; +} static void supervise(int client, pid_t pid, int output, struct dbl_result *out) { unsigned char bytes[DBL_MAX_OUTPUT + 1] = {0}; @@ -226,16 +265,16 @@ static void supervise(int client, pid_t pid, int output, } if (out->status != DBL_STATUS_OK) { /* A failed turn publishes no output, but a worker that exited on its own - account said why on the pipe it shares with stdout, and that tail is the - only reason the host can ever see: without it a failure reads `exit=1`. - Keep a bounded tail of it and erase the rest here; the broker redacts it - before it crosses any boundary. The other failures get none: an - output-limit tail is the very payload the bound refused to publish, a - cancelled turn has no reader left, and a prelaunch failure ran nothing. */ - size_t keep = out->status != DBL_STATUS_WORKER_FAILED ? 0 - : used > DBL_MAX_DIAGNOSTIC ? DBL_MAX_DIAGNOSTIC - : used; - memmove(bytes, bytes + (used - keep), keep); + account said why on the pipe it shares with stdout, and that window is + the only reason the host can ever see: without it a failure reads + `exit=1`. Keep a bounded window of it and erase the rest here; the + broker redacts it before it crosses any boundary. The other failures get + none: an output-limit window is the very payload the bound refused to + publish, a cancelled turn has no reader left, and a prelaunch failure ran + nothing. */ + size_t keep = out->status != DBL_STATUS_WORKER_FAILED + ? 0 + : diagnostic_window(bytes, used); erase(bytes + keep, sizeof(bytes) - keep); out->output_length = 0; out->diagnostic_length = (uint32_t)keep; diff --git a/src/runtime/native/fixtureWorker.c b/src/runtime/native/fixtureWorker.c index d2f3674..5da8b2e 100644 --- a/src/runtime/native/fixtureWorker.c +++ b/src/runtime/native/fixtureWorker.c @@ -6,4 +6,4 @@ #include #include #include "engineBrokerLauncher.h" -int main(int argc,char**argv){unsigned char bundle[8200]={0};ssize_t size=pread(4,bundle,sizeof(bundle),0);if(size<6)return 21;size_t offset=0;unsigned provider_length=bundle[0]|(bundle[1]<<8);offset=2;if(offset+provider_length+2>(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output");if(!strcmp(provider,"sleep"))sleep(30);if(!strcmp(provider,"stderr-failure")){fprintf(stderr,"grok: session store unwritable\n");return 1;}if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;id_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s tmpdir=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target,getenv("TMPDIR")?getenv("TMPDIR"):"unset");for(int i=0;i(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output");if(!strcmp(provider,"sleep"))sleep(30);if(!strcmp(provider,"stderr-failure")){fprintf(stderr,"grok: session store unwritable\n");return 1;}if(!strcmp(provider,"stderr-flood")){fprintf(stderr,"HEAD-OF-ERROR grok: profile refused ");for(size_t i=0;i<4096;i++)if(fputc(0x6d,stderr)==EOF)return 32;fprintf(stderr," TAIL-OF-ECHO");fflush(stderr);return 1;}if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;id_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s tmpdir=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target,getenv("TMPDIR")?getenv("TMPDIR"):"unset");for(int i=0;i Date: Fri, 18 Sep 2026 02:12:03 +0200 Subject: [PATCH 101/124] fix: give the brokered worker a blocking stdout pipe so a large write cannot kill it --- src/contracts/runtimeContractManifest.ts | 6 ++-- src/runtime/native/AGENTS.md | 27 ++++++++++++++++++ .../artifacts/daimon-engine-broker-arm64 | Bin 67456 -> 67456 bytes ...daimon-engine-broker-arm64.provenance.json | 2 +- .../native/artifacts/daimon-engine-broker-x64 | Bin 55392 -> 55392 bytes .../daimon-engine-broker-x64.provenance.json | 2 +- .../native/engineBrokerLauncherServer.inc | 9 +++++- 7 files changed, 40 insertions(+), 6 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 548340c..3cf1242 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -133,9 +133,9 @@ export const GROK_ENGINE_BROKER = { projectionVersion: "noopolis.daimon.grok-broker-projection.v1", slotPreflightVersion: "noopolis.daimon.grok-slot-preflight.v2", artifacts: { - sourceSha256: "d6bc575ab239f3cf3140b5ec296f72f9890617a6c38ca4328a7b77d6fd9f2c61", - x64Sha256: "3fd834c512a926002e215540568f75b8f1e9d9a5362ad8a9cb661ffa771884d5", - arm64Sha256: "ffa965b506160f432839e69ffe36518c1dc8c013d651bd4c8a200d4c44df2277" + sourceSha256: "2d36898f02793a89a4a58601fa77c0d8716cc028588daad6daf76ca44d28446f", + x64Sha256: "5f92d83fe8159d1ac371f62d5382c0b52ce7fd167e3855460de66187a4c505a8", + arm64Sha256: "8bb4f6b1053abe27001219ab640601289b39c8ff8c86831cd7d8782f0d9dab8c" } } as const; export const AGY_SUBSCRIPTION_REALM = { diff --git a/src/runtime/native/AGENTS.md b/src/runtime/native/AGENTS.md index 94627d9..e7930de 100644 --- a/src/runtime/native/AGENTS.md +++ b/src/runtime/native/AGENTS.md @@ -40,6 +40,33 @@ binary unrunnable: Grok 1.0.34 re-executes itself inside bubblewrap by path (`/usr/local/bin/grok`), so the image path's root ownership, not the launcher descriptor, is what protects the sandboxed process. +**The worker's end of that pipe is a blocking pipe.** `O_NONBLOCK` is a +property of the open file description, not of a descriptor, so creating the +merged stdout/stderr pipe with `pipe2(..., O_NONBLOCK)` handed non-blocking +writes to the worker along with `pipes[1]`: Grok 1.0.34 makes the first EAGAIN +from a headless stdout write fatal (`stdout write failed: Resource temporarily +unavailable (os error 11)`) and exits 1 before it issues a single model +request, so the turn burns a wake and buys nothing. It stayed invisible until +the MCP tools became reachable and the init frame that enumerates them grew to +roughly 9.5 KB — past a pipe buffer, which is not always the 64 KiB default +(8 KiB inside the Docker Desktop VM this suite runs in). So the pipe is created +`O_CLOEXEC` only and `O_NONBLOCK` is set afterwards on `pipes[0]` alone, the +read end this process polls; that one is load bearing, because the post-exit +drain loop has no `poll` and would otherwise park on a write end some surviving +grandchild still holds. + +A blocking child cannot wedge the launcher. `serve()` runs in its own forked +handler per connection, so one worker's backpressure never reaches another +turn; `supervise` drains the pipe on every pass of a 250 ms `poll`, and both +bounds act on a child that is asleep in `write()`: crossing `DBL_MAX_OUTPUT` +stops reading (`p[1].events = 0`) and `kill(-pid, SIGKILL)`s the worker's whole +process group in the same iteration, and a client disconnect does the same — +neither is refusable by a process sleeping on a pipe. `worker_spill_case` is +the cover: the fixture shrinks its own stdout pipe to the kernel minimum, +reports the capacity it actually got, and writes four times that in one +`write`, so it straddles the buffer on any host without assuming 64 KiB while +staying under `DBL_MAX_OUTPUT`. + The result frame's last word is `diagnostic_length`, not padding: on `DBL_STATUS_WORKER_FAILED` the supervisor keeps the last `DBL_MAX_DIAGNOSTIC` bytes of the worker's merged stdout/stderr and sends them after the fixed diff --git a/src/runtime/native/artifacts/daimon-engine-broker-arm64 b/src/runtime/native/artifacts/daimon-engine-broker-arm64 index 3ce7ade6e2817f7cc88e9b1fc5c33df78c69292f..b339cf2bf459884bb0824c6dc096754e5fb6642d 100755 GIT binary patch delta 1202 zcmYjQe`s4(6h8O8m-Z!Tw{~fg{;$KgfFKHN}Og8K=v9s6n+6DUHaqf4{ zchC9mci)rG3GzAN{p2n}(v!OeIkK3(_wZYdJ7OFJvQq~Y(8>Nzfa8Rt?{<$wvqlK0 z%@A)AV!Y(}ga5IeYdQl*6JF5RM^0kT3wHruW`J#9->W<%&eXO zrG1Sh{pMulq_GLdhZ_g z)PYm|)UA+;njM{z>*Ns~N}qKrU7YL(;0yr7F+=s9hRP9SM|ha90o)h6EsI}WXNj5rSlfYav;&W; z@(rm}fh8o&Wzhl+8*yNg7CrW7LThZlCirPwa&c{f%Gaj$5;&g}G^=Y< zORA#m4hhqBFzzqSprgGH4QLum&4&dD1bNe+B5p9DGbLiz zmI0nE{M-0{l=%N}wL@RtX924A?j~2LuFo!hf?tf*VydN&kOA7-AFOWwn#<3x zK^D_b)5Sw}ddFV%!C{G{T~D^y$Y1G`!|C#lo_WSbbpYx$#$GG~Xjx$F3`QB_s~FYm zjO8)rF%~f1!B|lakW4>!dW9!%Qg_(3)BDD({?b?a;L@V|<_-6Suk{zN=-q~;#V(^8 zQ}lA!LPGSTaI@}*ImTY6%i-OYXAM99vT%L=+VZd4z->Hy@vL}ZTzr9A#}qP3e;PBH x-?_@zA=rEdFq}b=ztiH_pn3I2#)JozDvWuo<9j2{Xn z9Fb%n$Rf>)Yy6W4(ZtbYi%Sz-vkx0G-I8ri3aGg0u7xZvn;*<@Yi088cd%w$a(Uk8 z_wRk)=e<);3hGJW@@N%-#8}l{j!Y$b`zI`* zXq6Cko77Zkj>TjP)M{JE3NJ?fl^MB*i@J-{lvU5`{3T(97wUNtIdAv4VKlYa4avGC z7Lzdc(Q*eJfWef;J3O@*8#Q5FgDIN%rlqc5fDQ96`;(alBR#wOwe4WczMAIC;+LOb zodLYs7rrD}XWv*a$vtiZ%AWKk{hS&G;LHHTqte1ad0{tJ_wg{~0(kt0dILMe4;EQW zdq+-+v?t)Ken>#cb?qKsIvN;nWz4B7(s?tkb^MFB$LYrbX*(>lS}ACZ-dx7*pRwAV z4QK}*SL4g$@hpo0#?okk&Mmm9N^^ml{qho95&bqsI!xeAqi6|5yHIqx0F(@hmQl0> zfTB5%TmW84T=iZvg7S1;pI8>Z0WhKhcwYgq-@!G|NMp_Z2?3OBCl}Ri(fE>hHGvUH z(5)_syMyK^C#b1uJg&;4UA)u7ff^>DrVi^CaM`P0qQx1so4`O=n6q)APIXO*(RXzQ^G?xTfcmXH4l|506F0l*JPy zn(GqAOJOif)uv5~t}#FSL-eMdxBYIyLK~jhzh+=)P2C>;|GK}qk}iT9tQA84XsRZY z)Y@Dle2(A51-Y3Jh5B0Bt-JnYu{7^M=k4)I+~FbptYs&eqmNqZNh#eLDks-yb0|c* z>8X&*dI@js{2@pZ{DshncxU2z$VRLcI3zzHjo~-wZ>=>6X^%wc=oTpq3HzTw7?#=n>`G|wONv}jI4d4ENQ|bLkwPT<8&fVGD z(=+!UtOuW^?aT?&=|R&$>g-LD4m#CqD?WFVu~t}p+7OPT$h$PxySI4ZN5+KzGDV0B Xn3YHe7f_O69}7^=i?in;QszLZAeYX6KAq1!Cf^8<5Knyn z|9@nhW9Jc|@qKI%jV3R;C$k57suaHZ|NrHoUtq%oUKj%%@p2ZB2Xc`bh|>$?Os)@f z;Q9+R9pt_~oyjKy(->i=(r5a6cJjs`876-D$@hcg_*wq{|IaVq0CW+9 t;enUmHuD5Kq%wZk?7L$%3#0zzpL-k_lO|j3m14D-^8bJR=E%M5(g53YuD$>O delta 377 zcmaE`f%(A(<_+t$8P9Isplzqjm_GTv-!I0h$*=rnEn*lM7{0ZrQ~)VY{uVzV<1Y14HY9Qi+!qAkofuAh9s87=MfEWcvVFM)t|E0pgMG85kHmdbg+uvN161 zQUG#%I^VxYzWD!t>jD1OOF+KiCBttno&R5)zwrOR2T-tMFGxo>tA8p;V~vWzJ_ZgB z29M4b6(68YFJ@eVXbqbDB0yQZ=KcTwk#UZlM}Qjp6o3E!-(90(@?zO!r9e;po>%|> zzr6bkEGF>64`}?$tw0{gE=Le&1(1`LrpGVe0<@5UVRC<<0~asU5q&n3Zw96@_D*&P z@@Mp&JRwM*iTB*(gF!MvNIIZ8C*Kc} Date: Fri, 18 Sep 2026 02:12:03 +0200 Subject: [PATCH 102/124] test: cover a worker write four times its own pipe buffer in the native suite --- ...ngineBrokerLauncherIntegrationLauncher.inc | 49 +++++++++++++++++++ src/runtime/native/fixtureWorker.c | 3 +- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc index a1f8367..2350808 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc @@ -153,6 +153,54 @@ static void worker_flood_case(void) { close(p); close(c); } +/* A worker's single write larger than the pipe buffer must complete, not kill + it. The launcher created its stdout/stderr pipe O_NONBLOCK as a whole, and + O_NONBLOCK is a property of the open file description, so the worker + inherited a non-blocking stdout: Grok 1.0.34 turns the first EAGAIN from a + headless stdout write into `stdout write failed: Resource temporarily + unavailable` and exits 1 before it ever reaches the model. Once its init + frame outgrew a pipe buffer that was every turn, at $0 apiece. + The fixture shrinks its own stdout pipe to the kernel minimum, reports the + capacity it actually got, and writes four times that in ONE write, so the + case straddles the buffer on any host instead of assuming 64 KiB — and + stays under DBL_MAX_OUTPUT, so it is the write that is bounded here, never + the turn's output. */ +static void worker_spill_case(void) { + int s = connect_socket(), p = sealed("prompt"), + c = sealed_bundle("spill-output", "mcp.Spill-2"); + struct dbl_request q = request(); + struct dbl_result r; + unsigned capacity = 0, count = 0; + size_t head, filler = 0; + char *out, *line, extra; + send_request(s, &q, p, c); + check(read_all(s, &r, sizeof(r)) && result_padding_zero(&r) && + r.status == DBL_STATUS_OK && r.stage == DBL_STAGE_OUTPUT && + r.failure_class == DBL_FAILURE_NONE && r.exit_code == 0 && + r.term_signal == 0 && r.diagnostic_length == 0 && + r.worker_pid > 0 && r.worker_uid == 2200 && + r.output_length > 0 && r.output_length <= DBL_MAX_OUTPUT, + "worker spill survived its own oversized write"); + out = calloc(1, (size_t)r.output_length + 1u); + check(out && read_all(s, out, r.output_length), "worker spill output"); + line = strchr(out, '\n'); + check(line && sscanf(out, "SPILL cap=%u count=%u", &capacity, &count) == 2 && + capacity > 0 && count == r.output_length && + count >= capacity * 4u, + "worker spill straddles the pipe buffer"); + head = (size_t)(line - out) + 1u; + while (head + filler + 10u < (size_t)r.output_length && + out[head + filler] == 's') + filler++; + check(head + filler + 10u == (size_t)r.output_length && + !memcmp(out + r.output_length - 10, "SPILL-TAIL", 10), + "worker spill bytes arrived intact"); + check(read(s, &extra, 1) == 0, "worker spill EOF"); + free(out); + close(s); + close(p); + close(c); +} static void org_cases(void) { check(!setgid(DBL_BROKER_UID) && !setuid(DBL_BROKER_UID), "drop broker uid"); client_case(); @@ -162,6 +210,7 @@ static void org_cases(void) { output_boundary_case("overflow-output", 1); worker_failure_case(); worker_flood_case(); + worker_spill_case(); int s = connect_socket(), p = sealed("prompt"), c = sealed_bundle("provider.Token-1", "mcp.Token-2"); struct dbl_request q = request(); diff --git a/src/runtime/native/fixtureWorker.c b/src/runtime/native/fixtureWorker.c index 5da8b2e..551bea5 100644 --- a/src/runtime/native/fixtureWorker.c +++ b/src/runtime/native/fixtureWorker.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include "engineBrokerLauncher.h" -int main(int argc,char**argv){unsigned char bundle[8200]={0};ssize_t size=pread(4,bundle,sizeof(bundle),0);if(size<6)return 21;size_t offset=0;unsigned provider_length=bundle[0]|(bundle[1]<<8);offset=2;if(offset+provider_length+2>(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output");if(!strcmp(provider,"sleep"))sleep(30);if(!strcmp(provider,"stderr-failure")){fprintf(stderr,"grok: session store unwritable\n");return 1;}if(!strcmp(provider,"stderr-flood")){fprintf(stderr,"HEAD-OF-ERROR grok: profile refused ");for(size_t i=0;i<4096;i++)if(fputc(0x6d,stderr)==EOF)return 32;fprintf(stderr," TAIL-OF-ECHO");fflush(stderr);return 1;}if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;id_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s tmpdir=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target,getenv("TMPDIR")?getenv("TMPDIR"):"unset");for(int i=0;i(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output"),spill=!strcmp(provider,"spill-output");if(!strcmp(provider,"sleep"))sleep(30);if(!strcmp(provider,"stderr-failure")){fprintf(stderr,"grok: session store unwritable\n");return 1;}if(!strcmp(provider,"stderr-flood")){fprintf(stderr,"HEAD-OF-ERROR grok: profile refused ");for(size_t i=0;i<4096;i++)if(fputc(0x6d,stderr)==EOF)return 32;fprintf(stderr," TAIL-OF-ECHO");fflush(stderr);return 1;}if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;iDBL_MAX_OUTPUT)return 34;char*blob=malloc(count);if(!blob)return 35;memset(blob,'s',count);int head=snprintf(blob,count,"SPILL cap=%d count=%zu\n",cap,count);if(head<=0||(size_t)head+10u>=count)return 36;blob[head]='s';memcpy(blob+count-10,"SPILL-TAIL",10);if(write(1,blob,count)!=(ssize_t)count)return 37;free(blob);return 0;}char prompt_bytes[128]={0};FILE*prompt_file=fopen("/proc/self/fd/3","r");if(!prompt_file)return 26;size_t prompt_read=fread(prompt_bytes,1,sizeof(prompt_bytes)-1,prompt_file);fclose(prompt_file);if(!prompt_read)return 27;char fds[128]={0};size_t fds_used=0;DIR*fd_dir=opendir("/proc/self/fd");if(!fd_dir)return 29;struct dirent*fd_entry;int fd_seen[64]={0};while((fd_entry=readdir(fd_dir))){if(fd_entry->d_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s tmpdir=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target,getenv("TMPDIR")?getenv("TMPDIR"):"unset");for(int i=0;i Date: Fri, 18 Sep 2026 02:48:08 +0200 Subject: [PATCH 103/124] test: seal the spend the proxy measured when a worker's output never reaches the host --- src/runtime/AGENTS.md | 16 +++++++ src/runtime/grokEngineBrokerUsage.test.ts | 55 ++++++++++++++++++++++- src/runtime/native/AGENTS.md | 20 +++++++++ 3 files changed, 90 insertions(+), 1 deletion(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index e87dc52..2f33635 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -117,6 +117,22 @@ so it replays with the sealed record and reaches the operator through `engineBrokerControlClient.ts`'s failure message. Nothing new is written to disk: the reason travels inside the response the broker already seals. +A turn whose worker said nothing still records what it spent. The launcher can +refuse to publish a worker's output (`DBL_MAX_OUTPUT`, `native/AGENTS.md`) and +the native transport can fail outright, and in both cases `result.text` never +exists, so there are no stream frames to read usage from. `streamOrMeterUsage` +then falls to the proxy's own per-request measurements — the meter admitted and +settled every forwarded request, so the broker knows the spend even when the +worker never speaks — and `finishBrokerTurnWithUsage` seals and appends it with +`outcome: "failed"`. That is the whole of the guarantee and it is pinned by +"a worker whose work succeeded but whose output crossed the launcher bound" +(`grokEngineBrokerUsage.test.ts`), which builds the launcher's own +output-limit frame at the ABI offsets and decodes it with the shipped client. +Deleting the meter fallback, or refusing that frame shape in +`decodeNativeBrokerResult`, both turn it red. The one window that stays open is +the documented one: a crash before the turn record's rename, which the next +boot seals `usage: null`. + Evaluator inference grants (`grokInferenceGrants.ts`) let Paideia judges and the DSPy optimizer — uid 2000, the trusted evaluator side — spend the broker's Grok credential without holding it. `request_inference_grant {model, diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index e316a57..8f0f50d 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -7,7 +7,7 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import test from "node:test"; -import type { NativeBrokerTurn, NativeBrokerTurnResult } from "./engineBrokerNativeClient.js"; +import { decodeNativeBrokerResult, ENGINE_BROKER_NATIVE_RESULT_BYTES, type NativeBrokerTurn, type NativeBrokerTurnResult } from "./engineBrokerNativeClient.js"; import type { EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; import { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; @@ -291,3 +291,56 @@ test("a response that called nothing records an empty list, and one that cannot }, undefined, () => 1, () => "bad gateway"); }); + +/** + * The exact 128-byte frame `supervise()` emits when a worker crosses + * `DBL_MAX_OUTPUT`: it stops reading, SIGKILLs the process group, and publishes + * `output_length = 0` with `DBL_STATUS_OUTPUT_FAILED`. Built here at the wire + * offsets the header's `_Static_assert`s pin, so the test drives the real + * decoder rather than a hand-made exception. + */ +function outputLimitFrame(turnId: string): Buffer { + const frame = Buffer.alloc(ENGINE_BROKER_NATIVE_RESULT_BYTES); + frame.writeUInt32LE(2, 0); frame.writeUInt32LE(3, 4); frame.writeUInt32LE(2_200, 8); frame.writeUInt32LE(0, 12); + frame.writeInt32LE(4_242, 16); frame.writeInt32LE(0, 20); frame.writeInt32LE(9, 24); + frame.writeBigUInt64LE(99n, 32); frame.write(turnId, 40, "utf8"); + frame.writeUInt32LE(7, 108); frame.writeUInt32LE(7, 112); frame.writeUInt32LE(0, 116); frame.writeUInt32LE(0, 120); + return frame; +} + +test("a worker whose work succeeded but whose output crossed the launcher bound still seals the spend the proxy measured", async () => { + await withBroker(async ({ turn, usageRows, requestRows, upstreamCalls }) => { + // The worker does its real work through the real proxy — two admitted, + // metered upstream requests — and only then loses its whole output: the + // launcher refused to publish it and the turn's text never exists. The + // frame is decoded by the shipped client, so the failure reaches the turn + // exactly as the native transport delivers it. + const worker: Worker = async (send) => { + assert.equal(await send(), 200); + assert.equal(await send(), 200); + throw decodeNativeBrokerResult(outputLimitFrame(turnIdFor("foreman", "wake-output-limit")), turnIdFor("foreman", "wake-output-limit"), []) as never; + }; + await assert.rejects(turn("wake-output-limit", worker), (error: unknown) => { + assert.ok(error instanceof EngineBrokerTurnFailure); + // No limit tripped and the credential is live: this is the worker's + // transport failing, not the turn being refused. + assert.equal(error.code, "engine_failed"); + assert.equal(error.diagnostic?.failureClass, "output_limit"); + // Mutation guard: the turn has no stream to read usage from, so this can + // only come from the proxy's own per-request measurements. Falling back + // to `null` here would report a fabricated zero for real spend. + assert.deepEqual(error.accounting, { outcome: "failed", usage: { input: 5_136, cacheRead: 256, cacheWrite: 0, output: 158, total: 5_550 }, model: "grok-4.6", requests: 2, limitReason: "none" }); + return true; + }); + assert.equal(upstreamCalls(), 2); + const [row, extra] = await usageRows(); + assert.equal(extra, undefined); + assert.deepEqual([row?.outcome, row?.reason, row?.total, row?.calls, row?.complete], ["failed", "unknown", 5_550, 2, false]); + assert.notEqual(row?.total, 0, "a zero row would be byte-identical to a measured zero"); + assert.deepEqual((await requestRows()).map((value) => value.request), [0, 1], "every request the proxy answered keeps its own row"); + // The sealed record is the durable truth: a replay returns that spend and never meters again. + await assert.rejects(turn("wake-output-limit", twoRequests), (error: unknown) => + error instanceof EngineBrokerTurnFailure && error.accounting?.usage?.total === 5_550); + assert.equal((await usageRows()).length, 1, "the replayed failure is not metered again"); + }); +}); diff --git a/src/runtime/native/AGENTS.md b/src/runtime/native/AGENTS.md index e7930de..736fec4 100644 --- a/src/runtime/native/AGENTS.md +++ b/src/runtime/native/AGENTS.md @@ -67,6 +67,26 @@ reports the capacity it actually got, and writes four times that in one `write`, so it straddles the buffer on any host without assuming 64 KiB while staying under `DBL_MAX_OUTPUT`. +**Crossing `DBL_MAX_OUTPUT` is a reported status, not a lost turn.** This is +worth stating because it has been guessed at twice: a trip sets +`output_limited`, stops reading, `SIGKILL`s the worker's process group, reaps +it, and then — `disconnected` is still 0, so the branch at the end of +`supervise` runs — writes the complete 128-byte result frame with +`DBL_STATUS_OUTPUT_FAILED`, `DBL_STAGE_OUTPUT`, `DBL_FAILURE_OUTPUT_LIMIT` and +`output_length = 0`. `closed_result` admits exactly that shape, the client +relays it, and `decodeNativeBrokerResult` raises a named +`NativeBrokerTurnFailure`. So a trip costs the turn its *text* and nothing +else: the broker still seals the turn and still meters the spend the proxy +measured. A lost terminal frame, an unnamed transport failure or an unmetered +turn therefore cannot be explained by this bound, and the only branch that +sends nothing at all is a client that already disconnected. + +The bound is the whole turn's stdout, not one frame. A live single-tool-call +brokered turn already emitted 26,486 bytes, 23,320 of them one tool-result +frame (`.runtime/grok-p1b/worker-a2-output.jsonl`), against a `--max-turns` of +48 and a 16 KiB tool-result spill bound, so 64 KiB is reachable by an ordinary +working turn rather than only by a runaway one. + The result frame's last word is `diagnostic_length`, not padding: on `DBL_STATUS_WORKER_FAILED` the supervisor keeps the last `DBL_MAX_DIAGNOSTIC` bytes of the worker's merged stdout/stderr and sends them after the fixed From 734056b1226c42ff08eb3a784750bcbbe7dc2a5b Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 02:58:21 +0200 Subject: [PATCH 104/124] fix: fail a brokered worker's model request fast instead of retrying a refusal blindly --- src/contracts/runtimeContractManifest.ts | 6 +++--- src/runtime/AGENTS.md | 16 ++++++++++++++++ .../receipt.extra-canary.json | 2 +- .../grok-slot-preflight/receipt.legacy-v1.json | 2 +- .../receipt.missing-canary.json | 2 +- .../receipt.readable-canary.json | 2 +- .../receipt.unknown-member.json | 2 +- .../grok-slot-preflight/receipt.valid.v2.json | 2 +- src/runtime/grokBrokerWorkerConfig.test.ts | 15 +++++++++++++++ src/runtime/grokBrokerWorkerConfig.ts | 17 ++++++++++++++++- 10 files changed, 56 insertions(+), 10 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 3cf1242..03af646 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -53,9 +53,9 @@ export const GROK_ENGINE_BROKER = { systemPromptSha256: "2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8", // sha256 of `renderGrokBrokerWorkerConfig({ model, reasoningEffort })`, the only accepted config.toml bytes. configSha256: { - "grok-4.6": { low: "eed6a451150a72b2cb528b30c23b3d51c7d3bc38c67a8985d4dcdf956ff214d3", medium: "8850502dbebf8918c5161c63efcc4ccf18719488300f4cec1deceb2c112b451f", high: "3ce44ace503362326b47149b528b942ce638fe146313d62502f248acf9c7333d" }, - "grok-4.5": { low: "7aa13e90b9bc08d1a018f48b7a84de1dab41db586627ee2d5a25f69011ba7e25", medium: "218ba37e57a6f02fa36b265b4e154e68e30bd2d4794feb130cc226fdda7732a9", high: "0bb4ad8bfa5062169b28422d1d534b45420d4e46b1e546bda1c578eb34303646" }, - "grok-build": { low: "83ac7202442286a65c359cc596b0b8db7bc4529ee70e98224f6cd6f66deb6878", medium: "0146313f28739888eb4e861f1bfb285f7ee4e0a9164669256ebdf6492a2790ce", high: "bbe72aaf70c417dc7007823a7e9e1a7d1fa8d57e50bde6f24036083b32bcc859" } + "grok-4.6": { low: "ab58499ac32678097c146479896f2b8a8e2b0e39aea22dc0a60b6227e370538e", medium: "df1a5cc84346e7f6bf6090492fbd19faaefb42953e3bb2e8c6cbc0572242403f", high: "65b0212564fb74042b1503d293fb8d3620276033264c0efade2a539ca09218e3" }, + "grok-4.5": { low: "8247127c3625ff7c5d8d527a53596b89ec6557a821ac46cfd00bd122b90daff6", medium: "59288cee61297bb8c002097061a48f77b09d310754187a253ee089f7172a9155", high: "c63c3387ce92d94ec3f690abfe98942afcd7c9e17ff84816bbe751f340ab251f" }, + "grok-build": { low: "fb343f2809903f26d21681470943235031f946e99085542fd89555eb7782cbb5", medium: "8a587ef75c90eab70d19b24583e60051d6fba9d90c558839fdbb15588b4cc656", high: "a23724e00d670caee185ba7690d2daa868173e53905cf446f5329666f01ab4e3" } }, // Worker `GROK_HOME` layout the broker attests before every turn. The home and // its `sessions/` directory are root-owned, worker-group writable and sticky so diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 2f33635..682556b 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -173,6 +173,22 @@ must be run with `--model daimon-inference-grok`. The inference ledger directory must be provisioned setgid to the organization group (e.g. `2100:2000 2750`) for uid 2000 to read rows the broker creates `0640`. +Every model block the worker can reach carries `max_retries = 0`. Grok 1.0.34's +default retries a refused or failed request with backoff **past 45 s**, blindly: +one live turn emitted the same refusal fifteen times over five minutes, spent +$0 and died with no account of why. The session-title sink and the evaluator +client (`grokInferenceClientConfig.ts`) always pinned it; the worker's own +model — the single path that spends money — was left on the default, so the one +place a stall costs a wake was the only one that could idle for minutes after +its work was done, silently, because a retried request that never reaches +upstream writes no ledger row and prints no proxy line. Daimon owns the retry +decision here because the thing being retried is Daimon's own proxy: a +genuinely transient fault is already answered 503 and is the broker's to +retry, and everything else is a refusal that repeating cannot fix. The worker +fails fast instead and the turn reaches the host with a status. These bytes are +manifest-pinned per model and effort, so changing them rotates +`GROK_ENGINE_BROKER.worker.configSha256` and every deployment must re-vendor. + `grokBrokerProjection.ts` is the public, I/O-free projection of one brokered Grok agent's slot (`noopolis.daimon.grok-broker-projection.v1`): Daimon's own deny collectors plus the caller's evaluator paths, profile/config/prompt diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json index a1f00ca..b28307c 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json @@ -4,7 +4,7 @@ "worker_uid": 2200, "generation": 3, "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", - "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", + "projection_sha256": "0614586f310b6d4fc50eba85d5a493e03a01d91af8c629989b4b1006dcc3f3a5", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", "sandbox_runtime": "bubblewrap", diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.legacy-v1.json b/src/runtime/fixtures/grok-slot-preflight/receipt.legacy-v1.json index 3ccf85c..fd1e2de 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.legacy-v1.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.legacy-v1.json @@ -2,7 +2,7 @@ "version": "noopolis.daimon.grok-slot-preflight.v1", "slot": 0, "worker_uid": 2200, - "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", + "projection_sha256": "0614586f310b6d4fc50eba85d5a493e03a01d91af8c629989b4b1006dcc3f3a5", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", "sandbox_runtime": "bubblewrap", diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json index 0b57369..03a3c9d 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json @@ -4,7 +4,7 @@ "worker_uid": 2200, "generation": 3, "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", - "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", + "projection_sha256": "0614586f310b6d4fc50eba85d5a493e03a01d91af8c629989b4b1006dcc3f3a5", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", "sandbox_runtime": "bubblewrap", diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json index 19cb96f..c2f4908 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json @@ -4,7 +4,7 @@ "worker_uid": 2200, "generation": 3, "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", - "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", + "projection_sha256": "0614586f310b6d4fc50eba85d5a493e03a01d91af8c629989b4b1006dcc3f3a5", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", "sandbox_runtime": "bubblewrap", diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json b/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json index 91774b2..d5978c9 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json @@ -4,7 +4,7 @@ "worker_uid": 2200, "generation": 3, "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", - "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", + "projection_sha256": "0614586f310b6d4fc50eba85d5a493e03a01d91af8c629989b4b1006dcc3f3a5", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", "sandbox_runtime": "bubblewrap", diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v2.json b/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v2.json index 94542eb..bf3e344 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v2.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v2.json @@ -4,7 +4,7 @@ "worker_uid": 2200, "generation": 3, "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", - "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", + "projection_sha256": "0614586f310b6d4fc50eba85d5a493e03a01d91af8c629989b4b1006dcc3f3a5", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", "sandbox_runtime": "bubblewrap", diff --git a/src/runtime/grokBrokerWorkerConfig.test.ts b/src/runtime/grokBrokerWorkerConfig.test.ts index 091418a..0642b89 100644 --- a/src/runtime/grokBrokerWorkerConfig.test.ts +++ b/src/runtime/grokBrokerWorkerConfig.test.ts @@ -35,6 +35,21 @@ test("worker config disables every bundled 1.0.34 skill, workflows, and the per- assert.match(section(config, "[cli]"), /auto_update = false\nuse_leader = false/u); }); +test("every model the worker can reach fails fast rather than retrying a refusal blindly", () => { + // Grok 1.0.34's default retries a refused request with backoff past 45 s. + // The sink and the evaluator client always pinned this; the worker's own + // model — the one path that spends money — did not, so a refusal there could + // stall a turn for minutes after its work was done with nothing logged. + for (const model of GROK_BROKER_MODELS) { + for (const reasoningEffort of GROK_BROKER_REASONING_EFFORTS) { + const config = renderGrokBrokerWorkerConfig({ model, reasoningEffort }); + for (const block of ["[model.daimon-broker-grok]", "[model.daimon-session-title-disabled]"]) { + assert.match(section(config, block), /\nmax_retries = 0\n/u, `${block} ${model}/${reasoningEffort}`); + } + } + } +}); + test("the declared model and effort reach the worker's only model as its sole allowed effort", () => { const config = renderGrokBrokerWorkerConfig({ model: "grok-build", reasoningEffort: "medium" }); assert.match(section(config, "[model.daimon-broker-grok]"), /\nmodel = "grok-build"\n/u); diff --git a/src/runtime/grokBrokerWorkerConfig.ts b/src/runtime/grokBrokerWorkerConfig.ts index 5a2f88c..fde8952 100644 --- a/src/runtime/grokBrokerWorkerConfig.ts +++ b/src/runtime/grokBrokerWorkerConfig.ts @@ -79,6 +79,21 @@ export const renderGrokLeanBaseConfig = (): string => [ "[workflows]", "enabled = false", "" ].join("\n"); +/** + * `max_retries = 0` on the worker's own model, for the same reason its two + * siblings already carry it (the session-title sink above, + * `grokInferenceClientConfig.ts` for the evaluator): with Grok 1.0.34's + * default, a refused or failed request is retried with backoff **past 45 s** + * instead of failing in ~0.35 s, and the retries are blind — one live turn + * emitted the same refusal fifteen times over five minutes, spent $0, and died + * with no account of why. Only this model block was left on the default, so + * the one request path that spends money was also the only one that could + * stall a turn for minutes after its work was done. Daimon owns the retry + * decision here because the proxy is the thing being retried: a genuinely + * transient fault is already answered 503 and is the broker's to retry, and + * anything else is a refusal that repeating cannot fix. The worker instead + * fails fast and the turn reaches the host with a status. + */ /** * The only source of broker worker `config.toml` bytes. * @@ -107,7 +122,7 @@ export function renderGrokBrokerWorkerConfigWith(policy: GrokBrokerModelPolicy, "[models]", `default = "${GROK_BROKER_WORKER_MODEL_ID}"`, `default_reasoning_effort = "${declared.reasoningEffort}"`, `session_summary = "${GROK_SESSION_TITLE_SINK_MODEL_ID}"`, "", ...renderSessionTitleSink(proxyPort), `[model.${GROK_BROKER_WORKER_MODEL_ID}]`, `model = "${declared.model}"`, `base_url = "http://127.0.0.1:${proxyPort}/v1"`, `env_key = "${GROK_BROKER_PROVIDER_CAPABILITY_ENV}"`, - 'api_backend = "chat_completions"', "context_window = 131072", "supports_backend_search = false", "", + 'api_backend = "chat_completions"', "context_window = 131072", "supports_backend_search = false", "max_retries = 0", "", `[[model.${GROK_BROKER_WORKER_MODEL_ID}.reasoning_efforts]]`, `value = "${declared.reasoningEffort}"`, `label = "${label}"`, "default = true", "", "[mcp_servers.daimon]", `url = "${mcpUrl}"`, 'bearer_token_env_var = "DAIMON_MCP_CAPABILITY"', "" ].join("\n"); From d7e36db3323bbbf4838de9c3d97885e62b1f6ab2 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 02:58:30 +0200 Subject: [PATCH 105/124] fix: trip the launcher's total-output bound from the post-exit drain too --- src/contracts/runtimeContractManifest.ts | 6 ++--- src/runtime/native/AGENTS.md | 25 +++++++++++++++++- .../artifacts/daimon-engine-broker-arm64 | Bin 67456 -> 67456 bytes ...daimon-engine-broker-arm64.provenance.json | 2 +- .../native/artifacts/daimon-engine-broker-x64 | Bin 55392 -> 55392 bytes .../daimon-engine-broker-x64.provenance.json | 2 +- .../native/engineBrokerLauncherServer.inc | 25 +++++++++++++++--- 7 files changed, 50 insertions(+), 10 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 03af646..3df8e89 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -133,9 +133,9 @@ export const GROK_ENGINE_BROKER = { projectionVersion: "noopolis.daimon.grok-broker-projection.v1", slotPreflightVersion: "noopolis.daimon.grok-slot-preflight.v2", artifacts: { - sourceSha256: "2d36898f02793a89a4a58601fa77c0d8716cc028588daad6daf76ca44d28446f", - x64Sha256: "5f92d83fe8159d1ac371f62d5382c0b52ce7fd167e3855460de66187a4c505a8", - arm64Sha256: "8bb4f6b1053abe27001219ab640601289b39c8ff8c86831cd7d8782f0d9dab8c" + sourceSha256: "d8c9640a2d0084f584721d0d4afdc524af7434e9461c1fc2f8adcdff6454ba6d", + x64Sha256: "4059ec576065130e857cc937b03b97a0c45fffab7d790fb153fcb170bfd0f310", + arm64Sha256: "eb0e2975cf3204bb80d25edc1fec00dbd623a95466176f3f36e7e83178deb8d2" } } as const; export const AGY_SUBSCRIPTION_REALM = { diff --git a/src/runtime/native/AGENTS.md b/src/runtime/native/AGENTS.md index 736fec4..86d0c78 100644 --- a/src/runtime/native/AGENTS.md +++ b/src/runtime/native/AGENTS.md @@ -81,11 +81,34 @@ measured. A lost terminal frame, an unnamed transport failure or an unmetered turn therefore cannot be explained by this bound, and the only branch that sends nothing at all is a client that already disconnected. +**Both readers of that buffer trip the same bound**, through +`output_limit_crossed`. The poll loop always did; the post-exit drain did not, +so a worker that exited with more than `DBL_MAX_OUTPUT` still in the pipe left +`used` at the buffer's last byte with `output_limited` clear, and the turn was +published `DBL_STATUS_OK` with `output_length = DBL_MAX_OUTPUT + 1` — which +`closed_result` refuses, so the client replaced it with a fabricated +`prelaunch_failed`/`protocol` frame carrying no pid and no start ticks. That +frame says the worker never ran, about a turn that ran and whose work may have +succeeded, which is the one class of lie this boundary must never tell. +The window is real but narrow: `poll` is level-triggered, so the loop sees any +buffered byte, and the drain can only inherit data written in the gap between +`poll()` returning and `waitpid()` reaping. It is therefore **not +reproducible on demand in this suite** — the fix is by construction, and the +adversarial cases that do cross the bound (`output_boundary_case`, +`worker_flood_case`, `worker_spill_case`) only prove it did not regress. Do not +add a test that claims to cover it by feeding the bound through the poll loop: +that routes around the defect. + The bound is the whole turn's stdout, not one frame. A live single-tool-call brokered turn already emitted 26,486 bytes, 23,320 of them one tool-result frame (`.runtime/grok-p1b/worker-a2-output.jsonl`), against a `--max-turns` of 48 and a 16 KiB tool-result spill bound, so 64 KiB is reachable by an ordinary -working turn rather than only by a runaway one. +working turn rather than only by a runaway one. **Known limit, deliberately not +raised yet:** the same day changed the pipe's blocking mode, and raising the +bound alongside it would mix two variables in the next live run; the measured +ceiling is 26 KB against 64 KiB, so it is not the thing in the way. Revisit +once a trial scores, and decide the number on the spread of real turns rather +than on headroom-by-guess. The result frame's last word is `diagnostic_length`, not padding: on `DBL_STATUS_WORKER_FAILED` the supervisor keeps the last `DBL_MAX_DIAGNOSTIC` diff --git a/src/runtime/native/artifacts/daimon-engine-broker-arm64 b/src/runtime/native/artifacts/daimon-engine-broker-arm64 index b339cf2bf459884bb0824c6dc096754e5fb6642d..2dbe88d27e03d69ea06aa08648efa8f8bc54e551 100755 GIT binary patch delta 3240 zcmYLM4^)&@7Ju*i2ABc)=kQl#X9h%K(9m?`@5sj>f{>fxxY@dA8=_4&a+6RLF&Wa` za&b3a?-a-FxQ%UYJCmM82g_+|gEH%^wjo+4+iC~2%#)p!qbLOU_V)&?@0>focmI6% z-FNSO_dDZ$M(2J;*V%8_u!Dr24eb2e@t)07%n#`I(j&8I0be)!+xamps2Y7XA-YI7 zS!^MsU-Fh<3|j!13$hRoxy2mIo6`4C1D{N1G}iyjPtsHx%U`vu$FDKt5gO;;nUO)E zKae>_XYv>3bWs~$l+{NE{Uce85vloXAzY5C#5(kp78q0$d}cyAi6@`8&2ORw{%iAp zLzOs(zdp~eGZZiceU*?xZd&}mT-BFM+_hs3I`5dOSci6zyT+Tsyx(UkY}}}@;8qd{ z!ks!Mx@Pl2TXw{lIx{Dz&f@EBIn=|M&1kellKc@xwSGK<*rcyPHI2V+t6rN(oHyX^ z9mRO6T`_h9)l};uxG@njd^mwPn-tO^&D$K>@a0>=DRd+@i0EkwdEE4F_(YwaI7vy> zS&~cwb+8LYk)t$%1PEWVB->z$B>`hA39RA2U2>28jZx7R%qwg0C}c?{Y;W%;kK-W5 zQx}>@U<@{LZx{A8_VPbnXwm|5Z!eF}H?|RY7P+)V?i_qf3s|VTb|v1}FGTW)s#?c^ z`S5Jkk$5t@jKfpoNxDk3Ci>*&XZYd5w49}QbxqydC9py!Q6 zX^9KrUAv3ehGJyS6>AVQhaV_18lvQJ77`odZxtn{o4~mi5SsWA+7v!q zl$`n{{L2)%@<#Jz^hAyba}nFjEyc!k*$?B;?o!jR$1JO&xNG~|?%E5Fk-%wwFD%}N z#WGmP85CTv1#VkhY#@PN-d=3F_fuGCIivYO_^89)=fL;gA^BO>cJTdDAL3K2GESN6 z-fXE)v6g}VKfskxVuX1vy+r z%;GL$*(5y|BGEq>U3kL^8NJb4jCy)Ez=Jjfd` zi1t&cd!YGyv?+KO#*|sr#>`>lSWuummQ?@&Ny~Hms-J(XZRlKe=5m?PzN-J_^Fn#+BHl3`_OXg^H(iP1v zmMiSSE;W>&F!E%r!(-={O4I3oc$oH-p-}FR$n*JgTIq_|>2t`Ym@&Ps!`dL%h*CC3 z__VwoYZdFs)_;1b**!3+5_T<$j9o*F$NcGKnUo&qWsVel9QiuON~&FwdzdHSD|&Pq;6HH&&F=tNcjC zqKcO=3(S*Z@gR1`UG%9qfM-J3Uc~ozs9et|=?vX|NYITi^{6TzZ ze#Uc_KR{#np5?`fr=TM`3|$=lc7r_3&o56){0B5<1sbGJA?>xY7+-QjWqINrXvP|r zdB>Y7yr12|$VW1F^Sa7Kw1z)dc^^H-zpGqN*YPz~m2?l^UzJDg{KKl<@dwV&!tw47IL z%%xxZ8#j)Jx4B*s;tB@GroRiZ0|Ui;RET-F<8Gh^>;~dXLaqQe05!i5dx0*X2e=7% zTxHn@9)NWk5XJ0?U4a4&E((EOSZF9SQ@6e0!lq8qqCC2eHRTSBab;A!Baz&qX+ zq6et;3Na!16GHp~X?yy#5ZyrcheDhK&N(Z@CIsGl4uN3`#(j(f0A0XNVDBeFTm+r~ zjsp9EGjKrnr$Q6}`$vUX6~Q0bVzpoC7Gg2x7b%PQNgQAvsNe|VlQKL~rh%N0l>p^q z201}H_?a!4^AACTm_knie)ocoLnhxOWifI4l|NIt!^@yq)Z_c0(@h6(I zmD5N0Uz?ZEIQ~U*9^K5Nw>HsDynSmj-NyHA%}zWK5aJ=}3C}XnTI4Z-f4sGlKFZU# zRfd%ym*4PTZ!4g`;xBE>peOiy+g8(lp4yU0U*`6f7w92=zNHn`Yqq!2Rs76$W8&U_ z3vqYIBO$ARJCMo`_}KQ`=r#)SDI+|zb**N94r)!^%<{SE_+5WEclKtVoMptSW*yCHKjE}#5e;CbH F{tvGR=!5_O delta 3204 zcmYjU4NzRw6~6bqh20GhS(oKky1O7DEQutB zM-p`$Y~}X0WujBNj5@m8I7T;UwlxYyZ5$Jgo5UuqZ5Kj~9e+wDApr@w{VovOH*@y8 z=jWY!?m6e4H>*Es(4RE)pD-`&A#=}CcJ?icuVbG17Q;dMN+vDkcir^G@+1~hts%D% z10NFISA<|ffDXt-cC!y z7ng6N%7iD}S`;>zOBsTmAf$}ji~m!oxW@L7Tc_I$!EdYzo6c3(OnY2#MpZ*M5xqIq zNg^@0Q>Vp18ZUR}8>iG83u0;p-|8-)I%jUHl`bNslZxsZN+j+yyfdn1^4Hz#yp(t^ z!?l5sly-&m`D?WcpA+JJTd{u7t62MDYPzckZta9zNEE^wL*z7_JlJa@UQ*R~(w9ae zS4^b1MTU_;UTrawh>ve6$v69NAd&WN5^?j#OE%>FX-W*liafgl3Mt8j?RDSsShd9Z z!MS!4`G(Jyr1{^!B)q~vZkSY6mk*YuARoMnLb}U;B>rmbel6mpdh_k@_>2%~(ryOm zFc1s5$-@!p_Fr5P-j*y9>AWJ`pAu5+kGCqGu1A6c_s z^~|=T5_6gBlwFq@s2%(ij5f;2c<2(2>rFgRm>TP=Nqc+)XeX%w&nfhGLE?4u>D?zU@j!E99YRa zU#puj8?r*Zd3y9i9<0tbnwea`%a2!Ya39A7lNV03x+7!oAaB4Z+E1Z+^x<#Nrr>=T zM-Ek+cEpjPm`Yt|_#L-3Sss$sXkghg6wSAmzjRxr^C|cP=t^$?8AvNM!H%64`j2!p`kc=kgOrb~o1s zs<=_hrssH`_L$it_ebP4d`PQVo3?Nc*%T=Y>lUmna*Zfub1Ytv_hOxZ^=$W>LF&+> zvnpX16Ug*M#CSMd?a8Hdn77oXO=VCbP(BkDXy0$;k8?fXv$Q{ z&N*4C>qRN^Lk9kNtVKLoiOhLX1J@v5UT3HK`1-o7igA*^R`;td8F1%Iz^8l(WloH2 z8~E5F@GgZpF%2zJFZo*xb9|E2o55rI-(K@^CkaF^;ii?zxiBYB#f{k;E#`x(>k|^$ z>`cN-cWdcallg_!R{AoJtFNQm_}cm!^P`vG3HQasS$z$i;vd(qst;inIA%rhD0atP z3?<+Ifvdv(dwliA>P(E1{<+&vY2ZPkB{AvwP541sUA`GnicfdwhkWgtyJ;#9uBo)0 zgpL?6baD9W4YG|-ugS6<4g?zI?3s^#_Eqt4IlUK8KFw#Je;FID9* z!xp{CJi7LmbR%zBcekbNqMW~}n7=8Z95m@T|9D*`J@;+e!Ibr03W%pkbWJ0<-VDCf8YfnzQoWt@Ujp+ z7$&|~g;<9BJq*<3JqO||LY9SvxEH7cp8y7c0pJ1PA*nwogq#b*z!cp3jYC3I0gnMY zfWFs+cn)~t4I$FytN^Z1iJ#naScuIK90mRic<)<690cmaLR;EK~Y08j_^17n{EF%BFDP5~!?i8x^3j1XnO6H`KLH1g06 zSM{=2geb=BA|B&+k^##=U68GiGCWeIfqo5H6;M77kQej@x9rSa{v|YsY3@nE?-QUY z$mE;EvpD&8$}?2%@EmA@x9lucieKeV?aZOi@xwdwmBcXrY-b7To!wbP5AyY$?eqX2 z=}e=C_}R{Uo8gZ_+#@~VSq<8ZJbJmUtAXz4Kj>p?D$@?cjPy@!u=<2|U%mz>T$ z5tr=3pXMSxG(9~jf|SduvyYagCsmb;f*`%`lCv8`sb(z>&ixJXQeB5DvUkuW=L38n zphEhT$d9F(2nz)IR9}T`gHP`02R-Y1#*~?z6(eebD_+vQ|6|OP7GALdzEik}Z>1dH z7Ks;q1>&pelWInW2|{i)6A-gh_3TibG%}#H@EqLaC%Dqcm0Z%LkrTdjS@ln!tS0}c zzjl%_wI(R7{|F(d;s>aVT78AvSpK={v#zNWpnQHVxtdG*b4hl&52=&go)cA4&HV^N zdaqL<*>UR5A&7+kDP7s&@SG=bVU#M*AAXnWZmgL`vl7scKLo22W2F(o7Log)8Kqc;2ArV9B#mP7x~N z)twOtfd6re$k?dYsj0jIH4_k*;#n0ZEE}eHs#e7dl9#s}L3A$vtpaa1l%+NEd652c zxj-Yov}MAaGpPE9A6m-G-LIcN&e&Z4ivolUCNQnE2LeqH&@k(B{NZsf@~CW6G0t|C zbU1^05K1net4g+MzBpMSUCPX$^vr44-o<_LVYO?e`?Tca8tP`tCHvGvy?OrhNDU1Z zKha<*UQPZ`6_2T6w@RX9JrQM9eNnQ8+QQRM8ipxe^@k60kzZ8OHP4@JJNstuvsW0? z$OX)si}blk)%jeq_lBu!KoX&;>wSl3?Xxy4GpiCFDr0XJOgMwNUD@S)B(IGh1~nX0 z7KMSB)VvxDjST7uKcO@=O!5{&hjTP?Q6CUYQvE1wa?+ zH@c|5L-#1Mcstb_1Sdl-6J0FDmMf-LeNE6^RH}7mJ9=t2*E{u{ta#ie&!p) zu;V%d!wdXN6So;I45mW%0`K^S{`2nf<+qv{t$h%;5^>qvu*L*EE`+K;+xUPX5}m;k z!d`nD2YEUOL;o=Zj^!Y~(HuMy0wYbMwl*s_+D>P=#*B&y@dkjkbJL3Has{n$O`Y~3 zxQb`5=$?6X*>TlxEnSRa>_025vNpVc(e>|;OW?Uom7d%O?m~LVHB$UD?R6zgehhp- z4?%&manG$4c*o>i;*eCno5MQbztU89Tve(mza%7o0gT0_;GS#dpTJp4_R2v-$+3ex z4gq?(elJ!#j1#Y%ig=*wf^2Se=SCi#RNl!sX?WKN&tgW%&_U>M8=ZARzM-6HDyN!` zKj2!b35^e_^n_Git~1>gfb9&z#aY+acGw zfga$$5OD2lg3E4FeO?1pJ!t~I{1DTp(99h54{t)1-w)hRJ&o4oOi9ce;P=_I+{_0+ z-}$9!D3^5V>*(>E<>O8-0^D?z?>T(%RnluL_I{vk4qOPmj#jEgaqr^nJS>fH$u{1e zqp0+D+M|vUU#C}9_vnd_nI(AmIkP9c1gp8fTDmZI%h07?m}%;d(TKd+*LSX=(lqm(N&L|jnjDXx6@(T zMAv9+NYY*Nb){9GcEs?STElQi!a?o0(^% zp7tkkF}-Q-TVfsEIX6M9q(|nCyZIh-3wcNLPldllzJ#0ER?VYNr!n~>#Wjge0`K=p09HGh`4m42U}EoRaw^Gb@_ z&1K>BT;^u*V~&r~orld$`J@3_!(bIldnm3-hW_w%4=`*0FES%te6uJJ#^;H2>5J)! zdFA3PIuT%~?x#Hk zia3WR&Cea<#NzJVebJDy>t{d3>1CXFVZZ;BuAQGFw$g+1mxviObHO~zl~3r#1ycb9 z7d$og>qi8kE2lgGUZt2JWzrJsmq4VL9J`b4DWg(hQtW?B5yy3rUDR8+L>v`3SC}bU zjvffyQ9O5uI6vThU~}GW@1_YtO|T*w-;u*G5&Dg$6r)a(gymRNT&Q|Z+q$gP5JWEo zeG@I-Izv>cf9sl29W(gF?bo0B7h||Rr?yLpnb?w3w$YS^jpCL-TZ3iD_NM8!Dm!89 z8;ocsWh{kn4yuDXKre$1Y=Iqo zzJz+lGC>`n{BxEHx`?0OV9X0z3%Y5DP$dlDr4c~M-x&*n20=dqO??xFLF+*;gF4<~ z%#N=0>|*RbP)93c<^22+V@;f5pPU1=_cKBd@c;$|uynHE{U%~Y%p%setN?Ekut=%N4s delta 3867 zcmZ8j4OCQB9)EY3iNRRjxPW4!4DR?rSvp58+yLj%3_j=ysBliTZcDmLQKQl>D0SQh zc4j=jr7`WMnPqKd>298l8czuaVIUISEYYHMD+OD(-sA_UMX2fB-+gaL>zy<6fA{zQ z{^NieBNk;k@b!T&Pm&oMIqDYeu8rB4j5fNi-EIg9UfsZt8;ih)0JtT`UO_)mtkz1wAu@x}ST&?z8Og<6fs9If}CwNi}tTv%K83B$7v?n)f zvj0mxKQZw7RmSwXjU+0y@^n$fz!o|&Z>*v(H7DftI@aEFtr9s%C>r1fGU~bIX0E9> zJC-$?na(r+6RL?M(`iFqif;}S^dOx>l6p`=@pzKC@@he-M39>|TBk5^&lB(tF>d3h&!>DCr-mm#mCR8cRsxuD>TD4{tgs=4`r!iE7Z zrKsn%%O@BE)kVNjmT7l~%ro4o*J0)3Ym-%9hclw}Ld=cES|xR#G&dst02dyZ3xZ4x)IyWPdPK*y0^rdHfbqt+=y|ApQagXqwqkUmR`85g(zjeyla~%N z7L16&<5olxTI0xW69lWEJpxe}M_F_)xA)Qg`RTqrFv;=ffJ-PxqIyZasx3Wk zp#B(Y)zQQlx*VvMaSCUMGvYkpJOX=>cH}E)567(4K%#vLkeEap&l?dJfe$A+++!JJ zT5hy~(RKscsrl%NyT(+-FV_VRL(r|VmLJ&|BnJ!>9^?4;Kf!Iloxow(evKii-N45g zxK($dGcm5f$uWB7uF2kq!A30W?#GUUx~+{LQCe9Hm4UYLNkb&sycjr(gFGFCq1_w< zCvp&zqJyb1Fu^ctZL@Hrj6UI>FmB-&hQmQ%?cB6vxNN4o-M7sc1x?Ank9g)@tvw-! zEU!L-VjQ?2K4H-|V{?XfJ1XG0QWjV22X`AC?HNm6qgkHxTXujC=p`s{c0b;C;~kT8 zrx%du?c}fq_-$J4N%2)1@^8lEhrxJ*zl+}YM$G>MYcVU}h#*Ri_3$`c&?^iDaBlPc z7jVo(JkSk5wy>dVBacq>c5zNN-W8Q+u{vg`2Rht_d9zTgE4vKkYQypSTuU~fKg3k} zV=A9*GTfDd?To<1J|5X%jN3&Q79>yI0~v5SIL8J#?rN8Tb#lxOOpE)RHWoOn7arE9 z>{I$~K}JF!779B2Puf>-uUvM{7*Xety4x{x4s>4mOh>X?_=wAmyB!-mKofD-Vq~&| zk=kF%jR75${n>&TTn902Ijx^B|KcQ6`ToHE)aq#O{97~rI>`64VY!75fHwRe!%(5x zr5&W1@~X_~Re%SN@>362wyZ{-5j1iJ&^9|Zgw{*<$Ym*KA3_dC#4$L6d3%nc(&wqY zZ~{3&^9nsTJ@}wef|s8*dcsRk&;9k#ZG|t5YWla4rnZ9?6wND$$LR-WIr`Cnin)j7|+i2(M-O5z= zeK_9KzToF|7Hws(ZhB^rVX+hox#O}u##?OgX6w8HEF?c;HTYs;T6QmeN69m#RnZ~k zH{=bvXu)fwmkuvTCwpjWapw5V#wirINx$)09hb2)+ZuTEYv{t_v80Sv6;HIy`9|N0 zfdf}~p)H%~v&9p}H&tGX7ft;ZVdLaU!HKb+9)jvu^y}h0@->}OQb6kI@{)>jX6*KW z{|9acA7gx!9zSB7pDT3Gau=-P&>n@Yk~?kbH4iXb`Y|$7yZIp`L0D0T)Nbt!I;qr4 zo}f>a{(ADV!-lCdh~F3`rXA{mFRpSFAYw0}XG_PVZ;3fwblIqC3+^N1{L((8_JtB@ zq)QhTPIwe~M>{X+GB#}%Ohvb1#S1&7hwfiEpIo7~MJvglY5k%SQ|=*pc+qX-%F~M< zoj86OF5mgybT}19$5)CgE#rWQqvzY4?7d1_vUpPRWn>2t`!$h^v}^H7a&LHcnTwdx zJ`AraUl30=hP#%$STyTwwjcx}HA%Qej=)6h8%!!knMy>Qjb-JFW&dewx1|Gu`uH8C zn|9nm*3l1ltQ|)H;x^PCeUmY4o>Om&87>@>w??UV=SI>OzPi&CPpea1Z#T`h`g+ze z_646(%Gh)qz7bFbbnrRGc7awukCNiRZUS8bItW?~8rjC!C!jr`8fZW051>si!47V) z2&fCRAC%u{gP;%b^~;O}Kucd?Y)ic06Poa{3xNGq#v-8gpr3#af@+`@+hG{AAJm4% z{jHg?dqDeJ8T0b>2aGjwiX-J)&>rmA4fsN+A7HEp)P{4*HcAj&pbpSd&}`5OP-Rpy zGQ>*-fDNEEpiN;<^99o_jl3xNNw%j9n7&Z7nt(4ADFDyS_Xlur&FBy3oCN0{biM34Y#7*f zVAsnfqWc2f*q%IQ$qVRpOxg`P8=U3xxW zvYl>=JWsOe+>Uustatus = DBL_STATUS_OUTPUT_FAILED; + out->stage = DBL_STAGE_OUTPUT; + out->failure_class = DBL_FAILURE_OUTPUT_LIMIT; + return 1; +} static void supervise(int client, pid_t pid, int output, struct dbl_result *out) { unsigned char bytes[DBL_MAX_OUTPUT + 1] = {0}; @@ -212,12 +230,9 @@ static void supervise(int client, pid_t pid, int output, ssize_t got = read(output, bytes + used, sizeof(bytes) - used); if (got > 0) used += (size_t)got; - if (used > DBL_MAX_OUTPUT) { + if (output_limit_crossed(used, out)) { output_limited = 1; p[1].events = 0; - out->status = DBL_STATUS_OUTPUT_FAILED; - out->stage = DBL_STAGE_OUTPUT; - out->failure_class = DBL_FAILURE_OUTPUT_LIMIT; } } if (disconnected || output_limited) @@ -236,6 +251,8 @@ static void supervise(int client, pid_t pid, int output, if (got <= 0) break; used += (size_t)got; + if (output_limit_crossed(used, out)) + output_limited = 1; } if (WIFEXITED(status)) out->exit_code = WEXITSTATUS(status); From 128dd560fc55c70311ea4811a140d575c8a312a0 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 04:19:47 +0200 Subject: [PATCH 106/124] test: take the facade before any mount listens so a contended fixed port fails red instead of parking the runner --- src/runtime/engineBrokerMcpFacade.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/runtime/engineBrokerMcpFacade.test.ts b/src/runtime/engineBrokerMcpFacade.test.ts index 057e798..80bdfc0 100644 --- a/src/runtime/engineBrokerMcpFacade.test.ts +++ b/src/runtime/engineBrokerMcpFacade.test.ts @@ -46,8 +46,9 @@ const startFacade = async (): Promise => { }; test("MCP facade routes only valid active capabilities to the registered mount", async () => { + const facade=await sharedFacade(); let calls=0;const target=createServer((_request,response)=>{calls++;response.writeHead(200,{"content-type":"application/json"});response.end('{"ok":true}');});await new Promise((resolve)=>target.listen(0,"127.0.0.1",resolve));const address=target.address();if(address===null||typeof address==="string")throw new Error(); - const facade=await sharedFacade();const token=facade.register("agent","turn-capabilities",`http://127.0.0.1:${address.port}/mcp`);const call=(value:string)=>fetch(FACADE_URL,{method:"POST",headers:{authorization:`Bearer ${value}`,"content-type":"application/json"},body:"{}"}); + const token=facade.register("agent","turn-capabilities",`http://127.0.0.1:${address.port}/mcp`);const call=(value:string)=>fetch(FACADE_URL,{method:"POST",headers:{authorization:`Bearer ${value}`,"content-type":"application/json"},body:"{}"}); try{assert.equal((await call("wrong-token-abcdefghijklmnopqrstuvwxyz0123456789")).status,403);assert.equal((await call(token)).status,200);assert.equal(calls,1);facade.revoke("turn-capabilities");assert.equal((await call(token)).status,403);assert.equal(calls,1);}finally{facade.revoke("turn-capabilities");await new Promise((resolve)=>target.close(()=>resolve()));} }); @@ -231,6 +232,9 @@ test("the facade forwards a closed header allowlist and never the worker's beare }); test("the facade withholds a mount response header that is not on the allowlist", async () => { + // The facade comes first: nothing must be listening while the fixed port is + // still in doubt, or a refused start leaks this mount and parks the runner. + const facade = await sharedFacade(); const target = createServer((_request, response) => { response.writeHead(200, { "content-type": "application/json", @@ -244,7 +248,6 @@ test("the facade withholds a mount response header that is not on the allowlist" await new Promise((resolve) => target.listen(0, "127.0.0.1", resolve)); const address = target.address(); if (address === null || typeof address === "string") throw new Error("target address unavailable"); - const facade = await sharedFacade(); const capability = facade.register("alpha", "turn-response-headers", `http://127.0.0.1:${address.port}/mcp`); try { const answered = await fetch(FACADE_URL, { From 2ef45890c50df36925719a80545a2e2a4a260f06 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 04:20:49 +0200 Subject: [PATCH 107/124] fix: end a per-wake MCP mount's leftover connections so a finished turn cannot park on its own teardown --- src/pi/AGENTS.md | 10 ++++ src/pi/cliSession.ts | 12 ++++- src/pi/cliSessionMcpMountClose.test.ts | 64 ++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 src/pi/cliSessionMcpMountClose.test.ts diff --git a/src/pi/AGENTS.md b/src/pi/AGENTS.md index eb26811..4685c74 100644 --- a/src/pi/AGENTS.md +++ b/src/pi/AGENTS.md @@ -9,3 +9,13 @@ removing redundant nested denies that cannot be mounted by its Linux sandbox. An intervening readable path or workspace root makes a deeper deny necessary. Verify changes with generated production arguments and real local sandbox commands; a model call is neither required nor permitted for this check. + +The per-wake MCP mount is torn down on the wake's own completion path, so that +teardown must be bounded. `Server.close()` waits for every open connection, and +a connection the MCP transport has no record of — a socket opened before +`initialize`, or an idle keep-alive socket a client's pool still holds, which is +what relaying a turn through the broker MCP facade leaves behind — is not the +transport's to end. Close the transport first, then end the remaining +connections; never wait for the client to release them. A finished turn that +parks here publishes nothing and dies to an outer deadline, which loses exactly +the terminal evidence the turn existed to produce. diff --git a/src/pi/cliSession.ts b/src/pi/cliSession.ts index ec4de9e..f161009 100644 --- a/src/pi/cliSession.ts +++ b/src/pi/cliSession.ts @@ -189,7 +189,17 @@ const startMcp = async ( await startupSettled; await transport.close().catch(() => undefined); await mcpServer.close().catch(() => undefined); - if (httpServer.listening) await new Promise((resolve) => httpServer.close(() => resolve())); + // `close` only stops accepting and then waits for every open connection, + // including the ones the transport has no record of and so cannot end (a + // socket opened before `initialize`, or a client pool's idle keep-alive + // socket, which relaying a turn through the broker MCP facade leaves + // behind). That wait is unbounded and sits on the wake's own completion + // path: measured, one such connection parked a finished broker turn with + // its result in hand and published nothing. By here this one wake's engine + // has returned, failed or been cancelled, so anything still connected is a + // leftover — see `AGENTS.md`, and the facade, which bounds itself the same + // way. + if (httpServer.listening) await new Promise((resolve) => { httpServer.close(() => resolve()); httpServer.closeAllConnections(); }); lifecycle = "closed"; })(); const mount = { get endpoint(): string { return endpoint; }, close }; diff --git a/src/pi/cliSessionMcpMountClose.test.ts b/src/pi/cliSessionMcpMountClose.test.ts new file mode 100644 index 0000000..69b74ed --- /dev/null +++ b/src/pi/cliSessionMcpMountClose.test.ts @@ -0,0 +1,64 @@ +import assert from "node:assert/strict"; +import { connect, type Socket } from "node:net"; +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { setTimeout as delay } from "node:timers/promises"; + +import { createCliSessionFactory } from "./cliSession.js"; + +type PublishedTurn = { readonly message: { readonly content: readonly unknown[] } }; + +/** + * A finished turn must not wait on a connection to its own per-wake MCP mount. + * + * The mount's HTTP server is torn down on the wake's completion path, and + * `Server.close()` only stops accepting: it waits for every open connection. + * The MCP transport ends the sessions it knows about, but a connection it has + * no record of — a socket opened before `initialize`, or an idle HTTP + * keep-alive socket a client's connection pool is still holding — is not its + * to end. The broker MCP facade is exactly such a client: relaying a turn's + * session leaves pooled connections to the mount behind it. + * + * Measured before this was bounded: the broker turn returned its result and + * the wake then never completed and never published — the terminal evidence + * that is the whole point of letting a finished turn finish. + */ +const TURN_COMPLETION_BOUND_MS = 5_000; + +test("a finished Grok broker turn publishes without waiting on an open connection to its own MCP mount", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-mount-close-")); + const peers: Socket[] = []; + const hangUp = (): void => { for (const peer of peers) peer.destroy(); }; + try { + const { session } = await createCliSessionFactory({ + engine: "grok", + command: "/nonexistent-engine", + grokBrokerTurn: async (_prompt, mcpEndpoint) => { + // A peer holding a connection the transport never saw an `initialize` + // on: none of this connection is the mount's own session state. + const peer = connect({ host: "127.0.0.1", port: Number(new URL(mcpEndpoint).port) }); + peers.push(peer); + peer.on("error", () => undefined); + await new Promise((resolve, reject) => { peer.once("connect", resolve); peer.once("error", reject); }); + return "Filed and delivered."; + } + })({ cwd: root }); + const published: PublishedTurn[] = []; + session.subscribe((event) => { published.push(event as unknown as PublishedTurn); }); + try { + const settled = session.prompt("wake").then(() => "completed" as const); + const outcome = await Promise.race([settled, delay(TURN_COMPLETION_BOUND_MS).then(() => "parked" as const)]); + // Hang the peer up before asserting, so a regression reports the parked + // wake instead of parking the suite's own teardown behind it. + hangUp(); + assert.equal(outcome, "completed", `the finished wake did not complete within ${TURN_COMPLETION_BOUND_MS}ms`); + assert.equal(published.length, 1); + assert.deepEqual(published[0]?.message.content, [{ type: "text", text: "Filed and delivered." }]); + } finally { hangUp(); await session.disposeAsync?.(); } + } finally { + hangUp(); + await rm(root, { recursive: true, force: true }); + } +}); From 25f84ef02c65c2f8b327626a2082542e3705eb17 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 04:24:06 +0200 Subject: [PATCH 108/124] fix: end the broker provider proxy's leftover sockets on shutdown instead of waiting for a worker to release them --- src/runtime/grokBrokerProxy.test.ts | 17 +++++++++++++++++ src/runtime/grokBrokerProxy.ts | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index 526bd78..93d4a9f 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -1,9 +1,12 @@ import assert from "node:assert/strict"; import { request as httpRequest } from "node:http"; +import { connect } from "node:net"; import test from "node:test"; +import { setTimeout as delay } from "node:timers/promises"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; import { GROK_SESSION_TITLE_SINK_KEY } from "./grokBrokerWorkerConfig.js"; import { estimateGrokRequestUsage, GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; +import { DEFAULT_GROK_BROKER_MODEL_POLICY } from "./grokBrokerModelPolicy.js"; import { ENGINE_BROKER_AUTH_STALE } from "./engineBrokerProtocol.js"; import { GROK_INFERENCE_AUTH_STALE_BODY } from "./grokInferenceProxy.js"; @@ -227,3 +230,17 @@ test("a response whose usage cannot be decoded is still delivered, and charged t assert.equal(snapshot.timings[0]?.toolCalls, undefined, "an undecodable response records no tool-call attempt either"); } finally { await proxy.close(); } }); + +test("proxy shutdown ends a worker's leftover keep-alive socket instead of waiting for it", async () => { + const proxy = await startGrokBrokerProxy({ accessToken: async () => "token", markRejected: async () => undefined }, async () => ({ status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from("data: done\n\n") }), DEFAULT_GROK_BROKER_MODEL_POLICY, 0); + // A worker's HTTP client keeps its connection to the proxy pooled; nothing + // in the proxy's own state accounts for it, so a shutdown that waits for the + // client to release it has no bound. + const socket = connect({ host: "127.0.0.1", port: proxy.port }); + socket.on("error", () => undefined); + await new Promise((resolve, reject) => { socket.once("connect", resolve); socket.once("error", reject); }); + try { + const outcome = await Promise.race([proxy.close().then(() => "closed" as const), delay(5_000).then(() => "parked" as const)]); + assert.equal(outcome, "closed", "proxy shutdown parked on a socket the proxy does not track"); + } finally { socket.destroy(); } +}); diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index 9857cff..7ee024e 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -35,7 +35,7 @@ export async function startGrokBrokerProxy(authority: GrokBrokerCredentialAuthor const guards=new MapPromise>();const turns=new Map();const server = createServer((request, response) => { void serve(request, response, authority, upstream, capabilities,guards,turns,declared,grants); }); await new Promise((resolve, reject) => { server.once("error", reject); server.listen(listenPort, "127.0.0.1", () => { server.off("error", reject); resolve(); }); }); const address = server.address() as AddressInfo; - return { port: address.port, capabilities,registerIsolationGuard(turnId,guard){guards.set(turnId,guard);},revokeIsolationGuard(turnId){guards.delete(turnId);},registerTurn(turnId,turn){turns.set(turnId,{policy:parseGrokBrokerModelPolicy(turn.policy),meter:turn.meter});},revokeTurn(turnId){turns.delete(turnId);}, close: () => new Promise((resolve, reject) => server.close((error) => error === undefined ? resolve() : reject(error))) }; + return { port: address.port, capabilities,registerIsolationGuard(turnId,guard){guards.set(turnId,guard);},revokeIsolationGuard(turnId){guards.delete(turnId);},registerTurn(turnId,turn){turns.set(turnId,{policy:parseGrokBrokerModelPolicy(turn.policy),meter:turn.meter});},revokeTurn(turnId){turns.delete(turnId);}, close: () => new Promise((resolve, reject) => { server.close((error) => error === undefined ? resolve() : reject(error)); /* `close` waits for every open connection, and a worker keeps its client pool's socket to this proxy open with nothing here accounting for it — so that wait has no bound. Ending them is this listener's to do, exactly as the MCP facade does. */ server.closeAllConnections(); }) }; } /** From 6c33505409048bddd57797543cbd448e52658f2a Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 11:15:39 +0200 Subject: [PATCH 109/124] test: split the MCP tunnel drain test out of the facade suite --- src/runtime/engineBrokerMcpTunnel.test.ts | 59 +++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 src/runtime/engineBrokerMcpTunnel.test.ts diff --git a/src/runtime/engineBrokerMcpTunnel.test.ts b/src/runtime/engineBrokerMcpTunnel.test.ts new file mode 100644 index 0000000..b22bf36 --- /dev/null +++ b/src/runtime/engineBrokerMcpTunnel.test.ts @@ -0,0 +1,59 @@ +import assert from "node:assert/strict"; +import { createServer } from "node:http"; +import { connect, type Socket } from "node:net"; +import test from "node:test"; + +import { awaitMcpTunnelDrain } from "./engineBrokerMcpFacade.js"; + +const withDeadline = async (work: Promise, ms: number, message: string): Promise => { + let timer: NodeJS.Timeout | undefined; + try { return await Promise.race([work, new Promise((_resolve, reject) => { timer = setTimeout(() => reject(new Error(message)), ms); })]); } + finally { if (timer) clearTimeout(timer); } +}; + +/** + * The relay parks on this await whenever a tunnel is backpressured, and a + * parked await is invisible from outside: no status, no refusal, no line. So + * the assertion is that it settles at all — on a client that hung up + * mid-write, on the turn's abort, and on a genuine drain — with a deadline + * standing in for the hang. + */ +test("a backpressured MCP tunnel always settles: on a hang-up, on the turn's abort, and on a real drain", async () => { + const parked = new Map>(); + // One controller per phase: the turn whose abort is under test must not be + // the turn that is still relaying. + const controllers = new Map(); + const server = createServer((request, response) => { + const phase = request.url ?? ""; + const controller = new AbortController(); controllers.set(phase, controller); + response.writeHead(200, { "content-type": "text/event-stream" }); + // A paused client cannot absorb this, so `write` reports backpressure and + // the relay would park exactly here. + response.write("data: open\n\n"); + if (phase === "/drain") assert.equal(response.write(Buffer.alloc(16 * 1024 * 1024, 0x61)), false, "a paused client must backpressure the tunnel"); + parked.set(phase, awaitMcpTunnelDrain(response, controller.signal).then(() => "drained", (error: Error) => error.message)); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); if (address === null || typeof address === "string") throw new Error(); + const open = (phase: string): Promise => new Promise((resolve) => { + const socket = connect(address.port, "127.0.0.1", () => socket.write(`GET ${phase} HTTP/1.1\r\nhost: facade\r\n\r\n`)); + socket.once("data", () => { socket.pause(); resolve(socket); }); + }); + const settled = (phase: string): Promise => withDeadline(parked.get(phase)!, 3_000, `the ${phase} await never settled: the relay is parked`); + const sockets: Socket[] = []; + try { + sockets.push(await open("/hangup")); + sockets[0]!.destroy(); + assert.equal(await settled("/hangup"), "MCP tunnel closed", "a client that hung up mid-write wakes the await"); + sockets.push(await open("/abort")); + controllers.get("/abort")!.abort(); + assert.equal(await settled("/abort"), "MCP tunnel aborted", "the turn's own abort wakes the await"); + const draining = await open("/drain"); + sockets.push(draining); + draining.resume(); + assert.equal(await settled("/drain"), "drained", "a client that resumes reading resolves the await"); + } finally { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve) => server.close(() => resolve())); + } +}); From c05306eacee8cb51170349478ff86d9ed9e1ff36 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 11:15:42 +0200 Subject: [PATCH 110/124] feat: seal the brokered worker's in-flight MCP tool calls so a hung call is visible as one --- src/runtime/AGENTS.md | 26 ++++ src/runtime/engineBrokerControlClient.ts | 12 +- src/runtime/engineBrokerMcpCallLog.test.ts | 73 +++++++++++ src/runtime/engineBrokerMcpCallLog.ts | 137 +++++++++++++++++++++ src/runtime/engineBrokerMcpFacade.test.ts | 134 ++++++++++++-------- src/runtime/engineBrokerMcpFacade.ts | 33 ++++- src/runtime/engineBrokerProtocol.test.ts | 25 ++++ src/runtime/engineBrokerProtocol.ts | 38 +++++- src/runtime/engineBrokerService.test.ts | 11 ++ src/runtime/engineBrokerService.ts | 2 +- src/runtime/grokEngineBrokerTurn.ts | 16 ++- src/runtime/grokEngineBrokerUsage.test.ts | 38 +++++- 12 files changed, 480 insertions(+), 65 deletions(-) create mode 100644 src/runtime/engineBrokerMcpCallLog.test.ts create mode 100644 src/runtime/engineBrokerMcpCallLog.ts diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 682556b..5a158bb 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -288,6 +288,32 @@ outcome but a real drain rejects, so the relay tears the tunnel down instead of writing into a socket that is gone. Never widen it into a transparent proxy: the whole point of the boundary is that the allowlist is closed. +The facade is also the only place an MCP tool call is observable *while it is +still running*. Daimon writes a tool receipt on completion, so a call that +started and never returned is byte-identical, in every artifact, to a call that +was never made — and that was the last unlit path under a live hang where the +worker stopped acting after its eighth provider response, the per-request +ledger published `open: 0`, and the trial deadline killed it seven minutes +later. `engineBrokerMcpCallLog.ts` records each relayed `tools/call` POST and +whether the facade ever answered it, and the observation rides the *sealed +terminal response* of a failed turn (`mcpCalls`, optional and v2-only) — +the seam the worker's redacted last words and the sealed usage already take, +because the slot's control root is tmpfs that dies with the container. It +replays with the record and reaches the operator through +`engineBrokerControlClient.ts` as `mcp=/ answered` plus +`mcp_outstanding=@ms`. Its rules are the per-request ledger's: names +and timings only (never arguments, never a result, never a session id or +bearer; a name that is not a plain short identifier is ``, and the +list is bounded with a `` last entry); absence stays absence (a turn +the facade never registered observes as *nothing*, a turn that called nothing +observes `started: 0`, and a POST body the facade could not read counts in +`undecoded` rather than inventing a name); and it can never fail, delay or +refuse a turn. "Answered" means one thing and it is load bearing: the relay +reached its own `end()`. A tunnel torn down when the worker dies did not +answer, so the call it was blocked on stays outstanding with the elapsed time +it had reached — otherwise the turn's death would erase the evidence the +instrument exists to keep. + Worker `GROK_HOME` layout the deployment must provision (attested before every turn by `grokWorkerHomeAttestation.ts`, recorded in `GROK_ENGINE_BROKER.worker.home`): `$GROK_HOME` and `$GROK_HOME/sessions` `root: 1771`; `config.toml`, diff --git a/src/runtime/engineBrokerControlClient.ts b/src/runtime/engineBrokerControlClient.ts index a02d758..6744615 100644 --- a/src/runtime/engineBrokerControlClient.ts +++ b/src/runtime/engineBrokerControlClient.ts @@ -2,10 +2,20 @@ import { createHash, randomUUID } from "node:crypto"; import { createConnection } from "node:net"; import { ENGINE_BROKER_VERSION, encodeEngineBrokerFrame,EngineBrokerFrameDecoder,parseEngineBrokerResponse } from "./engineBrokerProtocol.js"; import type { EngineBrokerInferenceFailureCode, EngineBrokerInferenceRequest, EngineBrokerInferenceResponse } from "./engineBrokerInferenceProtocol.js"; +import type { EngineBrokerMcpCallObservation } from "./engineBrokerMcpCallLog.js"; import type { EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; import type { GrokBrokerModel, GrokBrokerReasoningEffort } from "./grokBrokerModelPolicy.js"; import type { GrokInferencePurpose } from "./inferenceUsageLedger.js"; +/** + * What the broker saw of the worker's MCP tool calls on a failed turn + * (`engineBrokerMcpCallLog.ts`). Absent for a turn with no observation at all; + * `outstanding` names every call that started and was never answered, with how + * long it had been waiting — the one thing a completion-only tool receipt can + * never say. + */ +const renderMcpCalls=(calls:EngineBrokerMcpCallObservation|undefined):string=>calls===undefined?"":`; mcp=${calls.answered}/${calls.started} answered${calls.undecoded===0?"":`; mcp_undecoded=${calls.undecoded}`}${calls.outstanding.length===0?"":`; mcp_outstanding=${calls.outstanding.map((call)=>`${call.name}@${call.outstandingMs}ms`).join(",")}`}`; + export type EngineBrokerInferenceGrant = Omit, "version" | "kind" | "requestId">; /** A refused grant request; `code` is closed (`auth_stale` is the stale shared realm, `grant_limit` the live-grant cap). */ export class EngineBrokerInferenceGrantRefused extends Error { @@ -45,6 +55,6 @@ export class EngineBrokerControlClient implements EngineBrokerTurnClient { const turnId=createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"),requestId=randomUUID();const request={version:ENGINE_BROKER_VERSION,kind:"start_turn",requestId,turnId,agentId,wakeId,prompt,mcpEndpoint,...(options.limits===undefined?{}:{limits:options.limits})} as const;const socket=createConnection({path:this.socketPath});const decoder=new EngineBrokerFrameDecoder(); return new Promise((resolve,reject)=>{let accepted=false,settled=false;const fail=()=>{if(settled)return;settled=true;cleanup();reject(new Error("engine broker unavailable"));};const cleanup=()=>{signal?.removeEventListener("abort",abort);socket.destroy();};const abort=()=>fail();signal?.addEventListener("abort",abort,{once:true});if(signal?.aborted)return abort();socket.once("connect",()=>socket.write(encodeEngineBrokerFrame(request)));socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const response=parseEngineBrokerResponse(value);if((response.kind!=="accepted"&&response.kind!=="completed"&&response.kind!=="failed")||response.requestId!==requestId||response.turnId!==turnId)throw new Error();if(response.kind==="accepted"){if(accepted)throw new Error();accepted=true;continue;}if(!accepted||settled)throw new Error();settled=true;cleanup(); if(options.model!==undefined&&response.model!==options.model){reject(new Error(`engine broker turn used model ${response.model}, not the declared ${options.model}`));return;} - if(response.kind==="completed")resolve(response.text);else reject(new Error(`engine broker turn failed (${response.code}${response.limitReason==="none"?"":`; limit=${response.limitReason}`}${response.diagnostic ? `; ${response.diagnostic.stage}/${response.diagnostic.failureClass}; exit=${response.diagnostic.exitCode}; signal=${response.diagnostic.termSignal}${response.diagnostic.reason===undefined?"":`; reason=${response.diagnostic.reason}`}` : ""})`));}}catch{fail();}});socket.once("error",fail);socket.once("close",()=>{if(!settled)fail();});}); + if(response.kind==="completed")resolve(response.text);else reject(new Error(`engine broker turn failed (${response.code}${response.limitReason==="none"?"":`; limit=${response.limitReason}`}${response.diagnostic ? `; ${response.diagnostic.stage}/${response.diagnostic.failureClass}; exit=${response.diagnostic.exitCode}; signal=${response.diagnostic.termSignal}${response.diagnostic.reason===undefined?"":`; reason=${response.diagnostic.reason}`}` : ""}${renderMcpCalls(response.mcpCalls)})`));}}catch{fail();}});socket.once("error",fail);socket.once("close",()=>{if(!settled)fail();});}); } } diff --git a/src/runtime/engineBrokerMcpCallLog.test.ts b/src/runtime/engineBrokerMcpCallLog.test.ts new file mode 100644 index 0000000..8550608 --- /dev/null +++ b/src/runtime/engineBrokerMcpCallLog.test.ts @@ -0,0 +1,73 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { EngineBrokerMcpCallLog, ENGINE_BROKER_MCP_CALL_INVALID, ENGINE_BROKER_MCP_CALL_TRUNCATED, ENGINE_BROKER_MCP_OUTSTANDING_MAX } from "./engineBrokerMcpCallLog.js"; + +const body = (value: unknown): Buffer => Buffer.from(JSON.stringify(value), "utf8"); +const call = (name: unknown, id = 1): Buffer => body({ jsonrpc: "2.0", id, method: "tools/call", params: { name, arguments: { text: "argument-bytes" } } }); + +/** A controllable clock: an outstanding call's whole value is how long it has been outstanding. */ +const clock = (): { log: EngineBrokerMcpCallLog; advance: (ms: number) => void } => { + let at = 1_000; + return { log: new EngineBrokerMcpCallLog(() => at), advance: (ms: number): void => { at += ms; } }; +}; + +test("an unanswered tool call is outstanding with its name and its elapsed time; an answered one is neither", () => { + const { log, advance } = clock(); + log.open("turn"); + const answered = log.begin("turn", call("daimon__moltnet_send")); + advance(20); answered.answer(); answered.close(); + const hung = log.begin("turn", call("daimon__moltnet_read", 2)); + advance(420_000); + const observed = log.observe("turn"); + assert.deepEqual(observed?.outstanding, [{ name: "daimon__moltnet_read", outstandingMs: 420_000 }]); + assert.equal(observed?.started, 2); assert.equal(observed?.answered, 1); assert.equal(observed?.undecoded, 0); + // A relay torn down by the turn's death answered nothing, and the elapsed + // time freezes where it stopped rather than growing with the report. + hung.close(); advance(5_000); + assert.deepEqual(log.observe("turn")?.outstanding, [{ name: "daimon__moltnet_read", outstandingMs: 420_000 }]); + assert.ok(!JSON.stringify(log.observe("turn")).includes("argument-bytes"), "names and timings only"); +}); + +test("absence stays absence: an unopened turn observes undefined, a turn that called nothing observes zero", () => { + const { log } = clock(); + assert.equal(log.observe("turn"), undefined); + log.open("turn"); + assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 0, outstanding: [] }); + // Everything that is not a tool call records nothing at all, so `started` + // stays a count of tool calls and not of traffic. + for (const value of [body({ jsonrpc: "2.0", id: 1, method: "tools/list" }), body({ jsonrpc: "2.0", method: "notifications/initialized" }), Buffer.alloc(0)]) log.begin("turn", value).answer(); + log.begin("turn", undefined).answer(); + assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 0, outstanding: [] }); + log.close("turn"); + assert.equal(log.observe("turn"), undefined, "a revoked turn keeps nothing"); +}); + +test("a body the facade could not read counts as undecoded, never as a call with a name", () => { + const { log } = clock(); + log.open("turn"); + log.begin("turn", Buffer.from("{not json", "utf8")); + log.undecodable("turn"); + assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 2, outstanding: [] }); + log.undecodable("absent"); +}); + +test("a tool name that is not a plain identifier is recorded as invalid, and a batch names each of its calls", () => { + const { log } = clock(); + log.open("turn"); + log.begin("turn", call("moltnet read\nBearer sk-live-000")); + log.begin("turn", body([])); + log.begin("turn", Buffer.from(`[${call("memory_recall", 2).toString("utf8")},${call("world_probe", 3).toString("utf8")}]`, "utf8")); + assert.deepEqual(log.observe("turn")?.outstanding.map((entry) => entry.name), [ENGINE_BROKER_MCP_CALL_INVALID, "memory_recall", "world_probe"]); +}); + +test("the outstanding list is bounded, and the earliest calls are the ones kept", () => { + const { log, advance } = clock(); + log.open("turn"); + for (let index = 0; index < ENGINE_BROKER_MCP_OUTSTANDING_MAX + 4; index += 1) { log.begin("turn", call(`tool_${index}`, index)); advance(1); } + const observed = log.observe("turn"); + assert.equal(observed?.started, ENGINE_BROKER_MCP_OUTSTANDING_MAX + 4); + assert.equal(observed?.outstanding.length, ENGINE_BROKER_MCP_OUTSTANDING_MAX); + assert.equal(observed?.outstanding[0]?.name, "tool_0"); + assert.equal(observed?.outstanding.at(-1)?.name, ENGINE_BROKER_MCP_CALL_TRUNCATED); +}); diff --git a/src/runtime/engineBrokerMcpCallLog.ts b/src/runtime/engineBrokerMcpCallLog.ts new file mode 100644 index 0000000..fc49dbf --- /dev/null +++ b/src/runtime/engineBrokerMcpCallLog.ts @@ -0,0 +1,137 @@ +/** + * The in-flight MCP tool calls of one brokered turn. + * + * Daimon writes a tool receipt only when a call *completes*, so a call that + * started and never returned is byte-identical, in every artifact, to a call + * that was never made. A live Grok turn stopped acting after its eighth + * provider response and was killed by the trial deadline seven minutes later + * with the proxy's per-request ledger reporting `open: 0` — every provider + * request closed — which leaves exactly one unlit path: a tool call the worker + * issued and the facade never answered. + * + * This is that light, and it follows the per-request ledger's rules rather + * than inventing its own: + * + * - **names and timings only.** The tool name off the JSON-RPC envelope and + * two clocks. Never arguments, never a result, never a session id, never a + * capability or bearer. A name that is not a plain short identifier is + * recorded as {@link ENGINE_BROKER_MCP_CALL_INVALID} rather than passed + * through, and the list is bounded at + * {@link ENGINE_BROKER_MCP_OUTSTANDING_MAX} with + * {@link ENGINE_BROKER_MCP_CALL_TRUNCATED} as its last entry. + * - **absence stays absence.** A turn the log never opened observes as + * `undefined`; a turn that made no call observes `started: 0`, which is not + * the same statement as an answered call. A POST whose body the facade could + * not read counts in `undecoded` and never as a call with a name, because a + * fabricated name is byte-identical to a measured one — and because "zero + * calls started" is exactly the reading this instrument exists to make + * trustworthy. + * - **it cannot fail a turn.** Every operation here is arithmetic over a map, + * the one parse is wrapped, and the facade treats a missing handle as a + * no-op. + * + * "Answered" means the facade wrote a complete response back to the worker — + * the relay reached its own `end()`. A relay that was torn down (the worker + * died, the tunnel broke, the turn aborted) did *not* answer, so its calls stay + * outstanding with the elapsed time they had reached. That is the whole point: + * the turn's death must not retroactively mark the call it was blocked on as + * finished. + */ +export const ENGINE_BROKER_MCP_OUTSTANDING_MAX = 16; +export const ENGINE_BROKER_MCP_CALL_INVALID = ""; +export const ENGINE_BROKER_MCP_CALL_TRUNCATED = ""; +/** A plain short identifier, or one of the two sentinels above. */ +export const ENGINE_BROKER_MCP_CALL_NAME = /^(?:||[A-Za-z0-9_.-]{1,64})$/u; +const TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/u; + +export type EngineBrokerOutstandingMcpCall = Readonly<{ name: string; outstandingMs: number }>; +/** + * What the facade saw of one turn's tool calls: how many started, how many the + * facade answered, how many POST bodies it could not read, and the ones still + * unanswered with the time each has been outstanding. + */ +export type EngineBrokerMcpCallObservation = Readonly<{ started: number; answered: number; undecoded: number; outstanding: readonly EngineBrokerOutstandingMcpCall[] }>; + +/** One relayed POST's calls. `answer` marks a complete relay; `close` ends it unanswered. Both are idempotent. */ +export interface EngineBrokerMcpCallHandle { answer(): void; close(): void } +const INERT: EngineBrokerMcpCallHandle = { answer: () => undefined, close: () => undefined }; + +type CallRecord = { readonly name: string; readonly startedAt: number; endedAt?: number }; +type TurnLog = { started: number; answered: number; undecoded: number; readonly live: Set; readonly ended: CallRecord[] }; + +export class EngineBrokerMcpCallLog { + private readonly turns = new Map(); + constructor(private readonly now: () => number = Date.now) {} + + /** A turn the facade registered. Re-opening an id resets it: a turn id is unique per turn. */ + open(turnId: string): void { this.turns.set(turnId, { started: 0, answered: 0, undecoded: 0, live: new Set(), ended: [] }); } + close(turnId: string): void { this.turns.delete(turnId); } + + /** A POST body the facade is about to relay. Anything that is not a `tools/call` records nothing. */ + begin(turnId: string, body: Uint8Array | undefined): EngineBrokerMcpCallHandle { + const log = this.turns.get(turnId); + if (log === undefined || body === undefined || body.byteLength === 0) return INERT; + const names = toolCallNames(body); + if (names === undefined) { log.undecoded += 1; return INERT; } + if (names.length === 0) return INERT; + const startedAt = this.now(); + const records = names.map((name): CallRecord => ({ name, startedAt })); + log.started += records.length; + for (const record of records) log.live.add(record); + let settled = false; + return { + answer: (): void => { + if (settled) return; settled = true; + log.answered += records.length; + for (const record of records) log.live.delete(record); + }, + close: (): void => { + if (settled) return; settled = true; + const endedAt = this.now(); + for (const record of records) { + log.live.delete(record); record.endedAt = endedAt; + // Retain only as many as can be reported; the earliest are the ones + // a hang is about, so a later flood cannot displace them. + if (log.ended.length < ENGINE_BROKER_MCP_OUTSTANDING_MAX) log.ended.push(record); + } + } + }; + } + + /** A POST whose body the facade refused to read (over its own bound), which is a call it cannot name. */ + undecodable(turnId: string): void { const log = this.turns.get(turnId); if (log !== undefined) log.undecoded += 1; } + + observe(turnId: string): EngineBrokerMcpCallObservation | undefined { + const log = this.turns.get(turnId); + if (log === undefined) return undefined; + const at = this.now(); + const pending = [...log.live, ...log.ended].sort((left, right) => left.startedAt - right.startedAt); + const outstanding = pending.map((record): EngineBrokerOutstandingMcpCall => ({ name: record.name, outstandingMs: Math.max(0, (record.endedAt ?? at) - record.startedAt) })); + return { + started: log.started, answered: log.answered, undecoded: log.undecoded, + outstanding: outstanding.length > ENGINE_BROKER_MCP_OUTSTANDING_MAX + ? [...outstanding.slice(0, ENGINE_BROKER_MCP_OUTSTANDING_MAX - 1), { name: ENGINE_BROKER_MCP_CALL_TRUNCATED, outstandingMs: 0 }] + : outstanding + }; + } +} + +/** + * The tool names one JSON-RPC POST asks for: `undefined` when the body did not + * decode at all (which is a fact of its own, not zero calls), `[]` when it + * decoded and asked for no tool. A batch names each of its calls. + */ +function toolCallNames(body: Uint8Array): readonly string[] | undefined { + let parsed: unknown; + try { parsed = JSON.parse(Buffer.from(body.buffer, body.byteOffset, body.byteLength).toString("utf8")); } catch { return undefined; } + const entries: readonly unknown[] = Array.isArray(parsed) ? parsed : [parsed]; + const names: string[] = []; + for (const entry of entries) { + if (!isRecord(entry) || entry.method !== "tools/call") continue; + const name = isRecord(entry.params) ? entry.params.name : undefined; + names.push(typeof name === "string" && TOOL_NAME.test(name) ? name : ENGINE_BROKER_MCP_CALL_INVALID); + } + return names; +} + +const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value); diff --git a/src/runtime/engineBrokerMcpFacade.test.ts b/src/runtime/engineBrokerMcpFacade.test.ts index 80bdfc0..54261ed 100644 --- a/src/runtime/engineBrokerMcpFacade.test.ts +++ b/src/runtime/engineBrokerMcpFacade.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; -import { createServer, type Server as HttpServer, type IncomingMessage } from "node:http"; -import { connect, type Socket } from "node:net"; +import { createServer, type Server as HttpServer, type IncomingMessage, type ServerResponse } from "node:http"; + import { randomUUID } from "node:crypto"; import test from "node:test"; @@ -52,6 +52,89 @@ test("MCP facade routes only valid active capabilities to the registered mount", try{assert.equal((await call("wrong-token-abcdefghijklmnopqrstuvwxyz0123456789")).status,403);assert.equal((await call(token)).status,200);assert.equal(calls,1);facade.revoke("turn-capabilities");assert.equal((await call(token)).status,403);assert.equal(calls,1);}finally{facade.revoke("turn-capabilities");await new Promise((resolve)=>target.close(()=>resolve()));} }); +/** + * Daimon writes a tool receipt only on completion, so a call that started and + * never returned reads exactly like a call that was never made — the one path + * a seven-minute live hang left unlit. The facade is where that difference is + * visible, and it has to survive the tear-down that ends the turn: a tunnel + * destroyed when the worker dies must not mark the call it was blocked on as + * answered, or the instrument erases the very evidence it exists to keep. + */ +test("MCP facade reports a tool call that started and never returned as outstanding, by name", async () => { + const facade = await sharedFacade(); + const held: ServerResponse[] = []; + // Two ways for a mount not to answer: never reply at all (`moltnet_read`), + // or open the stream and never deliver the result (`memory_recall`). + const target = createServer((request, response) => { + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.on("end", () => { + const asked = Buffer.concat(chunks).toString("utf8"); + if (asked.includes("moltnet_read")) { held.push(response); return; } + if (asked.includes("memory_recall")) { response.writeHead(200, { "content-type": "text/event-stream" }); response.write(": open\n\n"); held.push(response); return; } + response.writeHead(200, { "content-type": "application/json" }); response.end('{"jsonrpc":"2.0","id":1,"result":{}}'); + }); + }); + await new Promise((resolve) => target.listen(0, "127.0.0.1", resolve)); + const address = target.address(); if (address === null || typeof address === "string") throw new Error(); + const turnId = "turn-outstanding", token = facade.register("agent", turnId, `http://127.0.0.1:${address.port}/mcp`); + const post = (name: string, signal?: AbortSignal): Promise => fetch(FACADE_URL, { method: "POST", signal, headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name, arguments: { text: "argument-bytes-that-must-never-be-recorded" } } }) }); + const pending = new AbortController(); + /** + * A live call's elapsed time grows with every read, so two identical reads + * mean every relay has settled — the only moment at which "answered" is + * final. Polling for a name instead would read the log mid-teardown. + */ + const settled = async (): Promise> => { + for (let attempt = 0; attempt < 100; attempt += 1) { + const before = JSON.stringify(facade.observe(turnId)); + await new Promise((resolve) => setTimeout(resolve, 60)); + if (JSON.stringify(facade.observe(turnId)) === before) return facade.observe(turnId); + } + throw new Error("the facade's observation never settled"); + }; + const names = (observed: ReturnType): readonly string[] => (observed?.outstanding ?? []).map((call) => call.name); + try { + const answered = await post("daimon__moltnet_send"); + assert.equal(answered.status, 200); await answered.text(); + const hanging = post("daimon__moltnet_read", pending.signal).catch(() => undefined); + for (let attempt = 0; attempt < 200 && held.length === 0; attempt += 1) await new Promise((resolve) => setTimeout(resolve, 25)); + assert.equal(held.length, 1, "the mount never received the hung tool call"); + + const observed = facade.observe(turnId); + assert.deepEqual(names(observed), ["daimon__moltnet_read"], "the unanswered call must be outstanding, by name"); + assert.equal(observed?.started, 2); assert.equal(observed?.answered, 1, "the completed call must not be outstanding"); assert.equal(observed?.undecoded, 0); + assert.ok((observed?.outstanding[0]?.outstandingMs ?? -1) >= 0, "an outstanding call reports how long it has been waiting"); + assert.ok(!JSON.stringify(observed).includes("argument-bytes"), "names and timings only: no arguments"); + + // The turn's own death tears the tunnel down. The call was still never + // answered, and must still say so. + pending.abort(); await hanging; + const afterTeardown = await settled(); + assert.deepEqual(names(afterTeardown), ["daimon__moltnet_read"], "a torn-down relay is not an answer"); + assert.equal(afterTeardown?.answered, 1); + + // A stream the facade opened and never finished relaying is not an answer + // either. Awaiting the headers and one chunk puts the facade inside its own + // streaming relay before the client walks away, which is the branch that + // decides whether a half-written tunnel counts as an answer. + const halted = new AbortController(); + const half = await post("memory_recall", halted.signal); + assert.equal(half.status, 200); await half.body!.getReader().read(); halted.abort(); + const afterHalfRelay = await settled(); + assert.deepEqual(names(afterHalfRelay), ["daimon__moltnet_read", "memory_recall"], "a half-relayed stream is not an answer"); + assert.equal(afterHalfRelay?.answered, 1); + + facade.revoke(turnId); + assert.equal(facade.observe(turnId), undefined, "a turn the facade never registered observes as absence, not as zero"); + } finally { + pending.abort(); facade.revoke(turnId); + for (const response of held) response.destroy(); + target.closeAllConnections(); + await new Promise((resolve) => target.close(() => resolve())); + } +}); + /** * The brokered worker's real route: a real Daimon MCP mount behind a real * Streamable HTTP transport, reached by a real MCP client through the facade. @@ -315,50 +398,3 @@ test("revoking a turn tears down its open server-to-client stream, and closing n await second.close().catch(() => undefined); await rig.close().catch(() => undefined); }); - -/** - * The relay parks on this await whenever a tunnel is backpressured, and a - * parked await is invisible from outside: no status, no refusal, no line. So - * the assertion is that it settles at all — on a client that hung up - * mid-write, on the turn's abort, and on a genuine drain — with a deadline - * standing in for the hang. - */ -test("a backpressured MCP tunnel always settles: on a hang-up, on the turn's abort, and on a real drain", async () => { - const parked = new Map>(); - // One controller per phase: the turn whose abort is under test must not be - // the turn that is still relaying. - const controllers = new Map(); - const server = createServer((request, response) => { - const phase = request.url ?? ""; - const controller = new AbortController(); controllers.set(phase, controller); - response.writeHead(200, { "content-type": "text/event-stream" }); - // A paused client cannot absorb this, so `write` reports backpressure and - // the relay would park exactly here. - response.write("data: open\n\n"); - if (phase === "/drain") assert.equal(response.write(Buffer.alloc(16 * 1024 * 1024, 0x61)), false, "a paused client must backpressure the tunnel"); - parked.set(phase, awaitMcpTunnelDrain(response, controller.signal).then(() => "drained", (error: Error) => error.message)); - }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - const address = server.address(); if (address === null || typeof address === "string") throw new Error(); - const open = (phase: string): Promise => new Promise((resolve) => { - const socket = connect(address.port, "127.0.0.1", () => socket.write(`GET ${phase} HTTP/1.1\r\nhost: facade\r\n\r\n`)); - socket.once("data", () => { socket.pause(); resolve(socket); }); - }); - const settled = (phase: string): Promise => withDeadline(parked.get(phase)!, 3_000, `the ${phase} await never settled: the relay is parked`); - const sockets: Socket[] = []; - try { - sockets.push(await open("/hangup")); - sockets[0]!.destroy(); - assert.equal(await settled("/hangup"), "MCP tunnel closed", "a client that hung up mid-write wakes the await"); - sockets.push(await open("/abort")); - controllers.get("/abort")!.abort(); - assert.equal(await settled("/abort"), "MCP tunnel aborted", "the turn's own abort wakes the await"); - const draining = await open("/drain"); - sockets.push(draining); - draining.resume(); - assert.equal(await settled("/drain"), "drained", "a client that resumes reading resolves the await"); - } finally { - for (const socket of sockets) socket.destroy(); - await new Promise((resolve) => server.close(() => resolve())); - } -}); diff --git a/src/runtime/engineBrokerMcpFacade.ts b/src/runtime/engineBrokerMcpFacade.ts index 03eee1f..25a741f 100644 --- a/src/runtime/engineBrokerMcpFacade.ts +++ b/src/runtime/engineBrokerMcpFacade.ts @@ -1,6 +1,7 @@ import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import { Readable } from "node:stream"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; +import { EngineBrokerMcpCallLog, type EngineBrokerMcpCallObservation } from "./engineBrokerMcpCallLog.js"; /** * The brokered worker's only route to its own per-wake Daimon MCP mount. The @@ -53,6 +54,12 @@ export async function startEngineBrokerMcpFacade() { const targets = new Map(); /** In-flight upstream calls per turn, so a revoke or a close tears down any open SSE tunnel. */ const inflight = new Map>(); + /** + * What the facade saw of each turn's tool calls. A tool receipt is written + * only on completion, so without this a call that started and never returned + * and a call never made are the same absence (`engineBrokerMcpCallLog.ts`). + */ + const calls = new EngineBrokerMcpCallLog(); const server = createServer((request, response) => { void route(request, response).catch((error: unknown) => { @@ -76,8 +83,13 @@ export async function startEngineBrokerMcpFacade() { // Only POST carries a JSON-RPC body; drain anything else so the socket // never stalls waiting for a body the facade will not forward. - const body = method === "POST" ? await bounded(request) : (request.resume(), undefined); + let body: Buffer | undefined; + // A body refused for size is a call the log can never name, and counting + // it keeps "no tool call started" an honest reading rather than a gap. + try { body = method === "POST" ? await bounded(request) : (request.resume(), undefined); } + catch (error) { calls.undecodable(scope.turnId); throw error; } const payload = body === undefined ? undefined : (body.buffer.slice(body.byteOffset, body.byteOffset + body.byteLength) as ArrayBuffer); + const call = calls.begin(scope.turnId, body); const controller = new AbortController(); const open = inflight.get(scope.turnId) ?? new Set(); @@ -86,8 +98,12 @@ export async function startEngineBrokerMcpFacade() { const abort = (): void => controller.abort(); response.on("close", abort); try { - await forward(target, method, headersFor(method, request), payload, controller.signal, response); + // Answered only on a relay that reached its own end: a tunnel torn down + // by the worker's death must not mark the call it was blocked on as + // finished. + if (await forward(target, method, headersFor(method, request), payload, controller.signal, response)) call.answer(); } finally { + call.close(); response.off("close", abort); open.delete(controller); if (open.size === 0) inflight.delete(scope.turnId); @@ -110,13 +126,20 @@ export async function startEngineBrokerMcpFacade() { if (url.protocol !== "http:" || url.hostname !== "127.0.0.1" || url.pathname !== "/mcp") throw new TypeError("invalid scoped MCP mount"); if (targets.has(turnId)) throw new Error("MCP turn already registered"); targets.set(turnId, url.href); + calls.open(turnId); return capabilities.issue(agentId, turnId, 15 * 60_000, 128); }, revoke(turnId: string): void { targets.delete(turnId); capabilities.revoke(turnId); endTurnStreams(turnId); + calls.close(turnId); }, + /** + * What the facade saw of this turn's tool calls, or `undefined` for a turn + * it never registered. Read on the failure path, before `revoke`. + */ + observe: (turnId: string): EngineBrokerMcpCallObservation | undefined => calls.observe(turnId), close: async (): Promise => { for (const turnId of [...inflight.keys()]) endTurnStreams(turnId); await new Promise((resolve, reject) => { @@ -157,7 +180,7 @@ async function forward( body: ArrayBuffer | undefined, signal: AbortSignal, response: ServerResponse -): Promise { +): Promise { const upstream = await fetch(target, { method, headers, body, signal, redirect: "manual" }); // MCP never redirects, and following one would let the mount aim the facade // at a host the capability was never scoped to. `manual` also reports an @@ -171,7 +194,7 @@ async function forward( } outbound["content-type"] ??= "application/json"; response.writeHead(upstream.status, outbound); - if (upstream.body === null) { response.end(); return; } + if (upstream.body === null) { response.end(); return true; } const stream = Readable.fromWeb(upstream.body as Parameters[0]); try { for await (const chunk of stream) { @@ -179,10 +202,12 @@ async function forward( if (!response.write(chunk as Uint8Array)) await awaitMcpTunnelDrain(response, signal); } response.end(); + return true; } catch { // The client hung up or the mount's stream broke: tear the tunnel down // rather than leaving a half-written response open. response.destroy(); + return false; } finally { stream.destroy(); } diff --git a/src/runtime/engineBrokerProtocol.test.ts b/src/runtime/engineBrokerProtocol.test.ts index 2ce17e4..6f57c10 100644 --- a/src/runtime/engineBrokerProtocol.test.ts +++ b/src/runtime/engineBrokerProtocol.test.ts @@ -77,3 +77,28 @@ test("a failed worker's redacted reason is an optional bounded member of its dia {...failed,diagnostic:{...prelaunch,reason:"no worker ran"}} ])assert.throws(()=>parseEngineBrokerResponse(bad),/invalid broker frame/u); }); + +/** + * The in-flight tool-call observation (`engineBrokerMcpCallLog.ts`) rides the + * sealed failed frame, so the seam that carries the worker's last words and + * its accounting carries this too — nothing new on a tmpfs that dies with the + * container. It is names and timings, bounded, and internally consistent: a + * frame that claims more answered than started, or more outstanding than + * started minus answered, is a fabrication and is refused rather than clamped. + */ +test("a failed frame carries the broker's in-flight MCP tool-call observation, bounded and consistent", () => { + const value = { version: start.version, kind: "failed", requestId: "request-1", turnId: "turn-1", code: "limit_exceeded", mcpCalls: { started: 3, answered: 2, undecoded: 0, outstanding: [{ name: "daimon__moltnet_read", outstandingMs: 419_000 }] }, outcome: "failed", usage: null, model: "grok-4.6", requests: 8, limitReason: "timeout" } as const; + assert.deepEqual(parseEngineBrokerResponse(value), value); + for (const mcpCalls of [ + { ...value.mcpCalls, answered: 4 }, + { ...value.mcpCalls, started: 2 }, + { ...value.mcpCalls, outstanding: [{ name: "daimon__moltnet_read", outstandingMs: 1 }, { name: "memory_recall", outstandingMs: 1 }] }, + { ...value.mcpCalls, outstanding: [{ name: "daimon__moltnet_read", outstandingMs: -1 }] }, + { ...value.mcpCalls, outstanding: [{ name: "moltnet read; Bearer sk-live", outstandingMs: 1 }] }, + { ...value.mcpCalls, outstanding: [{ name: "daimon__moltnet_read", outstandingMs: 1, arguments: { text: "secret" } }] }, + { ...value.mcpCalls, outstanding: Array.from({ length: 17 }, () => ({ name: "tool", outstandingMs: 1 })) }, + { started: 3, answered: 2, outstanding: [] } + ]) assert.throws(() => parseEngineBrokerResponse({ ...value, mcpCalls }), /invalid broker frame/u, JSON.stringify(mcpCalls)); + // A v1 record predates the instrument; a v1 frame that carries it is forged. + assert.throws(() => parseEngineBrokerV1TerminalResponse({ version: "noopolis.daimon.engine-broker.v1", kind: "failed", requestId: "request-1", turnId: "turn-1", code: "engine_failed", mcpCalls: value.mcpCalls }), /invalid broker frame/u); +}); diff --git a/src/runtime/engineBrokerProtocol.ts b/src/runtime/engineBrokerProtocol.ts index 224ca44..643e41c 100644 --- a/src/runtime/engineBrokerProtocol.ts +++ b/src/runtime/engineBrokerProtocol.ts @@ -1,4 +1,5 @@ import { isEngineBrokerInferenceRequestKind, isEngineBrokerInferenceResponseKind, parseEngineBrokerInferenceRequest, parseEngineBrokerInferenceResponse, type EngineBrokerInferenceRequest, type EngineBrokerInferenceResponse } from "./engineBrokerInferenceProtocol.js"; +import { ENGINE_BROKER_MCP_CALL_NAME, ENGINE_BROKER_MCP_OUTSTANDING_MAX, type EngineBrokerMcpCallObservation } from "./engineBrokerMcpCallLog.js"; import { parseEngineBrokerTurnAccounting, parseEngineBrokerTurnLimitOverrides, type EngineBrokerTurnAccounting, type EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; /** @@ -34,7 +35,7 @@ export type EngineBrokerResponse = | Readonly<{ version: typeof VERSION; kind: "ready"; requestId: string; brokerUid: 2100; providerProxyPort: 43123; mcpFacadePort: 43124; registrations: number; credentialStale: false; realmLease: true; workerIsolation: true }> | Readonly<{ version: typeof VERSION; kind: "accepted"; requestId: string; turnId: string }> | (Readonly<{ version: typeof VERSION; kind: "completed"; requestId: string; turnId: string; text: string; workerPid: number; workerUid: number; workerStartTime: string }> & EngineBrokerTurnAccounting) - | (Readonly<{ version: typeof VERSION; kind: "failed"; requestId: string; turnId: string; code: EngineBrokerFailureCode; diagnostic?: EngineBrokerFailureDiagnostic }> & EngineBrokerTurnAccounting) + | (Readonly<{ version: typeof VERSION; kind: "failed"; requestId: string; turnId: string; code: EngineBrokerFailureCode; diagnostic?: EngineBrokerFailureDiagnostic; mcpCalls?: EngineBrokerMcpCallObservation }> & EngineBrokerTurnAccounting) | EngineBrokerInferenceResponse; /** * The one name for a fenced credential realm. The turn's failure code, the @@ -115,8 +116,12 @@ function parseTerminal(input: JsonRecord, expected: typeof VERSION | typeof V1): const base = { kind: "completed", requestId: id(input.requestId), turnId: id(input.turnId), text: text(input.text, 262_144), workerPid: input.workerPid as number, workerUid: input.workerUid as number, workerStartTime: id(input.workerStartTime) } as const; return expected === VERSION ? { version: VERSION, ...base, ...parseEngineBrokerTurnAccounting(input, "completed") } : { version: V1, ...base }; } - const fields = ["version", "kind", "requestId", "turnId", "code", ...accounting]; - exact(input, input.diagnostic === undefined ? fields : [...fields, "diagnostic"]); + // `mcpCalls` is the broker's own observation of the worker's tool calls + // (`engineBrokerMcpCallLog.ts`), additive in v2 and never part of a v1 + // record, which predates the instrument entirely. + if (expected === V1 && input.mcpCalls !== undefined) throw new TypeError("invalid broker frame"); + const fields = ["version", "kind", "requestId", "turnId", "code", ...accounting, ...(input.diagnostic === undefined ? [] : ["diagnostic"]), ...(input.mcpCalls === undefined ? [] : ["mcpCalls"])]; + exact(input, fields); const codes: readonly string[] = expected === VERSION ? ENGINE_BROKER_FAILURE_CODES : ENGINE_BROKER_FAILURE_CODES.filter((code) => code !== "limit_exceeded"); if (!codes.includes(input.code as string)) throw new TypeError("invalid broker frame"); let diagnostic:EngineBrokerFailureDiagnostic|undefined; @@ -125,7 +130,32 @@ function parseTerminal(input: JsonRecord, expected: typeof VERSION | typeof V1): if (expected === V1) return { version: V1, ...base } as V1Failed; const accountingValue = parseEngineBrokerTurnAccounting(input, "failed"); if ((input.code === "limit_exceeded") !== (accountingValue.limitReason !== "none")) throw new TypeError("invalid broker frame"); - return { version: VERSION, ...base, ...accountingValue }; + const mcpCalls = input.mcpCalls === undefined ? undefined : parseMcpCallObservation(input.mcpCalls); + return { version: VERSION, ...base, ...(mcpCalls === undefined ? {} : { mcpCalls }), ...accountingValue }; +} + +/** + * Names and timings, bounded, and internally consistent: a report can never + * claim more answered calls than started ones, nor more outstanding ones than + * started minus answered. Every count is its own measurement, so a missing + * field is refused rather than defaulted — a zero the broker did not measure + * would read exactly like one it did. + */ +function parseMcpCallObservation(value: unknown): EngineBrokerMcpCallObservation { + const input = record(value); + exact(input, ["started", "answered", "undecoded", "outstanding"]); + const started = input.started, answered = input.answered, undecoded = input.undecoded; + if (![started, answered, undecoded].every((count) => Number.isSafeInteger(count) && (count as number) >= 0)) throw new TypeError("invalid broker frame"); + if (!Array.isArray(input.outstanding) || input.outstanding.length > ENGINE_BROKER_MCP_OUTSTANDING_MAX) throw new TypeError("invalid broker frame"); + const outstanding = input.outstanding.map((entry) => { + const call = record(entry); + exact(call, ["name", "outstandingMs"]); + if (typeof call.name !== "string" || !ENGINE_BROKER_MCP_CALL_NAME.test(call.name)) throw new TypeError("invalid broker frame"); + if (!Number.isSafeInteger(call.outstandingMs) || (call.outstandingMs as number) < 0) throw new TypeError("invalid broker frame"); + return { name: call.name, outstandingMs: call.outstandingMs as number }; + }); + if ((answered as number) > (started as number) || outstanding.length > (started as number) - (answered as number)) throw new TypeError("invalid broker frame"); + return { started: started as number, answered: answered as number, undecoded: undecoded as number, outstanding }; } function closedDiagnostic(value:JsonRecord):boolean{ diff --git a/src/runtime/engineBrokerService.test.ts b/src/runtime/engineBrokerService.test.ts index c48a192..1dac5d5 100644 --- a/src/runtime/engineBrokerService.test.ts +++ b/src/runtime/engineBrokerService.test.ts @@ -62,3 +62,14 @@ test("a failed worker's own reason reaches the client instead of a bare exit cod await assert.rejects(client.turn("agent-a","wake-a","hello","http://127.0.0.1:44001/mcp"),/engine broker turn failed \(engine_failed; wait\/exec; exit=1; signal=0; reason=grok: session store unwritable\)/u); },async()=>{throw new EngineBrokerTurnFailure("engine_failed",{status:"worker_failed",stage:"wait",failureClass:"exec",profileApplied:false,exitCode:1,termSignal:0,workerPid:31,workerUid:2200,startTicks:"9",reason:"grok: session store unwritable"},{outcome:"failed",usage:null,model:"grok-4.6",requests:4,limitReason:"none"});}); }); + +/** + * The end of the seam: an outstanding tool call has to be readable by whoever + * reads the failure, not just sealed. The live hang would have read + * `mcp=1/2 answered; mcp_outstanding=daimon__moltnet_read@419000ms`. + */ +test("an outstanding MCP tool call reaches the client by name, with how long it waited", async () => { + await withService(async (client) => { + await assert.rejects(client.turn("agent-a","wake-a","hello","http://127.0.0.1:44001/mcp"),/engine broker turn failed \(limit_exceeded; limit=timeout; mcp=1\/2 answered; mcp_undecoded=1; mcp_outstanding=daimon__moltnet_read@419000ms\)/u); + },async()=>{throw new EngineBrokerTurnFailure("limit_exceeded",undefined,{outcome:"failed",usage:null,model:"grok-4.6",requests:8,limitReason:"timeout"},{started:2,answered:1,undecoded:1,outstanding:[{name:"daimon__moltnet_read",outstandingMs:419_000}]});}); +}); diff --git a/src/runtime/engineBrokerService.ts b/src/runtime/engineBrokerService.ts index aef6e33..79f55f0 100644 --- a/src/runtime/engineBrokerService.ts +++ b/src/runtime/engineBrokerService.ts @@ -54,7 +54,7 @@ function failed(socket:Socket,request:Extract & EngineBrokerTurnAccounting; export class EngineBrokerTurnFailure extends Error { - constructor(readonly code: Exclude, readonly diagnostic?: NativeBrokerDiagnostic, readonly accounting?: EngineBrokerTurnAccounting) { super("engine broker turn failed"); } + constructor(readonly code: Exclude, readonly diagnostic?: NativeBrokerDiagnostic, readonly accounting?: EngineBrokerTurnAccounting, readonly mcpCalls?: EngineBrokerMcpCallObservation) { super("engine broker turn failed"); } } /** Everything one broker turn touches, injected so the accounting and limit paths run under test without a native launcher. */ export type GrokEngineBrokerTurnDependencies = Readonly<{ turns: EngineBrokerTurnRegistry; proxy: Readonly<{ capabilities: Readonly<{ issue(agentId: string, turnId: string): string; revoke(turnId: string): void }>; registerIsolationGuard(turnId: string, guard: () => Promise): void; revokeIsolationGuard(turnId: string): void; registerTurn(turnId: string, turn: GrokBrokerProxyTurn): void; revokeTurn(turnId: string): void }>; - mcp: Readonly<{ register(agentId: string, turnId: string, endpoint: string): string; revoke(turnId: string): void }>; + mcp: Readonly<{ register(agentId: string, turnId: string, endpoint: string): string; revoke(turnId: string): void; observe?(turnId: string): EngineBrokerMcpCallObservation | undefined }>; credentialStale(): boolean; prepareIsolation(registration: EngineBrokerServiceRegistration): Promise<() => Promise>; runNative(input: NativeBrokerTurn, signal: AbortSignal): Promise>; @@ -91,10 +92,15 @@ export async function runGrokEngineBrokerTurn(deps: GrokEngineBrokerTurnDependen const diagnostic = error instanceof NativeBrokerTurnFailure ? error.diagnostic : nativeDiagnostic && !attested ? { ...nativeDiagnostic, status: "worker_failed" as const, stage: "attestation" as const, failureClass: error instanceof GrokWorkerAttestationFailure ? error.failureClass : "profile_invalid" as const, profileApplied: false } : undefined; const stream = output === undefined ? undefined : decodeGrokStreamUsage(output); const accounting = { outcome: "failed", usage: streamOrMeterUsage(stream, snapshot), model: declared, requests: requestCount(stream, snapshot), limitReason: snapshot.limitReason } as const; - const failed: EngineBrokerTerminalResponse = { version: request.version, kind: "failed", requestId: request.requestId, turnId, code, ...(diagnostic ? { diagnostic } : {}), ...accounting }; + // Read before the `finally` revokes this turn's MCP registration, and + // swallowed like every other instrument here: it must never be the reason + // a turn reports something other than why it failed. + let mcpCalls: EngineBrokerMcpCallObservation | undefined; + try { mcpCalls = deps.mcp.observe?.(turnId); } catch { mcpCalls = undefined; } + const failed: EngineBrokerTerminalResponse = { version: request.version, kind: "failed", requestId: request.requestId, turnId, code, ...(diagnostic ? { diagnostic } : {}), ...(mcpCalls === undefined ? {} : { mcpCalls }), ...accounting }; const reason = snapshot.limitReason !== "none" ? limitReasonFor[snapshot.limitReason] : rejected ? "turn_rejected" : "unknown"; await finishBrokerTurnWithUsage(deps.turns, request, failed, metering, { notionalUsd: 0, complete: false, reason, estimatedRequests: snapshot.estimatedRequests, requests: requestRows(stream, snapshot), ...(stream?.sessionId === undefined ? {} : { session: stream.sessionId }) }); - throw new EngineBrokerTurnFailure(code, diagnostic, accounting); + throw new EngineBrokerTurnFailure(code, diagnostic, accounting, mcpCalls); } finally { clearTimeout(timer); signal?.removeEventListener("abort", onAbort); meter.abortInFlight(); deps.proxy.revokeTurn(turnId); deps.proxy.revokeIsolationGuard(turnId); deps.proxy.capabilities.revoke(turnId); deps.mcp.revoke(turnId); @@ -105,7 +111,7 @@ function replay(response: EngineBrokerTerminalResponse): GrokEngineBrokerTurnRes const accounting = { outcome: response.outcome, usage: response.usage, model: response.model, requests: response.requests, limitReason: response.limitReason }; if (response.kind === "completed") return { text: response.text, workerPid: response.workerPid, workerUid: response.workerUid, workerStartTime: response.workerStartTime, ...accounting, outcome: "completed" }; const code = response.code === "turn_conflict" || response.code === "unavailable" ? "engine_failed" : response.code; - throw new EngineBrokerTurnFailure(code, response.diagnostic as NativeBrokerDiagnostic | undefined, accounting); + throw new EngineBrokerTurnFailure(code, response.diagnostic as NativeBrokerDiagnostic | undefined, accounting, response.mcpCalls); } /** The proxy saw every forwarded request; the stream is the fallback when no request crossed this proxy. */ diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index 8f0f50d..b8d6701 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -11,6 +11,7 @@ import { decodeNativeBrokerResult, ENGINE_BROKER_NATIVE_RESULT_BYTES, type Nativ import type { EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; import { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; +import type { EngineBrokerMcpCallObservation } from "./engineBrokerMcpCallLog.js"; import { EngineBrokerTurnFailure, runGrokEngineBrokerTurn, type GrokEngineBrokerTurnDependencies } from "./grokEngineBrokerTurn.js"; import { TURN_REQUEST_LEDGER_VERSION } from "./turnRequestLedger.js"; import { dedupeTurnUsageRows, TURN_USAGE_LEDGER_VERSION } from "./turnUsageLedger.js"; @@ -69,7 +70,7 @@ const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId const registration: EngineBrokerServiceRegistration = { agentId: "foreman", slot: 0, workerUid: 2_200, workspace: "/workspace", profilePath: "/workers/0/.grok/sandbox.toml", eventsPath: "/workers/0/.grok/sessions/sandbox-events.jsonl", profileSha256: "a".repeat(64), usageLedgerPath: ledger, limits, model: { model: "grok-4.6", reasoningEffort: "low" } }; const deps: GrokEngineBrokerTurnDependencies = { turns: syncDirectory === undefined ? new EngineBrokerTurnRegistry(turnStore) : new EngineBrokerTurnRegistry(turnStore, undefined, syncDirectory), proxy, credentialStale: () => false, - mcp: { register: () => "mcp-capability-0123456789abcdef", revoke: () => undefined }, + mcp: { register: () => "mcp-capability-0123456789abcdef", revoke: () => undefined, observe: () => mcpObservation }, prepareIsolation: async () => async () => undefined, runNative: async (input: NativeBrokerTurn, signal: AbortSignal) => nativeResult(await worker(() => post(proxy.port, input.providerCapability), signal)) }; @@ -83,6 +84,9 @@ const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId } finally { await proxy.close(); await rm(root, { recursive: true, force: true }); } }; +/** What the facade would have seen of this turn's tool calls; the turn only reads it. */ +let mcpObservation: EngineBrokerMcpCallObservation | undefined; + const twoRequests: Worker = async (send) => { assert.equal(await send(), 200); assert.equal(await send(), 200); return stream(); }; test("a completed turn seals its accounting, writes one usage row and per-request rows, and a replay never re-meters", async () => { @@ -344,3 +348,35 @@ test("a worker whose work succeeded but whose output crossed the launcher bound assert.equal((await usageRows()).length, 1, "the replayed failure is not metered again"); }); }); + +/** + * The hang this instrument was built for: the worker stops acting with every + * provider request closed, the deadline kills it, and the only remaining + * question is whether it was waiting on a tool call. The answer has to reach + * the host, and the slot's control root is tmpfs that dies with the container — + * so it rides the seam the worker's last words and the sealed usage already + * ride: the sealed terminal response, which a replay hands back unchanged. + */ +test("a failed turn carries the facade's in-flight tool-call observation, and its replay still does", async () => { + await withBroker(async ({ turn }) => { + mcpObservation = { started: 3, answered: 2, undecoded: 0, outstanding: [{ name: "daimon__moltnet_read", outstandingMs: 419_000 }] }; + const worker: Worker = async (send) => { assert.equal(await send(), 200); assert.equal(await send(), 200); throw new Error("engine broker turn failed"); }; + const carried = (error: unknown): boolean => { + assert.ok(error instanceof EngineBrokerTurnFailure); + assert.deepEqual(error.mcpCalls, { started: 3, answered: 2, undecoded: 0, outstanding: [{ name: "daimon__moltnet_read", outstandingMs: 419_000 }] }); + return true; + }; + await assert.rejects(turn("wake-mcp-outstanding", worker), carried); + // The replay reads the durable record back through the frame parser, so + // this is the sealed bytes answering, not the live facade. + mcpObservation = undefined; + await assert.rejects(turn("wake-mcp-outstanding", worker), carried); + }); +}); + +test("a turn whose facade observed nothing seals no observation at all", async () => { + await withBroker(async ({ turn }) => { + mcpObservation = undefined; + await assert.rejects(turn("wake-mcp-absent", async (send) => { await send(); throw new Error("engine broker turn failed"); }), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.mcpCalls === undefined); + }); +}); From 1ec76f2286e844ff620d4bbaa7604363a74197ff Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 11:59:22 +0200 Subject: [PATCH 111/124] feat: seal a brokered turn's terminal evidence into the broker's ledger directory --- src/runtime/AGENTS.md | 34 ++++++ src/runtime/engineBrokerSealLedger.test.ts | 122 +++++++++++++++++++++ src/runtime/engineBrokerSealLedger.ts | 109 ++++++++++++++++++ src/runtime/engineBrokerServiceConfig.ts | 13 ++- src/runtime/grokEngineBrokerLedger.ts | 47 ++++++-- src/runtime/grokEngineBrokerMetering.ts | 7 +- src/runtime/grokEngineBrokerTurn.ts | 4 +- src/runtime/index.ts | 3 +- 8 files changed, 322 insertions(+), 17 deletions(-) create mode 100644 src/runtime/engineBrokerSealLedger.test.ts create mode 100644 src/runtime/engineBrokerSealLedger.ts diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 5a158bb..f4243e1 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -314,6 +314,40 @@ answer, so the call it was blocked on stays outstanding with the elapsed time it had reached — otherwise the turn's death would erase the evidence the instrument exists to keep. +That seam is enough for a turn that *fails with a reply* and not for the turn +the instrument was built for. A worker that crashes still produces a terminal +response; a worker that HANGS is cancelled by its client's deadline, and a +cancelled turn has no client left to answer, so the sealed response — with +`mcpCalls` and the worker's redacted last words riding on it — is sealed into a +turn record in the broker's own `0700` turn store and dies with the slot's +tmpfs. Six live runs reproduced that exactly. What *does* survive a slot is the +broker's ledger directory, which Paideia already recovers `usage.jsonl` and +`requests.jsonl` from on the failure path, so `engineBrokerSealLedger.ts` writes +a third stream beside them: one `noopolis.daimon.turn-seal.v1` row per sealed +terminal turn (`turns.jsonl`, `engineBrokerSealLedgerPathFor`), rendered from +the sealed response and nothing else. Its members are the accounting, the +failure `code`, the diagnostic's closed `status`/`stage`/`failure_class` with +the reason `engineBrokerNativeClient.ts` already redacted and bounded, and the +facade's `mcp` observation — names, counts and elapsed milliseconds. Never a +prompt, body, reply, bearer, capability or session id; the terminal response +carries none of those in the first place, and the projection is an allow-list +rather than a spread, so a future additive member of the response cannot become +a ledger field by accident. + +Two invariants make it worth having. The row is rendered for *every* terminal +turn including one whose `usage` is `null` — a turn cancelled before any spend +could be attributed writes no usage row at all, and is precisely the turn whose +outstanding call has no other route out. And absence stays absence three ways: +no `mcp` member when the facade never observed the turn, `started: 0` when it +observed a turn that called nothing, and no row when nothing sealed. Reading +any of those three as another is the failure this stream exists to prevent. The +line is sealed into the turn record's ledger bytes with the other two and +appended last, so a replay completes an interrupted append the same way and +readers dedupe on `turn`; `seal` is optional in `parseBrokerTurnLedgerLines`, so +a record written before the stream existed still replays. It is advisory +throughout: `recordLedgerLines` swallows every I/O fault, and nothing here can +refuse, delay or fail a turn. + Worker `GROK_HOME` layout the deployment must provision (attested before every turn by `grokWorkerHomeAttestation.ts`, recorded in `GROK_ENGINE_BROKER.worker.home`): `$GROK_HOME` and `$GROK_HOME/sessions` `root: 1771`; `config.toml`, diff --git a/src/runtime/engineBrokerSealLedger.test.ts b/src/runtime/engineBrokerSealLedger.test.ts new file mode 100644 index 0000000..9b5ecc2 --- /dev/null +++ b/src/runtime/engineBrokerSealLedger.test.ts @@ -0,0 +1,122 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import type { EngineBrokerMcpCallObservation } from "./engineBrokerMcpCallLog.js"; +import { engineBrokerSealLedgerPathFor, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +import { TURN_SEAL_LEDGER_VERSION } from "./engineBrokerSealLedger.js"; +import { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; +import { EngineBrokerTurnFailure, runGrokEngineBrokerTurn, type GrokEngineBrokerTurnDependencies } from "./grokEngineBrokerTurn.js"; + +/** + * The hung turn's evidence, across the one boundary it has to cross. + * + * A cancelled or timed-out turn never answers its client, so the sealed + * terminal response — the only carrier of `mcpCalls` and the worker's redacted + * last words — dies with the slot. These tests pin the other route: the same + * sealed response, projected into the broker's own ledger directory, which + * Paideia already recovers `usage.jsonl` and `requests.jsonl` from. + * + * The turn runs through its real registry, meter, seal and ledger path; only + * the launcher, the proxy registrations and the MCP facade's observation are + * stubbed, exactly as the live hang presented them. + */ +const registration = (usageLedgerPath: string): EngineBrokerServiceRegistration => ({ + agentId: "foreman", slot: 0, workerUid: 2_200, workspace: "/workspace", + profilePath: "/workers/0/.grok/sandbox.toml", eventsPath: "/workers/0/.grok/sessions/sandbox-events.jsonl", + profileSha256: "a".repeat(64), usageLedgerPath, limits: { maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }, + model: { model: "grok-4.6", reasoningEffort: "low" } +}); + +const proxy: GrokEngineBrokerTurnDependencies["proxy"] = { + capabilities: { issue: () => "provider-capability-0123456789ab", revoke: () => undefined }, + registerIsolationGuard: () => undefined, revokeIsolationGuard: () => undefined, + registerTurn: () => undefined, revokeTurn: () => undefined +}; + +/** A worker that does real work and then stops acting, until the deadline aborts it. */ +const hangs = (signal: AbortSignal): Promise => new Promise((_resolve, reject) => { + const fail = (): void => reject(new Error("engine broker turn failed")); + if (signal.aborted) fail(); else signal.addEventListener("abort", fail, { once: true }); +}); + +async function cancelledTurn(observe: () => EngineBrokerMcpCallObservation | undefined): Promise | undefined; usage: string; failure: EngineBrokerTurnFailure }>> { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-seal-")); + try { + const usageLedgerPath = path.join(root, "usage.jsonl"); + const deps: GrokEngineBrokerTurnDependencies = { + turns: new EngineBrokerTurnRegistry(path.join(root, "turns")), proxy, credentialStale: () => false, + mcp: { register: () => "mcp-capability-0123456789abcdef", revoke: () => undefined, observe }, + prepareIsolation: async () => async () => undefined, + runNative: async (_input, signal) => hangs(signal) + }; + const controller = new AbortController(); + const running = runGrokEngineBrokerTurn(deps, registration(usageLedgerPath), "wake-hung", "prompt", "http://127.0.0.1:43124/mcp", controller.signal); + setTimeout(() => controller.abort(), 5); + const failure = await running.then(() => { throw new Error("the hung turn resolved"); }, (error: unknown) => error as EngineBrokerTurnFailure); + const text = await readFile(engineBrokerSealLedgerPathFor(usageLedgerPath), "utf8").catch(() => ""); + const lines = text.split("\n").filter((line) => line.length > 0); + assert.ok(lines.length <= 1, "one sealed turn writes at most one seal row"); + return { seal: lines[0] === undefined ? undefined : JSON.parse(lines[0]) as Record, usage: await readFile(usageLedgerPath, "utf8").catch(() => ""), failure }; + } finally { await rm(root, { recursive: true, force: true }); } +} + +/** + * The finding this whole instrument exists for, on the host side of the seam. + * + * Live: ten provider requests, all closed, one tool receipt, then 430 s of + * silence and an abort at 489 s. The facade's log says the worker was blocked + * on `use_tool` the whole time, the sealed response carries it — and the + * sealed response never travels, because a cancelled turn has no client left to + * answer. The row in `turns.jsonl` is that evidence on a durable file the + * evaluator already reads. + * + * Mutation: drop the `seal` append from `appendBrokerTurnLedger`, or the `mcp` + * member from `renderBrokerTurnSealLine`, and this goes red. So does reverting + * `renderBrokerTurnLedger` to return `EMPTY_BROKER_TURN_LEDGER` for a turn with + * no attributable usage — which is exactly this turn. + */ +test("a cancelled turn's outstanding MCP call reaches the host on the broker's own ledger directory", async () => { + const { seal, usage, failure } = await cancelledTurn(() => ({ started: 1, answered: 0, undecoded: 0, outstanding: [{ name: "use_tool", outstandingMs: 430_112 }] })); + assert.equal(failure.code, "cancelled"); + // The usage ledger stays silent for a turn with nothing to attribute, so the + // seal row is the only host-visible account this turn has. + assert.equal(usage, ""); + assert.ok(seal, "a cancelled turn seals a row"); + assert.equal(seal.v, TURN_SEAL_LEDGER_VERSION); + assert.equal(seal.agent, "foreman"); + assert.equal(seal.wake, "wake-hung"); + assert.equal(seal.outcome, "failed"); + assert.equal(seal.code, "cancelled"); + assert.equal(seal.limit_reason, "none"); + assert.equal(seal.model, "grok-4.6"); + assert.deepEqual(seal.mcp, { started: 1, answered: 0, undecoded: 0, outstanding: [{ name: "use_tool", outstanding_ms: 430_112 }] }); +}); + +/** + * Absence stays absence, and the two absences are not the same statement. + * + * A turn that called no tool measured `started: 0`; a turn the facade never + * registered measured nothing at all and writes no `mcp` member; a turn that + * never sealed writes no row. Reading them as one another is precisely the + * mistake this session kept making. + * + * Mutation: render `mcp` unconditionally (as `{}` or as zeros) when the + * observation is `undefined`, and the second assertion goes red. + */ +test("a cancelled turn that called no tool is distinguishable from one the facade never observed, and both from no row at all", async () => { + const called = await cancelledTurn(() => ({ started: 0, answered: 0, undecoded: 0, outstanding: [] })); + assert.deepEqual(called.seal?.mcp, { started: 0, answered: 0, undecoded: 0, outstanding: [] }); + + const unobserved = await cancelledTurn(() => undefined); + assert.ok(unobserved.seal, "an unobserved turn still seals its row"); + assert.equal(Object.hasOwn(unobserved.seal, "mcp"), false); + + // And the third state: no seal row at all, which is what every one of the six + // live runs published before this stream existed. + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-seal-none-")); + try { await assert.rejects(readFile(engineBrokerSealLedgerPathFor(path.join(root, "usage.jsonl")), "utf8")); } + finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/src/runtime/engineBrokerSealLedger.ts b/src/runtime/engineBrokerSealLedger.ts new file mode 100644 index 0000000..52d38c1 --- /dev/null +++ b/src/runtime/engineBrokerSealLedger.ts @@ -0,0 +1,109 @@ +import { ENGINE_BROKER_MCP_CALL_NAME, ENGINE_BROKER_MCP_OUTSTANDING_MAX } from "./engineBrokerMcpCallLog.js"; +import type { EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; +import { TURN_USAGE_LEDGER, TURN_USAGE_MAX_IDENTIFIER_CHARS } from "./turnUsageLedger.js"; + +/** + * The operator-visible half of a sealed terminal turn, as a durable row. + * + * The seal itself already carries everything an operator needs to read a turn + * that stopped acting — the failure code, the worker's own redacted last words, + * and `engineBrokerMcpCallLog.ts`'s observation of the tool call the worker was + * blocked on. All of it travels in the control-protocol terminal response, and + * that response is the one artifact a *hung* turn never produces: the client is + * gone before the broker answers, the turn registry record lives in the + * broker's own `0700` turn store, and a training slot's control root is tmpfs + * that dies with the container. Six live runs reproduced the same signature and + * none of them could read the instrument built for it. + * + * What does survive a slot is the broker's ledger directory: Paideia already + * recovers `usage.jsonl` and `requests.jsonl` from it on the failure path. So + * this is a third stream beside those two, written by the broker — still the + * single sealed usage writer — from the sealed response and nothing else, at + * the moment that response is sealed. + * + * Its rules are the two ledgers' rules: + * + * - **Numbers, names, timings and closed vocabularies only.** The failure code, + * the accounting, the diagnostic's closed `status`/`stage`/`failure_class` + * and its already-redacted, already-bounded, control-character-free `reason` + * (`engineBrokerNativeClient.ts` produced it; nothing here re-derives it), + * plus tool-call names and elapsed milliseconds. Never a prompt, a body, a + * reply, a bearer, a capability or a session id — none of which the terminal + * response carries in the first place. + * - **Absence stays absence.** `mcp` is written only when the facade actually + * observed the turn, `diagnostic` only when the sealed response carried one, + * and `code` only for a failure. A turn that called no tool publishes + * `started: 0`, which is a measurement; a turn the facade never registered + * publishes no `mcp` member at all, which is not. + * - **It can never fail a turn.** The row is rendered from an + * already-validated frame and appended through `recordLedgerLines`, which + * swallows every I/O fault. + * + * A separate stream and a separate `v`, for `turnRequestLedger.ts`'s reason: + * Spawnfile's reader pins `noopolis.daimon.turn-usage.v1` and drops any other + * `v` outright, and Paideia's request reader refuses a row it cannot type. A + * row in a new file is invisible to both. + */ +export const TURN_SEAL_LEDGER_VERSION = "noopolis.daimon.turn-seal.v1" as const; + +/** Default location: beside `usage.jsonl` and `requests.jsonl`, no new mount. */ +export const TURN_SEAL_LEDGER = { + version: TURN_SEAL_LEDGER_VERSION, + directoryPath: TURN_USAGE_LEDGER.directoryPath, + filePath: `${TURN_USAGE_LEDGER.directoryPath}/turns.jsonl`, + rotatedFilePath: `${TURN_USAGE_LEDGER.directoryPath}/turns.jsonl.1`, + fileMode: TURN_USAGE_LEDGER.fileMode +} as const; + +/** A rendered seal line is bounded by its own contents: a 768-byte reason plus 16 bounded names. */ +export const TURN_SEAL_MAX_LINE_BYTES = 8_192; + +export type BrokerTurnSealEntry = Readonly<{ agent: string; wake: string; at: string }>; + +const bounded = (value: string): string => [...value].slice(0, TURN_USAGE_MAX_IDENTIFIER_CHARS).join(""); + +/** + * One newline-terminated row for one sealed terminal turn. + * + * Every member is copied from the terminal response the protocol already + * validated, so this projection cannot widen what the response admits; it is an + * allow-list rather than a spread, so a future additive member of the response + * does not silently become a ledger field. + */ +export const renderBrokerTurnSealLine = (terminal: EngineBrokerTerminalResponse, entry: BrokerTurnSealEntry): string => `${JSON.stringify({ + v: TURN_SEAL_LEDGER_VERSION, + agent: bounded(entry.agent), + wake: bounded(entry.wake), + engine: "grok", + at: entry.at, + turn: terminal.turnId, + outcome: terminal.outcome, + requests: terminal.requests, + model: terminal.model, + limit_reason: terminal.limitReason, + ...(terminal.kind === "failed" ? { code: terminal.code } : {}), + ...(terminal.kind === "failed" && terminal.diagnostic !== undefined + ? { + diagnostic: { + status: terminal.diagnostic.status, + stage: terminal.diagnostic.stage, + failure_class: terminal.diagnostic.failureClass, + exit_code: terminal.diagnostic.exitCode, + term_signal: terminal.diagnostic.termSignal, + ...(terminal.diagnostic.reason === undefined ? {} : { reason: terminal.diagnostic.reason }) + } + } + : {}), + ...(terminal.kind === "failed" && terminal.mcpCalls !== undefined + ? { + mcp: { + started: terminal.mcpCalls.started, + answered: terminal.mcpCalls.answered, + undecoded: terminal.mcpCalls.undecoded, + outstanding: terminal.mcpCalls.outstanding + .slice(0, ENGINE_BROKER_MCP_OUTSTANDING_MAX) + .map((call) => ({ name: ENGINE_BROKER_MCP_CALL_NAME.test(call.name) ? call.name : "", outstanding_ms: call.outstandingMs })) + } + } + : {}) +})}\n`; diff --git a/src/runtime/engineBrokerServiceConfig.ts b/src/runtime/engineBrokerServiceConfig.ts index af2a159..eb6d274 100644 --- a/src/runtime/engineBrokerServiceConfig.ts +++ b/src/runtime/engineBrokerServiceConfig.ts @@ -12,7 +12,7 @@ export const ENGINE_BROKER_SERVICE_V2 = "noopolis.daimon.engine-broker-service.v /** One root-provisioned broker slot. Every field is fixed at provisioning time; a wake can only lower `limits`. */ export type EngineBrokerServiceRegistration = Readonly<{ agentId: string; slot: number; workerUid: number; workspace: string; profilePath: string; eventsPath: string; profileSha256: string; - /** Per-slot usage ledger the broker appends turn rows to; per-request rows go to `requests.jsonl` beside it. */ + /** Per-slot usage ledger the broker appends turn rows to; per-request rows go to `requests.jsonl` and per-turn seals to `turns.jsonl` beside it. */ usageLedgerPath: string; limits: EngineBrokerTurnLimits; model: GrokBrokerModelPolicy; @@ -41,6 +41,13 @@ const absolute = (item: unknown): item is string => typeof item === "string" && /** The per-request stream written beside a registration's usage ledger. */ export const engineBrokerRequestLedgerPathFor = (usageLedgerPath: string): string => path.posix.join(path.posix.dirname(usageLedgerPath), "requests.jsonl"); +/** + * The per-turn seal stream written beside the other two + * (`engineBrokerSealLedger.ts`): the operator-visible half of a sealed terminal + * response, for the turn whose response never reaches a client. + */ +export const engineBrokerSealLedgerPathFor = (usageLedgerPath: string): string => path.posix.join(path.posix.dirname(usageLedgerPath), "turns.jsonl"); + /** * Strict `service.json` parser. * @@ -67,7 +74,7 @@ export function parseEngineBrokerServiceConfig(value: unknown): EngineBrokerServ const base = { agentId, slot: slot as number, workerUid: workerUid as number, workspace, profilePath, eventsPath, profileSha256 }; if (!v2) return { ...base, usageLedgerPath: TURN_USAGE_LEDGER.filePath, limits: DEFAULT_GROK_BROKER_TURN_LIMITS, model: DEFAULT_GROK_BROKER_MODEL_POLICY }; const usageLedgerPath = entry.usageLedgerPath; - if (!ledgerPath(usageLedgerPath) || usageLedgerPath === engineBrokerRequestLedgerPathFor(usageLedgerPath)) throw invalid(); + if (!ledgerPath(usageLedgerPath) || usageLedgerPath === engineBrokerRequestLedgerPathFor(usageLedgerPath) || usageLedgerPath === engineBrokerSealLedgerPathFor(usageLedgerPath)) throw invalid(); let limits: EngineBrokerTurnLimits; try { limits = parseEngineBrokerTurnLimits(entry.limits); } catch { throw invalid(); } return { ...base, usageLedgerPath, limits, model: parseServiceModel(entry.model) }; @@ -76,7 +83,7 @@ export function parseEngineBrokerServiceConfig(value: unknown): EngineBrokerServ if (!Object.hasOwn(value, "inferenceLedgerPath")) return base; const inferenceLedgerPath = value.inferenceLedgerPath; if (!ledgerPath(inferenceLedgerPath)) throw invalid(); - const subject = new Set([TURN_USAGE_LEDGER.filePath, TURN_REQUEST_LEDGER.filePath, ...registrations.flatMap((entry) => [entry.usageLedgerPath, engineBrokerRequestLedgerPathFor(entry.usageLedgerPath)])]); + const subject = new Set([TURN_USAGE_LEDGER.filePath, TURN_REQUEST_LEDGER.filePath, ...registrations.flatMap((entry) => [entry.usageLedgerPath, engineBrokerRequestLedgerPathFor(entry.usageLedgerPath), engineBrokerSealLedgerPathFor(entry.usageLedgerPath)])]); if (subject.has(inferenceLedgerPath)) throw invalid(); return { ...base, inferenceLedgerPath }; } diff --git a/src/runtime/grokEngineBrokerLedger.ts b/src/runtime/grokEngineBrokerLedger.ts index ebf6073..3d21e47 100644 --- a/src/runtime/grokEngineBrokerLedger.ts +++ b/src/runtime/grokEngineBrokerLedger.ts @@ -1,5 +1,6 @@ import { readFile } from "node:fs/promises"; +import { renderBrokerTurnSealLine, TURN_SEAL_LEDGER_VERSION, TURN_SEAL_MAX_LINE_BYTES } from "./engineBrokerSealLedger.js"; import { recordLedgerLines, renderGrokTurnRequestLines, TURN_REQUEST_LEDGER_VERSION, type GrokTurnRequest } from "./turnRequestLedger.js"; import { renderTurnUsageLine, TURN_USAGE_LEDGER_VERSION, type TurnUsageFailureReason } from "./turnUsageLedger.js"; import type { EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; @@ -16,15 +17,23 @@ import type { EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; * normal append finds the row and writes nothing, and readers dedupe on `turn` * should two replays race. */ -export type BrokerTurnLedgerLines = Readonly<{ usage: string | null; requests: string }>; +export type BrokerTurnLedgerLines = Readonly<{ usage: string | null; requests: string; seal?: string }>; +/** A v1 record, or a replay with no sealed bytes at all: nothing to append, including no seal. */ export const EMPTY_BROKER_TURN_LEDGER: BrokerTurnLedgerLines = Object.freeze({ usage: null, requests: "" }); export type BrokerTurnLedgerDetail = Readonly<{ agentId: string; wakeId: string; notionalUsd: number; complete: boolean; reason?: TurnUsageFailureReason; requests: readonly GrokTurnRequest[]; session?: string; estimatedRequests: number }>; export function renderBrokerTurnLedger(terminal: EngineBrokerTerminalResponse, detail: BrokerTurnLedgerDetail): BrokerTurnLedgerLines { - if (terminal.usage === null) return EMPTY_BROKER_TURN_LEDGER; - const { usage } = terminal, at = new Date().toISOString(); + const at = new Date().toISOString(); + // The seal row is rendered for *every* terminal turn, including one whose + // usage is null. That is the whole point: a turn cancelled before any usage + // could be attributed is exactly the turn whose outstanding MCP call and + // redacted last words have no other route to the host. + const seal = renderBrokerTurnSealLine(terminal, { agent: detail.agentId, wake: detail.wakeId, at }); + if (terminal.usage === null) return { usage: null, requests: "", seal }; + const { usage } = terminal; return { + seal, usage: renderTurnUsageLine({ agent: detail.agentId, wake: detail.wakeId, engine: "grok", at, usage: { input: usage.input, output: usage.output, cacheRead: usage.cacheRead, cacheWrite: usage.cacheWrite, total: usage.total, calls: terminal.requests, notionalUsd: detail.notionalUsd, complete: detail.complete }, @@ -38,11 +47,20 @@ export function renderBrokerTurnLedger(terminal: EngineBrokerTerminalResponse, d const MAX_USAGE_LINE_BYTES = 4_096, MAX_REQUEST_LINES_BYTES = 262_144; const rows = (text: string): Record[] => text.split("\n").filter((line) => line.length > 0).map((line) => { const value: unknown = JSON.parse(line); if (value === null || typeof value !== "object" || Array.isArray(value)) throw new Error(); return value as Record; }); -/** Strict check of stored ledger bytes: exactly the turn's own rows, newline-terminated, bounded. */ +/** + * Strict check of stored ledger bytes: exactly the turn's own rows, + * newline-terminated, bounded. + * + * `seal` is optional so a record sealed before this stream existed still + * replays; its absence means the turn owes no seal row, never that one was + * lost. + */ export function parseBrokerTurnLedgerLines(value: unknown, turnId: string): BrokerTurnLedgerLines { const invalid = () => new Error("broker turn registry unavailable"); - if (value === null || typeof value !== "object" || Array.isArray(value) || Object.keys(value).length !== 2 || !Object.hasOwn(value, "usage") || !Object.hasOwn(value, "requests")) throw invalid(); - const { usage, requests } = value as { usage: unknown; requests: unknown }; + if (value === null || typeof value !== "object" || Array.isArray(value) || !Object.hasOwn(value, "usage") || !Object.hasOwn(value, "requests")) throw invalid(); + const sealed = Object.hasOwn(value, "seal"); + if (Object.keys(value).length !== (sealed ? 3 : 2)) throw invalid(); + const { usage, requests, seal } = value as { usage: unknown; requests: unknown; seal?: unknown }; try { if (usage !== null) { if (typeof usage !== "string" || !usage.endsWith("\n") || Buffer.byteLength(usage) > MAX_USAGE_LINE_BYTES) throw invalid(); @@ -51,21 +69,32 @@ export function parseBrokerTurnLedgerLines(value: unknown, turnId: string): Brok } if (typeof requests !== "string" || (requests.length > 0 && (usage === null || !requests.endsWith("\n"))) || Buffer.byteLength(requests) > MAX_REQUEST_LINES_BYTES) throw invalid(); if (rows(requests).some((row) => row.v !== TURN_REQUEST_LEDGER_VERSION || row.turn !== turnId)) throw invalid(); + if (sealed) { + if (typeof seal !== "string" || !seal.endsWith("\n") || Buffer.byteLength(seal) > TURN_SEAL_MAX_LINE_BYTES) throw invalid(); + const parsed = rows(seal); + if (parsed.length !== 1 || parsed[0]!.v !== TURN_SEAL_LEDGER_VERSION || parsed[0]!.turn !== turnId) throw invalid(); + } } catch { throw invalid(); } - return { usage: usage as string | null, requests }; + return { usage: usage as string | null, requests, ...(sealed ? { seal: seal as string } : {}) }; } +export type BrokerTurnLedgerPaths = Readonly<{ usageLedgerPath: string; requestLedgerPath: string; sealLedgerPath: string }>; + /** Appends sealed lines on the first metering: no presence scan is needed, nothing was appended before the record existed. */ -export async function appendBrokerTurnLedger(lines: BrokerTurnLedgerLines, paths: Readonly<{ usageLedgerPath: string; requestLedgerPath: string }>): Promise { +export async function appendBrokerTurnLedger(lines: BrokerTurnLedgerLines, paths: BrokerTurnLedgerPaths): Promise { if (lines.usage !== null) await recordLedgerLines(paths.usageLedgerPath, lines.usage); await recordLedgerLines(paths.requestLedgerPath, lines.requests); + // Last, and never conditional on usage: a turn with no attributable spend is + // precisely the one whose seal row is its only account of itself. + if (lines.seal !== undefined) await recordLedgerLines(paths.sealLedgerPath, lines.seal); } /** On replay: append each stream's sealed lines only when that stream (current file or its `.1`) holds no row for this turn. Never rejects. */ -export async function ensureBrokerTurnLedgered(lines: BrokerTurnLedgerLines, turnId: string, paths: Readonly<{ usageLedgerPath: string; requestLedgerPath: string }>): Promise { +export async function ensureBrokerTurnLedgered(lines: BrokerTurnLedgerLines, turnId: string, paths: BrokerTurnLedgerPaths): Promise { try { if (lines.usage !== null && !await ledgerHasTurn(paths.usageLedgerPath, turnId)) await recordLedgerLines(paths.usageLedgerPath, lines.usage); if (lines.requests.length > 0 && !await ledgerHasTurn(paths.requestLedgerPath, turnId)) await recordLedgerLines(paths.requestLedgerPath, lines.requests); + if (lines.seal !== undefined && !await ledgerHasTurn(paths.sealLedgerPath, turnId)) await recordLedgerLines(paths.sealLedgerPath, lines.seal); } catch { /* advisory: a replay never fails on its ledger */ } } diff --git a/src/runtime/grokEngineBrokerMetering.ts b/src/runtime/grokEngineBrokerMetering.ts index 8b8fb31..d0df740 100644 --- a/src/runtime/grokEngineBrokerMetering.ts +++ b/src/runtime/grokEngineBrokerMetering.ts @@ -7,6 +7,8 @@ import type { TurnUsageFailureReason } from "./turnUsageLedger.js"; export type BrokerTurnMetering = Readonly<{ usageLedgerPath: string; requestLedgerPath: string; + /** Where the sealed response's operator-visible projection is appended (`engineBrokerSealLedger.ts`). */ + sealLedgerPath: string; agentId: string; wakeId: string; }>; @@ -31,8 +33,9 @@ export type BrokerTurnMeteringDetail = Readonly<{ notionalUsd: number; complete: * * Both terminal kinds meter: a failed turn spent real tokens, so its partial * usage is written with `outcome: failed` and its closed `limitReason`. A turn - * with no usage at all (`usage: null`) writes nothing — a zero row is - * byte-identical to a measured zero. + * with no usage at all (`usage: null`) writes no *usage* row — a zero row is + * byte-identical to a measured zero — but it still writes its seal row, which + * is a record of what the turn did rather than of what it spent. * * Appends never reject, so an append failure cannot escape into the caller's * `catch` and rewrite a completed turn as failed; the caller also refuses to diff --git a/src/runtime/grokEngineBrokerTurn.ts b/src/runtime/grokEngineBrokerTurn.ts index eef3a88..61dfaa1 100644 --- a/src/runtime/grokEngineBrokerTurn.ts +++ b/src/runtime/grokEngineBrokerTurn.ts @@ -7,7 +7,7 @@ import type { EngineBrokerMcpCallObservation } from "./engineBrokerMcpCallLog.js import { lowerEngineBrokerTurnLimits, mapGrokReportedModel, type EngineBrokerTurnAccounting, type EngineBrokerTurnLimitOverrides, type EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; import { NativeBrokerTurnFailure, type NativeBrokerDiagnostic, type NativeBrokerTurn, type NativeBrokerTurnResult } from "./engineBrokerNativeClient.js"; import type { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; -import { engineBrokerRequestLedgerPathFor, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +import { engineBrokerRequestLedgerPathFor, engineBrokerSealLedgerPathFor, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; import { ensureBrokerTurnLedgered } from "./grokEngineBrokerLedger.js"; import { finishBrokerTurnWithUsage, type BrokerTurnMetering, type BrokerTurnMeteringDetail } from "./grokEngineBrokerMetering.js"; import type { GrokBrokerProxyTurn } from "./grokBrokerProxy.js"; @@ -53,7 +53,7 @@ export async function runGrokEngineBrokerTurn(deps: GrokEngineBrokerTurnDependen const turnId = createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"); const request = { version: ENGINE_BROKER_VERSION, kind: "start_turn", requestId: randomUUID(), turnId, agentId, wakeId, prompt, mcpEndpoint, ...(overrides === undefined ? {} : { limits: overrides }) } as const; const begun = await deps.turns.begin(request, declared); - const metering: BrokerTurnMetering = { usageLedgerPath: registration.usageLedgerPath, requestLedgerPath: engineBrokerRequestLedgerPathFor(registration.usageLedgerPath), agentId, wakeId }; + const metering: BrokerTurnMetering = { usageLedgerPath: registration.usageLedgerPath, requestLedgerPath: engineBrokerRequestLedgerPathFor(registration.usageLedgerPath), sealLedgerPath: engineBrokerSealLedgerPathFor(registration.usageLedgerPath), agentId, wakeId }; if (begun !== "start") { await ensureBrokerTurnLedgered(begun.ledger, turnId, metering); return replay(begun.replay); } const controller = new AbortController(); const meter = new GrokBrokerTurnMeter(limits, () => controller.abort()); diff --git a/src/runtime/index.ts b/src/runtime/index.ts index b3f2ed1..e16c59c 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -17,7 +17,8 @@ export { GROK_INFERENCE_AUTH_STALE_BODY } from "./grokInferenceProxy.js"; export { assertGrokWorkerDenyPathPlacement, assertGrokWorkerDenyPathShape, assertGrokWorkerDenyPathsPlaceable, GROK_WORKER_BASE_PROFILE_GRANTS, grokWorkerCanSearch, grokWorkerDenyPathChain, GrokWorkerDenyPlacementError, readGrokWorkerDenyPathChain } from "./grokWorkerDenyPlacement.js"; export type { GrokWorkerDenyPathEntry, GrokWorkerDenyPathStep, GrokWorkerDenyPathWorker } from "./grokWorkerDenyPlacement.js"; export { grokWorkerSandboxProfileSha256, renderGrokWorkerSandboxProfile } from "./grokWorkerSandboxProfile.js"; -export { engineBrokerRequestLedgerPathFor, parseEngineBrokerServiceConfig, type EngineBrokerServiceConfig, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +export { engineBrokerRequestLedgerPathFor, engineBrokerSealLedgerPathFor, parseEngineBrokerServiceConfig, type EngineBrokerServiceConfig, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +export { TURN_SEAL_LEDGER, TURN_SEAL_LEDGER_VERSION } from "./engineBrokerSealLedger.js"; export { DEFAULT_GROK_BROKER_TURN_LIMITS, ENGINE_BROKER_LIMIT_REASONS, type EngineBrokerLimitReason, type EngineBrokerTurnAccounting, type EngineBrokerTurnLimits, type EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; export { dedupeTurnUsageRows } from "./turnUsageLedger.js"; From 0fe368a0a5d5ffe362c7e61cab34183af712d498 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 12:46:38 +0200 Subject: [PATCH 112/124] test: prove a worker still blocked in write past the launcher's output bound is stopped, not parked --- ...ngineBrokerLauncherIntegrationLauncher.inc | 37 +++++++++++++++++++ src/runtime/native/fixtureWorker.c | 6 ++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc index 2350808..5fb629b 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc @@ -201,6 +201,42 @@ static void worker_spill_case(void) { close(p); close(c); } +/* The bound is the WHOLE TURN's stdout, and a worker can cross it while it is + still working and still writing. `output_boundary_case` writes one byte past + the bound and then exits on its own account, so it proves the arithmetic and + not the termination: the worker was already gone when the trip fired. This + one straddles that boundary from the other side. The fixture writes eight + times the bound in ordinary frame-sized writes and then sleeps far longer + than this suite, so it fills its own pipe repeatedly and is asleep inside + `write()` when the trip happens, and NOTHING but the launcher can end it. + A launcher that stopped reading without killing — or killed without + answering — parks the worker and this client forever, which is why the + socket carries a deadline: a park has to fail red here, not hang the runner. + The deadline also bounds the answer: it must arrive while the fixture is + still sleeping, so a pass cannot be the fixture exiting by itself. */ +static void worker_stream_case(void) { + int s = connect_socket(), p = sealed("prompt"), + c = sealed_bundle("stream-output", "mcp.Stream-2"); + struct dbl_request q = request(); + struct dbl_result r; + struct timeval deadline = {.tv_sec = 30, .tv_usec = 0}; + char extra; + check(!setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &deadline, sizeof(deadline)), + "worker stream deadline"); + send_request(s, &q, p, c); + check(read_all(s, &r, sizeof(r)) && result_padding_zero(&r), + "worker stream answered before the deadline"); + check(r.status == DBL_STATUS_OUTPUT_FAILED && r.stage == DBL_STAGE_OUTPUT && + r.failure_class == DBL_FAILURE_OUTPUT_LIMIT && + r.output_length == 0 && r.diagnostic_length == 0 && + r.worker_pid > 0 && r.worker_uid == 2200 && r.start_ticks > 0 && + r.term_signal == SIGKILL, + "worker stream stopped at the bound"); + check(read(s, &extra, 1) == 0, "worker stream EOF"); + close(s); + close(p); + close(c); +} static void org_cases(void) { check(!setgid(DBL_BROKER_UID) && !setuid(DBL_BROKER_UID), "drop broker uid"); client_case(); @@ -211,6 +247,7 @@ static void org_cases(void) { worker_failure_case(); worker_flood_case(); worker_spill_case(); + worker_stream_case(); int s = connect_socket(), p = sealed("prompt"), c = sealed_bundle("provider.Token-1", "mcp.Token-2"); struct dbl_request q = request(); diff --git a/src/runtime/native/fixtureWorker.c b/src/runtime/native/fixtureWorker.c index 551bea5..19c211d 100644 --- a/src/runtime/native/fixtureWorker.c +++ b/src/runtime/native/fixtureWorker.c @@ -7,4 +7,8 @@ #include #include #include "engineBrokerLauncher.h" -int main(int argc,char**argv){unsigned char bundle[8200]={0};ssize_t size=pread(4,bundle,sizeof(bundle),0);if(size<6)return 21;size_t offset=0;unsigned provider_length=bundle[0]|(bundle[1]<<8);offset=2;if(offset+provider_length+2>(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output"),spill=!strcmp(provider,"spill-output");if(!strcmp(provider,"sleep"))sleep(30);if(!strcmp(provider,"stderr-failure")){fprintf(stderr,"grok: session store unwritable\n");return 1;}if(!strcmp(provider,"stderr-flood")){fprintf(stderr,"HEAD-OF-ERROR grok: profile refused ");for(size_t i=0;i<4096;i++)if(fputc(0x6d,stderr)==EOF)return 32;fprintf(stderr," TAIL-OF-ECHO");fflush(stderr);return 1;}if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;iDBL_MAX_OUTPUT)return 34;char*blob=malloc(count);if(!blob)return 35;memset(blob,'s',count);int head=snprintf(blob,count,"SPILL cap=%d count=%zu\n",cap,count);if(head<=0||(size_t)head+10u>=count)return 36;blob[head]='s';memcpy(blob+count-10,"SPILL-TAIL",10);if(write(1,blob,count)!=(ssize_t)count)return 37;free(blob);return 0;}char prompt_bytes[128]={0};FILE*prompt_file=fopen("/proc/self/fd/3","r");if(!prompt_file)return 26;size_t prompt_read=fread(prompt_bytes,1,sizeof(prompt_bytes)-1,prompt_file);fclose(prompt_file);if(!prompt_read)return 27;char fds[128]={0};size_t fds_used=0;DIR*fd_dir=opendir("/proc/self/fd");if(!fd_dir)return 29;struct dirent*fd_entry;int fd_seen[64]={0};while((fd_entry=readdir(fd_dir))){if(fd_entry->d_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s tmpdir=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target,getenv("TMPDIR")?getenv("TMPDIR"):"unset");for(int i=0;i(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output"),spill=!strcmp(provider,"spill-output");int stream=!strcmp(provider,"stream-output");if(!strcmp(provider,"sleep"))sleep(30);if(!strcmp(provider,"stderr-failure")){fprintf(stderr,"grok: session store unwritable\n");return 1;}if(!strcmp(provider,"stderr-flood")){fprintf(stderr,"HEAD-OF-ERROR grok: profile refused ");for(size_t i=0;i<4096;i++)if(fputc(0x6d,stderr)==EOF)return 32;fprintf(stderr," TAIL-OF-ECHO");fflush(stderr);return 1;}if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;iDBL_MAX_OUTPUT)return 34;char*blob=malloc(count);if(!blob)return 35;memset(blob,'s',count);int head=snprintf(blob,count,"SPILL cap=%d count=%zu\n",cap,count);if(head<=0||(size_t)head+10u>=count)return 36;blob[head]='s';memcpy(blob+count-10,"SPILL-TAIL",10);if(write(1,blob,count)!=(ssize_t)count)return 37;free(blob);return 0;}if(stream){char chunk[4096];memset(chunk,'t',sizeof(chunk));for(size_t i=0;id_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s tmpdir=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target,getenv("TMPDIR")?getenv("TMPDIR"):"unset");for(int i=0;i Date: Fri, 18 Sep 2026 12:47:40 +0200 Subject: [PATCH 113/124] fix: raise the launcher's whole-turn output bound to the control protocol's own text bound --- src/contracts/runtimeContractManifest.ts | 8 ++-- src/runtime/engineBrokerNativeClient.ts | 3 +- src/runtime/native/AGENTS.md | 42 +++++++++++++----- .../artifacts/daimon-engine-broker-arm64 | Bin 67456 -> 67456 bytes ...daimon-engine-broker-arm64.provenance.json | 2 +- .../native/artifacts/daimon-engine-broker-x64 | Bin 55392 -> 55392 bytes .../daimon-engine-broker-x64.provenance.json | 2 +- src/runtime/native/engineBrokerLauncher.h | 13 +++++- ...ngineBrokerLauncherIntegrationLauncher.inc | 39 ++++++++++++++++ src/runtime/native/fixtureWorker.c | 11 ++++- 10 files changed, 101 insertions(+), 19 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 3df8e89..95b0dfc 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -80,7 +80,7 @@ export const GROK_ENGINE_BROKER = { spillDirectory: { relativeToRuntimeHome: "tool-output", owner: "organization", group: "worker", mode: 0o2750, fileMode: 0o640 } } }, - bounds: { promptBytes: 65_536, capabilityBytes: 4_096, capabilityBundleBytes: 8_196, outputBytes: 65_536 }, + bounds: { promptBytes: 65_536, capabilityBytes: 4_096, capabilityBundleBytes: 8_196, outputBytes: 262_144 }, // Accounting and limits (P2). The broker is the single sealed usage writer. controlProtocolVersion: "noopolis.daimon.engine-broker.v2", turnRecordVersions: ["noopolis.daimon.engine-broker-turn.v1", "noopolis.daimon.engine-broker-turn.v2"], @@ -133,9 +133,9 @@ export const GROK_ENGINE_BROKER = { projectionVersion: "noopolis.daimon.grok-broker-projection.v1", slotPreflightVersion: "noopolis.daimon.grok-slot-preflight.v2", artifacts: { - sourceSha256: "d8c9640a2d0084f584721d0d4afdc524af7434e9461c1fc2f8adcdff6454ba6d", - x64Sha256: "4059ec576065130e857cc937b03b97a0c45fffab7d790fb153fcb170bfd0f310", - arm64Sha256: "eb0e2975cf3204bb80d25edc1fec00dbd623a95466176f3f36e7e83178deb8d2" + sourceSha256: "dd39aacfece496cc6528f6acdb4f1066a848a0fb5b0961f5c70b0ba00440dc24", + x64Sha256: "67e3624d3198e9c59e1ffafa4eca7c895dfe265d5b8bb0614cb547b68b8b93a7", + arm64Sha256: "c07d22225ff968bc289e5ddf0981cdd5d64040eee6e3ea45da7d03e1439dea98" } } as const; export const AGY_SUBSCRIPTION_REALM = { diff --git a/src/runtime/engineBrokerNativeClient.ts b/src/runtime/engineBrokerNativeClient.ts index de6406a..6a612e1 100644 --- a/src/runtime/engineBrokerNativeClient.ts +++ b/src/runtime/engineBrokerNativeClient.ts @@ -7,7 +7,8 @@ export const ENGINE_BROKER_NATIVE_REQUEST_BYTES = 396; export const ENGINE_BROKER_NATIVE_RESULT_BYTES = 128; /** `DBL_MAX_DIAGNOSTIC`: the launcher's bounded tail of a failed worker's own merged stdout/stderr. */ export const ENGINE_BROKER_NATIVE_DIAGNOSTIC_BYTES = 512; -const MAX_PROMPT = 65_536, MAX_CAPABILITY = 4_096, MAX_OUTPUT = 65_536; +/** `DBL_MAX_OUTPUT`: the launcher's bound on a whole turn's stdout, and the control protocol's own `text` bound, which is the next boundary this output crosses. */ +const MAX_PROMPT = 65_536, MAX_CAPABILITY = 4_096, MAX_OUTPUT = 262_144; const statuses = ["ok", "prelaunch_failed", "worker_failed", "output_failed", "cancelled"] as const; const stages = ["none", "peer", "request", "registration", "executable", "exec", "wait", "output", "attestation"] as const; const failures = ["none", "peer", "protocol", "registration", "executable", "exec", "wait", "output_limit", "cancelled", "profile_missing", "profile_invalid"] as const; diff --git a/src/runtime/native/AGENTS.md b/src/runtime/native/AGENTS.md index 86d0c78..be65878 100644 --- a/src/runtime/native/AGENTS.md +++ b/src/runtime/native/AGENTS.md @@ -99,16 +99,38 @@ adversarial cases that do cross the bound (`output_boundary_case`, add a test that claims to cover it by feeding the bound through the poll loop: that routes around the defect. -The bound is the whole turn's stdout, not one frame. A live single-tool-call -brokered turn already emitted 26,486 bytes, 23,320 of them one tool-result -frame (`.runtime/grok-p1b/worker-a2-output.jsonl`), against a `--max-turns` of -48 and a 16 KiB tool-result spill bound, so 64 KiB is reachable by an ordinary -working turn rather than only by a runaway one. **Known limit, deliberately not -raised yet:** the same day changed the pipe's blocking mode, and raising the -bound alongside it would mix two variables in the next live run; the measured -ceiling is 26 KB against 64 KiB, so it is not the thing in the way. Revisit -once a trial scores, and decide the number on the spread of real turns rather -than on headroom-by-guess. +The bound is the whole turn's stdout, not one frame, and it is now 256 KiB. +**This supersedes the "known limit, deliberately not raised yet" this file +carried while the pipe's blocking mode was the variable under test.** The +measurement that decided it: a live brokered turn emitted 26,482 bytes for +four tool calls, 23,320 of them one tool-result frame carrying all four +(`.runtime/grok-p1b/worker-a2-output.jsonl`); JSON framing and escaping +inflated those payloads by 1.007x, so a turn's stdout is close to the sum of +its tool results. The nine-tool-call turn this was raised for is about 210 KB +of the same shape, against a 64 KiB bound — so 64 KiB was reachable by an +ordinary working turn, and crossing it costs that turn its whole text. The new +number is not headroom-by-guess: it is the control protocol's own `text` bound +(`engineBrokerProtocol.ts`, 262144), the next boundary this output has to +cross, so a larger launcher bound would only move the refusal one layer up. +`worker_turn_case` writes exactly that measured shape — a 9,728-byte init +frame and nine 23,320-byte frames, 219,608 bytes — and asserts it is published +whole; restoring 65536 turns it red. + +**A bound that hangs would be worse than no bound, and this one does not.** +The hypothesis that a worker parks forever in `write()` once the bound is +crossed — plausible after the pipe became blocking, because a write that +cannot complete now blocks instead of erroring — was tested, not reasoned +about, and it is false. `worker_stream_case` writes eight times the bound in +frame-sized writes and then sleeps far longer than this suite, so it is asleep +inside `write()` with its pipe full when the trip fires and nothing but the +launcher can end it; the launcher answers `output_limit` with `term_signal` +SIGKILL in seconds. Its socket carries a 30-second deadline so a park fails +red instead of parking the runner. Deleting the `output_limited` half of +`if (disconnected || output_limited) kill(-pid, SIGKILL)` is the mutation that +proves it: the case then times out on that deadline, and +`output_boundary_case` does not notice, because its worker has already exited +by the time the trip fires. That is the boundary the two cases straddle — +a worker gone at the trip against a worker alive and blocked at it. The result frame's last word is `diagnostic_length`, not padding: on `DBL_STATUS_WORKER_FAILED` the supervisor keeps the last `DBL_MAX_DIAGNOSTIC` diff --git a/src/runtime/native/artifacts/daimon-engine-broker-arm64 b/src/runtime/native/artifacts/daimon-engine-broker-arm64 index 2dbe88d27e03d69ea06aa08648efa8f8bc54e551..7d34af45d6e98df65d0375adda12a2c361467705 100755 GIT binary patch delta 2778 zcmY*bdr*|u6~Fg>yU1f8;POz=?Beorts<2L6qQ}xhMHL1Y3evhpg3xRNrR|JFtZ<~ ziP`c;TrY~^Ynl8}cPml5&~!7^V00WCVyf#SnkLP+VyDUUF;Y<>2)DnxVkf;bcg{J# zbM86ko^!rCpZcCoeNVUha+JD*gqM2imU%W!@K4NsUk}n_Ud`xt{`;BNbS%cs2Q7BG zlgG~UNa4i1Z6N;aAN`&&V)uWuAR?kzv0s^JBi;#xxbLl|-jN2yZ6IoJtWoD3Q_klt!l&G~28x-%TqWHMUn!{2Eue4g~|1m1M2FPeZ)0pTY#1=5dlcqz{ zj;8iU98J}XXIo9tq%f3gZ#C*>Zyo~PK$0vy!0t1}vNe`iQ{XqGr1>-@S;nkwgP(60 zY19aD3_KDAYxxyx+8Pg{IGKG|W282;>u72TA+MN{iK75n?n9QJFvDOwGjwRmOiLa- z7zr7kpF|wZjC9Bd^6Yn$!Xb3zDQ%Z6KxdH6#&Z#mLfl?pN>S`?6^C!ESKLOEa{bBX zn+j&pt=v_R&lEfVxL~HeA8BZ~%m5<=X1FyWybP-%1x5^mGW<|83EYF7OrZ|phnj5x z!vOD=DMaz4!fbsNwEMZe(9FJgz}FX6(z(31&`1mUr9v}Zz=sQS=?$J^rNG{V-Q&t6 z`j|}(8nFA|_hsH&)a+zPWe(XHl-<>EY7&|!r!^(jl0S*=E+$s(9CcIbsm+4D4f1)| zJy3`RmuzZWLL4-UI5b>T`b($b!qm1%+j+$LH`p3=q)D^DPTm091YMnlFzW^0v)JhT z2qv16V^6Jue zsDzBkBHA-ZUucNB8C8RW<0ylfSVIbP%O(hCq<1sywME2=D+O_-puBn=WL-FpTFXGw zu?fmC4?6Y7#Cki<7Jv_nhW#`!Ee}|97>f&odVn^79sKJhDJy3}@+>4)>E?%S2Xt!f zbm)+-MW))GORS8kwP#he_9oJjo$=(37Rqwssklo_OWdzLFV*to z)X*}&Wk%jj?6nxRr z3TpGWE{&n|PyFp=W;(#V%gnPre1I`Ku4{^lC63JB>Er(i-oRY$-*n7xUib#p%C> zzbFi-wA%?Z)G)VW7?uy^fWlWdk1A=Q)x5nVg_iOIB{j3Ap%)R@e>cUtFVw}l@02n3 z&|8X|5psEtErVX=nYLX~c`_RE%;o*Ik`>dYvX<4*P4V?I56L5^9`KVG&+r_89|u3t zcE(FhYB2n^$B?mm$f(!9%xsz`noFKP}B#b__4&79zWoXgPczKH6*n8yhXy z7RHh-`z7WC26-=d5kc}$A%~`y;63HfQNoAI z4fH72SCr8g`HG5?=xrl7kGUBg;d?5U({K5eibd1A;nXxC@U);2sCU)FF ztuoUW_{u6{Qakixjpg3)<_hnZlOjddY6owxT13luf7SD}m&aCbpsV@T>MGjKPgmzs ztN&ls&6L`B{WC8m~q|c!ytkrkGymu{9-B&sWx@(KGzzni{&6_tzNd z`~K@SW||OQ|p#Xr%x4pL$h9MCmYNub&X2lLxkJaAx4qJNZW; zmV#!S7GfRfyP(@ab3PVg0Ms@hM5GS%ybxPJk9{V@MNrQrA^r(kepQHf^@M2Ggizsq z;dOWiRY8w|YBz*<1bPQFUQRh^2B_zzm@ZagZcvesS~qjjPwb{;NGs9f-Hb5qnJYA+L)a^tl*6((<%E9v|2`ezwJ+&jGiI-)b?> zTm|kA0h9xggT)#Edw M>BSH0_0-7z3w>l!q5uE@ delta 2755 zcmY*b3s98T6~6cXcL@(ck+)!XaRKpHie~_dq|c&Dd2}!>DO;-}Gs3-#T!J{KOCtFu$k;o)wJ1#~NjQ6>k@$vT zBhsir3JHF;|3;(g6KS~FRLewt6|^1f1&c{qbf0au%%^U4#$uqw>eRcF|SlZp91ur=sNgG(U}u@Aw@ zfzGH9$$XCaW*b`kFbT>@9 zNC9zlBRe`LrD_H^btW-ik5hf{P=Z)lz+5wcMSp|E03*Phh^`A~r&UX!;Ub}k2Iy;8 zd|^siDmdh(h`dUs5wj#|)u%MA`WkW&ZK#(a-?Z5)_Kz= zId5#0of7)OjA|DBl$EL5qGpTjkf??AsaE^wk+emTMvd^xz()xlNr_@bjNq-{W$=$S z{mD&@S|I#($B^N>Na>{ahb1OT|G-*H>m+HA{kt@$xC?LLIwG2osQh*S@iwVGRT|1) zAI6dum%v(pLGA$0BghPt$PtuNlzptqW}r@1Z>yu6_1V}9@rYIvL-K+-)R`Hn1!5rw zqaiDjGjogFhWwXYPV*@p6_{o}TnaXp9mxsEb5et3`dU zF?+d*E@k!QhNSJ#53ic%e$ZIvzVeu-h+4hII?HF#O4e7th@NCqE9&TrYYmRZ=%zW$WT2k?} zb-l;=bkMB#IbRI=3((b|xqs&T1gPrg94`XN?dE)m4*GMPcY$i>IqwC{y2$w&MD_P_ zz9NEY{faw|Hd?Dx^&=ua}FEjdsHF%PC zP(s#u5gR_FB+eC68s|Ezvvr)XAM)Bu+1vJA$L+S=UG_c4>}Bo0v+q4>-*?2m|1G;z z5&g-Pe_Zanbg}p9wURi^uPccoB{7W$-Sz_>JM%PsXS@zKYmJX{BO)WTu|s!A&H|eO zd;|NU$&@w^TsJI!A}sLD!&(e1k!5bM>G1VBr$4(wD9I#ubX(Vu?yt(3JVDI?rM ztO~^KMl)~|Hs)olrQfm^M>E3cAdK)TZ8Yfp;C{n)Z*13#ZXIDWUTM-_2Dh3WeWhut z1D#sin&uw~kY7e;Hk2(GWJ^6&kO<>)?B1f=X&>>*& zvGbcIj?Y5J)C=Eu2%|)-BH$ZXd~-sY1Dw{+d6rP&n};Q4=eD=FIi`gLX{xv4L25MZ L`u#FJHAw#h^mJ3@ diff --git a/src/runtime/native/artifacts/daimon-engine-broker-arm64.provenance.json b/src/runtime/native/artifacts/daimon-engine-broker-arm64.provenance.json index 8b9496e..a694add 100644 --- a/src/runtime/native/artifacts/daimon-engine-broker-arm64.provenance.json +++ b/src/runtime/native/artifacts/daimon-engine-broker-arm64.provenance.json @@ -1 +1 @@ -{"version":"daimon.engine-broker-native-build.v1","architecture":"arm64","target":"linux/arm64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:d8c9640a2d0084f584721d0d4afdc524af7434e9461c1fc2f8adcdff6454ba6d","binary_sha256":"sha256:eb0e2975cf3204bb80d25edc1fec00dbd623a95466176f3f36e7e83178deb8d2","install_path":"/opt/daimon/bin/daimon-engine-broker"} +{"version":"daimon.engine-broker-native-build.v1","architecture":"arm64","target":"linux/arm64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:dd39aacfece496cc6528f6acdb4f1066a848a0fb5b0961f5c70b0ba00440dc24","binary_sha256":"sha256:c07d22225ff968bc289e5ddf0981cdd5d64040eee6e3ea45da7d03e1439dea98","install_path":"/opt/daimon/bin/daimon-engine-broker"} diff --git a/src/runtime/native/artifacts/daimon-engine-broker-x64 b/src/runtime/native/artifacts/daimon-engine-broker-x64 index 3e42c7286b0330f78f6a9ef82948337c03c7dd50..f7c022d10d401f5dbe41751e5d41160afcd6c66c 100755 GIT binary patch delta 267 zcmaE`f%(A(<_&H7j4YGe^(}$qLH+eWvd|!gk!ABa0|iD#g~{&?Ll`9{yBOUN74Yfw zQQ>&e`QiV6pKc(_r`ttEV6vgHhNi~nr7oSAz8CNoDO#WkR$tW<{%w(M)3jJ!vLxC!8o2~-7 zpv_E^QDgH4vvx*Cfz1{c$C(*fCfSGq-6?MKR)B?pznzPjfx)NqgX11X2A0Vpw#Gmy t58D7BIn&k~NItb~1Clv*sz6UovOCTQq@o;NRsjuj*)0t;b8*5R82~rMS1bSk delta 267 zcmaE`f%(A(<_&H7jEs}p^(}$qLH+eWvd|!gk#X}m0|iD#jmhr~Ll^}nyBOUNmGJ5G zQQ>&e`QiV6pKc(_r`ttEVzQyJhNi;jr7oSJ!vLxC!8o2~-7 zpv_E^QDO52vvx*CiOm)k$C()!C)tPr-6?MKR)CR#znzPjfx)NqgX11X2FA%Ew#Gmy v58D7BIn&k~NItb~1Clv*sz6UovOCTQq@o;NRxz?{cG)e>$jGudVUG*|FYH$L diff --git a/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json b/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json index d973815..4dba2b9 100644 --- a/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json +++ b/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json @@ -1 +1 @@ -{"version":"daimon.engine-broker-native-build.v1","architecture":"x64","target":"linux/amd64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:d8c9640a2d0084f584721d0d4afdc524af7434e9461c1fc2f8adcdff6454ba6d","binary_sha256":"sha256:4059ec576065130e857cc937b03b97a0c45fffab7d790fb153fcb170bfd0f310","install_path":"/opt/daimon/bin/daimon-engine-broker"} +{"version":"daimon.engine-broker-native-build.v1","architecture":"x64","target":"linux/amd64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:dd39aacfece496cc6528f6acdb4f1066a848a0fb5b0961f5c70b0ba00440dc24","binary_sha256":"sha256:67e3624d3198e9c59e1ffafa4eca7c895dfe265d5b8bb0614cb547b68b8b93a7","install_path":"/opt/daimon/bin/daimon-engine-broker"} diff --git a/src/runtime/native/engineBrokerLauncher.h b/src/runtime/native/engineBrokerLauncher.h index 636e484..7542b51 100644 --- a/src/runtime/native/engineBrokerLauncher.h +++ b/src/runtime/native/engineBrokerLauncher.h @@ -8,7 +8,18 @@ #define DBL_MAX_PROMPT 65536u #define DBL_MAX_TOKEN 4096u #define DBL_MAX_CAPABILITY_BUNDLE (DBL_MAX_TOKEN * 2u + 4u) -#define DBL_MAX_OUTPUT 65536u +/* The WHOLE turn's stdout, not one frame, and sized against real turns rather + than headroom-by-guess. A live four-tool-call brokered turn emitted 26,482 + bytes, 23,320 of them one tool-result frame carrying four results + (`.runtime/grok-p1b/worker-a2-output.jsonl`), so 64 KiB was reachable by an + ordinary working turn: the nine-tool-call turn this was raised for lands + around 210 KB of the same shape, and a trip costs the turn its whole text. + The number is the control protocol's own `text` bound + (`engineBrokerProtocol.ts`, 262144), because that is the next boundary the + output must cross: a larger launcher bound would only move the refusal one + layer up. A runaway worker is still stopped here — crossing it stops + reading, SIGKILLs the worker's process group and reports `output_limit`. */ +#define DBL_MAX_OUTPUT 262144u /* Bounded tail of the worker's own merged stdout/stderr, kept only for a worker that exited on its own account (`DBL_STATUS_WORKER_FAILED`), so the reason reaches the host instead of `exit=1`. It is a diagnostic, never the diff --git a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc index 5fb629b..7b4bdc8 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc @@ -201,6 +201,44 @@ static void worker_spill_case(void) { close(p); close(c); } +/* An ordinary working turn must not be able to hit the bound, and its output + must arrive whole. The fixture writes the shape one live turn measured — a + 9,728-byte init frame and nine 23,320-byte tool-result frames, 219,608 + bytes in all — in frame-sized writes. That is more than three times the + 64 KiB this bound used to be, so at the old value this same turn was + published as `output_limit` with its whole text discarded; it is the case + that straddles the raise, and restoring 65536 turns it red. */ +static void worker_turn_case(void) { + int s = connect_socket(), p = sealed("prompt"), + c = sealed_bundle("turn-output", "mcp.Turn-2"); + struct dbl_request q = request(); + struct dbl_result r; + const uint32_t expected = 9728u + 23320u * 9u; + char *out, extra; + size_t index; + send_request(s, &q, p, c); + check(read_all(s, &r, sizeof(r)) && result_padding_zero(&r) && + r.status == DBL_STATUS_OK && r.stage == DBL_STAGE_OUTPUT && + r.failure_class == DBL_FAILURE_NONE && r.exit_code == 0 && + r.term_signal == 0 && r.diagnostic_length == 0 && + expected > 65536u && + r.output_length == expected, + "ordinary turn published whole"); + out = calloc(1, (size_t)expected + 1u); + check(out && read_all(s, out, expected), "ordinary turn output"); + check(!memcmp(out, "TURN-HEAD", 9) && + !memcmp(out + expected - 9, "TURN-TAIL", 9), + "ordinary turn ends intact"); + for (index = 9; index < (size_t)expected - 9u; index++) + if (out[index] != 'u') + break; + check(index == (size_t)expected - 9u, "ordinary turn bytes intact"); + check(read(s, &extra, 1) == 0, "ordinary turn EOF"); + free(out); + close(s); + close(p); + close(c); +} /* The bound is the WHOLE TURN's stdout, and a worker can cross it while it is still working and still writing. `output_boundary_case` writes one byte past the bound and then exits on its own account, so it proves the arithmetic and @@ -247,6 +285,7 @@ static void org_cases(void) { worker_failure_case(); worker_flood_case(); worker_spill_case(); + worker_turn_case(); worker_stream_case(); int s = connect_socket(), p = sealed("prompt"), c = sealed_bundle("provider.Token-1", "mcp.Token-2"); diff --git a/src/runtime/native/fixtureWorker.c b/src/runtime/native/fixtureWorker.c index 19c211d..14fd55a 100644 --- a/src/runtime/native/fixtureWorker.c +++ b/src/runtime/native/fixtureWorker.c @@ -10,5 +10,14 @@ /* Eight times the whole-turn output bound, written in ordinary frame-sized writes and followed by a sleep the launcher must cut short: the fixture never exits on its own account, so only the bound can end this turn. */ +/* The measured shape of a real brokered turn: a 9,728-byte init frame and + nine 23,320-byte tool-result frames, the frame size one live capture + actually produced (`.runtime/grok-p1b/worker-a2-output.jsonl`). It is far + past the 64 KiB this bound used to be and well inside what it is now, so it + is the case that straddles the raise. */ +#define TURN_INIT_BYTES 9728u +#define TURN_FRAME_BYTES 23320u +#define TURN_FRAMES 9u +#define TURN_BYTES (TURN_INIT_BYTES + TURN_FRAME_BYTES * TURN_FRAMES) #define STREAM_CHUNKS ((DBL_MAX_OUTPUT / 4096u) * 8u) -int main(int argc,char**argv){unsigned char bundle[8200]={0};ssize_t size=pread(4,bundle,sizeof(bundle),0);if(size<6)return 21;size_t offset=0;unsigned provider_length=bundle[0]|(bundle[1]<<8);offset=2;if(offset+provider_length+2>(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output"),spill=!strcmp(provider,"spill-output");int stream=!strcmp(provider,"stream-output");if(!strcmp(provider,"sleep"))sleep(30);if(!strcmp(provider,"stderr-failure")){fprintf(stderr,"grok: session store unwritable\n");return 1;}if(!strcmp(provider,"stderr-flood")){fprintf(stderr,"HEAD-OF-ERROR grok: profile refused ");for(size_t i=0;i<4096;i++)if(fputc(0x6d,stderr)==EOF)return 32;fprintf(stderr," TAIL-OF-ECHO");fflush(stderr);return 1;}if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;iDBL_MAX_OUTPUT)return 34;char*blob=malloc(count);if(!blob)return 35;memset(blob,'s',count);int head=snprintf(blob,count,"SPILL cap=%d count=%zu\n",cap,count);if(head<=0||(size_t)head+10u>=count)return 36;blob[head]='s';memcpy(blob+count-10,"SPILL-TAIL",10);if(write(1,blob,count)!=(ssize_t)count)return 37;free(blob);return 0;}if(stream){char chunk[4096];memset(chunk,'t',sizeof(chunk));for(size_t i=0;id_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s tmpdir=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target,getenv("TMPDIR")?getenv("TMPDIR"):"unset");for(int i=0;i(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output"),spill=!strcmp(provider,"spill-output");int stream=!strcmp(provider,"stream-output");int turn=!strcmp(provider,"turn-output");if(!strcmp(provider,"sleep"))sleep(30);if(!strcmp(provider,"stderr-failure")){fprintf(stderr,"grok: session store unwritable\n");return 1;}if(!strcmp(provider,"stderr-flood")){fprintf(stderr,"HEAD-OF-ERROR grok: profile refused ");for(size_t i=0;i<4096;i++)if(fputc(0x6d,stderr)==EOF)return 32;fprintf(stderr," TAIL-OF-ECHO");fflush(stderr);return 1;}if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;iDBL_MAX_OUTPUT)return 34;char*blob=malloc(count);if(!blob)return 35;memset(blob,'s',count);int head=snprintf(blob,count,"SPILL cap=%d count=%zu\n",cap,count);if(head<=0||(size_t)head+10u>=count)return 36;blob[head]='s';memcpy(blob+count-10,"SPILL-TAIL",10);if(write(1,blob,count)!=(ssize_t)count)return 37;free(blob);return 0;}if(turn){char*blob=malloc(TURN_BYTES);if(!blob)return 39;memset(blob,'u',TURN_BYTES);memcpy(blob,"TURN-HEAD",9);memcpy(blob+TURN_BYTES-9,"TURN-TAIL",9);size_t sent=0;while(sentd_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s tmpdir=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target,getenv("TMPDIR")?getenv("TMPDIR"):"unset");for(int i=0;i Date: Fri, 18 Sep 2026 13:00:25 +0200 Subject: [PATCH 114/124] feat: observe the brokered worker's standalone MCP GET tunnel --- src/runtime/engineBrokerControlClient.ts | 6 +- src/runtime/engineBrokerMcpCallLog.test.ts | 81 ++++++++++++++++++++-- src/runtime/engineBrokerMcpCallLog.ts | 65 ++++++++++++++++- src/runtime/engineBrokerMcpFacade.test.ts | 55 +++++++++++++++ src/runtime/engineBrokerMcpFacade.ts | 20 ++++-- src/runtime/engineBrokerProtocol.test.ts | 23 ++++++ src/runtime/engineBrokerProtocol.ts | 33 ++++++++- src/runtime/engineBrokerSealLedger.test.ts | 38 ++++++++++ src/runtime/engineBrokerSealLedger.ts | 21 +++++- 9 files changed, 323 insertions(+), 19 deletions(-) diff --git a/src/runtime/engineBrokerControlClient.ts b/src/runtime/engineBrokerControlClient.ts index 6744615..fd491f1 100644 --- a/src/runtime/engineBrokerControlClient.ts +++ b/src/runtime/engineBrokerControlClient.ts @@ -2,7 +2,7 @@ import { createHash, randomUUID } from "node:crypto"; import { createConnection } from "node:net"; import { ENGINE_BROKER_VERSION, encodeEngineBrokerFrame,EngineBrokerFrameDecoder,parseEngineBrokerResponse } from "./engineBrokerProtocol.js"; import type { EngineBrokerInferenceFailureCode, EngineBrokerInferenceRequest, EngineBrokerInferenceResponse } from "./engineBrokerInferenceProtocol.js"; -import type { EngineBrokerMcpCallObservation } from "./engineBrokerMcpCallLog.js"; +import type { EngineBrokerMcpCallObservation, EngineBrokerMcpTunnelObservation } from "./engineBrokerMcpCallLog.js"; import type { EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; import type { GrokBrokerModel, GrokBrokerReasoningEffort } from "./grokBrokerModelPolicy.js"; import type { GrokInferencePurpose } from "./inferenceUsageLedger.js"; @@ -14,7 +14,9 @@ import type { GrokInferencePurpose } from "./inferenceUsageLedger.js"; * long it had been waiting — the one thing a completion-only tool receipt can * never say. */ -const renderMcpCalls=(calls:EngineBrokerMcpCallObservation|undefined):string=>calls===undefined?"":`; mcp=${calls.answered}/${calls.started} answered${calls.undecoded===0?"":`; mcp_undecoded=${calls.undecoded}`}${calls.outstanding.length===0?"":`; mcp_outstanding=${calls.outstanding.map((call)=>`${call.name}@${call.outstandingMs}ms`).join(",")}`}`; +const renderMcpCalls=(calls:EngineBrokerMcpCallObservation|undefined):string=>calls===undefined?"":`; mcp=${calls.answered}/${calls.started} answered${calls.undecoded===0?"":`; mcp_undecoded=${calls.undecoded}`}${calls.outstanding.length===0?"":`; mcp_outstanding=${calls.outstanding.map((call)=>`${call.name}@${call.outstandingMs}ms`).join(",")}`}${renderMcpTunnels(calls.tunnels)}`; +/** The session's GET SSE tunnels: how many closed of how many opened, and each one still open with its age and whether the mount ever pushed through it. */ +const renderMcpTunnels=(tunnels:EngineBrokerMcpTunnelObservation|undefined):string=>tunnels===undefined?"":`; mcp_get=${tunnels.closed}/${tunnels.opened} closed, ${tunnels.delivered} delivered${tunnels.open.length===0?"":`; mcp_get_open=${tunnels.open.map((tunnel)=>`${tunnel.openMs}ms/${tunnel.delivered?"delivered":"silent"}`).join(",")}`}`; export type EngineBrokerInferenceGrant = Omit, "version" | "kind" | "requestId">; /** A refused grant request; `code` is closed (`auth_stale` is the stale shared realm, `grant_limit` the live-grant cap). */ diff --git a/src/runtime/engineBrokerMcpCallLog.test.ts b/src/runtime/engineBrokerMcpCallLog.test.ts index 8550608..7b0ab26 100644 --- a/src/runtime/engineBrokerMcpCallLog.test.ts +++ b/src/runtime/engineBrokerMcpCallLog.test.ts @@ -1,12 +1,15 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { EngineBrokerMcpCallLog, ENGINE_BROKER_MCP_CALL_INVALID, ENGINE_BROKER_MCP_CALL_TRUNCATED, ENGINE_BROKER_MCP_OUTSTANDING_MAX } from "./engineBrokerMcpCallLog.js"; +import { EngineBrokerMcpCallLog, ENGINE_BROKER_MCP_CALL_INVALID, ENGINE_BROKER_MCP_CALL_TRUNCATED, ENGINE_BROKER_MCP_OUTSTANDING_MAX, ENGINE_BROKER_MCP_TUNNEL_MAX } from "./engineBrokerMcpCallLog.js"; const body = (value: unknown): Buffer => Buffer.from(JSON.stringify(value), "utf8"); const call = (name: unknown, id = 1): Buffer => body({ jsonrpc: "2.0", id, method: "tools/call", params: { name, arguments: { text: "argument-bytes" } } }); /** A controllable clock: an outstanding call's whole value is how long it has been outstanding. */ +/** An observed turn whose facade never relayed a GET tunnel — a measurement, not an absence. */ +const NO_TUNNEL = { opened: 0, closed: 0, delivered: 0, open: [] }; + const clock = (): { log: EngineBrokerMcpCallLog; advance: (ms: number) => void } => { let at = 1_000; return { log: new EngineBrokerMcpCallLog(() => at), advance: (ms: number): void => { at += ms; } }; @@ -33,12 +36,12 @@ test("absence stays absence: an unopened turn observes undefined, a turn that ca const { log } = clock(); assert.equal(log.observe("turn"), undefined); log.open("turn"); - assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 0, outstanding: [] }); + assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 0, outstanding: [], tunnels: NO_TUNNEL }); // Everything that is not a tool call records nothing at all, so `started` // stays a count of tool calls and not of traffic. for (const value of [body({ jsonrpc: "2.0", id: 1, method: "tools/list" }), body({ jsonrpc: "2.0", method: "notifications/initialized" }), Buffer.alloc(0)]) log.begin("turn", value).answer(); log.begin("turn", undefined).answer(); - assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 0, outstanding: [] }); + assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 0, outstanding: [], tunnels: NO_TUNNEL }); log.close("turn"); assert.equal(log.observe("turn"), undefined, "a revoked turn keeps nothing"); }); @@ -48,7 +51,7 @@ test("a body the facade could not read counts as undecoded, never as a call with log.open("turn"); log.begin("turn", Buffer.from("{not json", "utf8")); log.undecodable("turn"); - assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 2, outstanding: [] }); + assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 2, outstanding: [], tunnels: NO_TUNNEL }); log.undecodable("absent"); }); @@ -71,3 +74,73 @@ test("the outstanding list is bounded, and the earliest calls are the ones kept" assert.equal(observed?.outstanding[0]?.name, "tool_0"); assert.equal(observed?.outstanding.at(-1)?.name, ENGINE_BROKER_MCP_CALL_TRUNCATED); }); + +/** + * The channel a brokered turn had no light on at all. + * + * Seven live runs closed every provider request, answered every tool call, and + * still sat idle to the deadline. The one thing none of them could say is + * whether the worker was parked on the session's standalone GET SSE tunnel, + * because the facade relayed it and recorded nothing. The boundary these + * assertions straddle is exactly that: a tunnel still open at observation + * against one that ended before it — one is a worker that may still be reading, + * the other is a channel already closed and therefore not the blocker. + * + * Mutation: drop `openTunnels.delete(record)` from `close`, and the closed + * tunnel keeps reporting itself open; drop `tunnelsOpened += 1`, and an open + * tunnel becomes indistinguishable from a turn that never opened one. + */ +test("a GET tunnel still open reports its age; one that closed first reports closed and nothing open", () => { + const { log, advance } = clock(); + log.open("turn"); + const parked = log.openTunnel("turn"); + advance(430_000); + const open = log.observe("turn")?.tunnels; + assert.deepEqual(open, { opened: 1, closed: 0, delivered: 0, open: [{ openMs: 430_000, delivered: false }] }); + + parked.close(); + advance(5_000); + assert.deepEqual(log.observe("turn")?.tunnels, { opened: 1, closed: 1, delivered: 0, open: [] }); +}); + +test("a turn that never opened a GET tunnel is not a turn that opened one, and neither is an unobserved turn", () => { + const { log, advance } = clock(); + log.open("turn"); + // Observed, and it measured zero: every count is a measurement. + assert.deepEqual(log.observe("turn")?.tunnels, { opened: 0, closed: 0, delivered: 0, open: [] }); + log.begin("turn", call("daimon__moltnet_read")).answer(); + assert.deepEqual(log.observe("turn")?.tunnels, { opened: 0, closed: 0, delivered: 0, open: [] }, "a POST is not a tunnel"); + const tunnel = log.openTunnel("turn"); + advance(1_000); + assert.equal(log.observe("turn")?.tunnels?.open.length, 1); + tunnel.close(); + // And the third state, which is not zero: a turn the facade never registered. + assert.equal(log.observe("absent"), undefined); + log.openTunnel("absent").deliver(); + assert.equal(log.observe("absent"), undefined, "an unopened turn keeps nothing"); +}); + +test("a tunnel the mount pushed through is a different fact from one held open in silence", () => { + const { log, advance } = clock(); + log.open("turn"); + const silent = log.openTunnel("turn"); + const pushing = log.openTunnel("turn"); + advance(90_000); + pushing.deliver(); pushing.deliver(); + assert.deepEqual(log.observe("turn")?.tunnels, { + opened: 2, closed: 0, delivered: 1, + open: [{ openMs: 90_000, delivered: false }, { openMs: 90_000, delivered: true }] + }); + silent.close(); silent.close(); + assert.deepEqual(log.observe("turn")?.tunnels?.closed, 1, "closing twice closes one tunnel"); +}); + +test("the open-tunnel list is bounded, and the counts still name every tunnel beyond it", () => { + const { log, advance } = clock(); + log.open("turn"); + for (let index = 0; index < ENGINE_BROKER_MCP_TUNNEL_MAX + 3; index += 1) { log.openTunnel("turn"); advance(1); } + const tunnels = log.observe("turn")?.tunnels; + assert.equal(tunnels?.opened, ENGINE_BROKER_MCP_TUNNEL_MAX + 3); + assert.equal(tunnels?.open.length, ENGINE_BROKER_MCP_TUNNEL_MAX); + assert.equal(tunnels?.open[0]?.openMs, ENGINE_BROKER_MCP_TUNNEL_MAX + 3, "the earliest tunnels are the ones kept"); +}); diff --git a/src/runtime/engineBrokerMcpCallLog.ts b/src/runtime/engineBrokerMcpCallLog.ts index fc49dbf..b680c48 100644 --- a/src/runtime/engineBrokerMcpCallLog.ts +++ b/src/runtime/engineBrokerMcpCallLog.ts @@ -30,6 +30,24 @@ * the one parse is wrapped, and the facade treats a missing handle as a * no-op. * + * The same map carries the facade's *other* channel, and for the same reason. + * A `tools/call` is a POST that answers; the standalone `GET` SSE tunnel the + * Streamable HTTP transport opens once per session is the route a server + * notification or progress frame takes, and it stays open for the whole + * session by design. A worker parked reading that tunnel is, in every artifact + * the broker writes, indistinguishable from a worker doing nothing at all: + * every provider request closed, every tool call answered, and the turn idle + * until its deadline. {@link EngineBrokerMcpCallLog.openTunnel} records the + * lifecycle — how many the facade relayed, how many ended, and for the ones + * still open at seal time how long each has been open and whether the mount + * ever pushed a single byte through it. A tunnel held open having delivered + * nothing is a different fact from one actively carrying frames, and it is the + * difference that decides whether the tunnel is the blocker. + * + * Observing is all it does. The facade's behaviour is unchanged: nothing here + * closes, times out or refuses a tunnel, because an instrument that tore down + * the stream would destroy the evidence it exists to gather. + * * "Answered" means the facade wrote a complete response back to the worker — * the relay reached its own `end()`. A relay that was torn down (the worker * died, the tunnel broke, the turn aborted) did *not* answer, so its calls stay @@ -44,27 +62,45 @@ export const ENGINE_BROKER_MCP_CALL_TRUNCATED = ""; export const ENGINE_BROKER_MCP_CALL_NAME = /^(?:||[A-Za-z0-9_.-]{1,64})$/u; const TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/u; +/** Open GET tunnels reported: a session opens one, so more than a handful is already the anomaly. */ +export const ENGINE_BROKER_MCP_TUNNEL_MAX = 8; + export type EngineBrokerOutstandingMcpCall = Readonly<{ name: string; outstandingMs: number }>; +/** One GET SSE tunnel still open at observation: how long it has been open, and whether the mount ever pushed through it. */ +export type EngineBrokerOpenMcpTunnel = Readonly<{ openMs: number; delivered: boolean }>; +/** + * What the facade saw of one turn's standalone GET SSE tunnels: how many it + * relayed, how many ended, how many ever carried a byte from the mount, and + * the ones still open with the age of each. + */ +export type EngineBrokerMcpTunnelObservation = Readonly<{ opened: number; closed: number; delivered: number; open: readonly EngineBrokerOpenMcpTunnel[] }>; /** * What the facade saw of one turn's tool calls: how many started, how many the * facade answered, how many POST bodies it could not read, and the ones still * unanswered with the time each has been outstanding. */ -export type EngineBrokerMcpCallObservation = Readonly<{ started: number; answered: number; undecoded: number; outstanding: readonly EngineBrokerOutstandingMcpCall[] }>; +export type EngineBrokerMcpCallObservation = Readonly<{ started: number; answered: number; undecoded: number; outstanding: readonly EngineBrokerOutstandingMcpCall[]; tunnels?: EngineBrokerMcpTunnelObservation }>; /** One relayed POST's calls. `answer` marks a complete relay; `close` ends it unanswered. Both are idempotent. */ export interface EngineBrokerMcpCallHandle { answer(): void; close(): void } const INERT: EngineBrokerMcpCallHandle = { answer: () => undefined, close: () => undefined }; +/** One relayed GET tunnel. `deliver` marks the first byte the mount pushed; `close` ends it. Both are idempotent. */ +export interface EngineBrokerMcpTunnelHandle { deliver(): void; close(): void } +const INERT_TUNNEL: EngineBrokerMcpTunnelHandle = { deliver: () => undefined, close: () => undefined }; type CallRecord = { readonly name: string; readonly startedAt: number; endedAt?: number }; -type TurnLog = { started: number; answered: number; undecoded: number; readonly live: Set; readonly ended: CallRecord[] }; +type TunnelRecord = { readonly openedAt: number; delivered: boolean }; +type TurnLog = { + started: number; answered: number; undecoded: number; readonly live: Set; readonly ended: CallRecord[]; + tunnelsOpened: number; tunnelsClosed: number; tunnelsDelivered: number; readonly openTunnels: Set; +}; export class EngineBrokerMcpCallLog { private readonly turns = new Map(); constructor(private readonly now: () => number = Date.now) {} /** A turn the facade registered. Re-opening an id resets it: a turn id is unique per turn. */ - open(turnId: string): void { this.turns.set(turnId, { started: 0, answered: 0, undecoded: 0, live: new Set(), ended: [] }); } + open(turnId: string): void { this.turns.set(turnId, { started: 0, answered: 0, undecoded: 0, live: new Set(), ended: [], tunnelsOpened: 0, tunnelsClosed: 0, tunnelsDelivered: 0, openTunnels: new Set() }); } close(turnId: string): void { this.turns.delete(turnId); } /** A POST body the facade is about to relay. Anything that is not a `tools/call` records nothing. */ @@ -98,6 +134,24 @@ export class EngineBrokerMcpCallLog { }; } + /** + * A GET SSE tunnel the facade is about to relay. Counted when it opens, not + * when it succeeds: a tunnel the mount refused still ends, so `opened` and + * `closed` stay a pair and an open one is exactly `opened - closed`. + */ + openTunnel(turnId: string): EngineBrokerMcpTunnelHandle { + const log = this.turns.get(turnId); + if (log === undefined) return INERT_TUNNEL; + const record: TunnelRecord = { openedAt: this.now(), delivered: false }; + log.tunnelsOpened += 1; + log.openTunnels.add(record); + let ended = false; + return { + deliver: (): void => { if (record.delivered) return; record.delivered = true; log.tunnelsDelivered += 1; }, + close: (): void => { if (ended) return; ended = true; log.tunnelsClosed += 1; log.openTunnels.delete(record); } + }; + } + /** A POST whose body the facade refused to read (over its own bound), which is a call it cannot name. */ undecodable(turnId: string): void { const log = this.turns.get(turnId); if (log !== undefined) log.undecoded += 1; } @@ -107,8 +161,13 @@ export class EngineBrokerMcpCallLog { const at = this.now(); const pending = [...log.live, ...log.ended].sort((left, right) => left.startedAt - right.startedAt); const outstanding = pending.map((record): EngineBrokerOutstandingMcpCall => ({ name: record.name, outstandingMs: Math.max(0, (record.endedAt ?? at) - record.startedAt) })); + const open = [...log.openTunnels] + .sort((left, right) => left.openedAt - right.openedAt) + .slice(0, ENGINE_BROKER_MCP_TUNNEL_MAX) + .map((record): EngineBrokerOpenMcpTunnel => ({ openMs: Math.max(0, at - record.openedAt), delivered: record.delivered })); return { started: log.started, answered: log.answered, undecoded: log.undecoded, + tunnels: { opened: log.tunnelsOpened, closed: log.tunnelsClosed, delivered: log.tunnelsDelivered, open }, outstanding: outstanding.length > ENGINE_BROKER_MCP_OUTSTANDING_MAX ? [...outstanding.slice(0, ENGINE_BROKER_MCP_OUTSTANDING_MAX - 1), { name: ENGINE_BROKER_MCP_CALL_TRUNCATED, outstandingMs: 0 }] : outstanding diff --git a/src/runtime/engineBrokerMcpFacade.test.ts b/src/runtime/engineBrokerMcpFacade.test.ts index 54261ed..ff7d85e 100644 --- a/src/runtime/engineBrokerMcpFacade.test.ts +++ b/src/runtime/engineBrokerMcpFacade.test.ts @@ -12,6 +12,7 @@ import { Type } from "@earendil-works/pi-ai"; import { defineTool } from "@earendil-works/pi-coding-agent"; import { createPiToolMcpServer } from "../mcp/toolServer.js"; +import type { EngineBrokerMcpTunnelObservation } from "./engineBrokerMcpCallLog.js"; import { awaitMcpTunnelDrain, ENGINE_BROKER_MCP_FACADE_PORT, startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; const FACADE_URL = `http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}/mcp`; @@ -265,6 +266,60 @@ test("the facade carries the mount's server-initiated SSE stream, which only the } }); +/** + * The channel the call log could not see. + * + * A brokered turn's tool calls all answer and its provider requests all close, + * and the worker can still sit idle to the deadline — parked on the standalone + * GET SSE tunnel, which stays open for the whole session and, until now, wrote + * nothing anywhere. This drives the real transport: the tunnel the real client + * opens must observe as open with an age while it is open, as *delivered* once + * the mount pushes a frame through it, and as closed once the client ends it. + * + * Mutation: remove `calls.openTunnel` from the facade's route and the first + * assertion goes red (an open tunnel reads as a turn that opened none); remove + * `tunnel?.close()` from the relay's `finally` and the last one does (a closed + * tunnel reads as still open, which is the reading the whole instrument is + * meant to make trustworthy). + */ +test("the facade observes the standalone GET tunnel: open with its age, whether it delivered, and closed when it ends", async () => { + const rig = await startRig(); + const tunnels = (): EngineBrokerMcpTunnelObservation | undefined => rig.facade.observe(rig.turnId)?.tunnels; + const until = async (reason: string, ready: () => boolean): Promise => { + for (let attempt = 0; attempt < 200 && !ready(); attempt += 1) await new Promise((resolve) => setTimeout(resolve, 25)); + assert.ok(ready(), reason); + }; + try { + // Before any request the turn is registered and has relayed nothing: a + // measured zero, which is not the same statement as an open tunnel. + assert.deepEqual(tunnels(), { opened: 0, closed: 0, delivered: 0, open: [] }); + const { client } = await connectClient(rig.capability); + const notified = new Promise((resolve) => client.setNotificationHandler(ToolListChangedNotificationSchema, () => resolve())); + try { + await until("the client never opened its GET tunnel", () => (tunnels()?.open.length ?? 0) > 0); + const open = tunnels(); + assert.equal(open?.opened, 1); assert.equal(open?.closed, 0); + assert.ok((open?.open[0]?.openMs ?? -1) >= 0, "an open tunnel reports how long it has been open"); + assert.equal(open?.open[0]?.delivered, false, "a tunnel the mount has not pushed through is held open in silence"); + assert.equal(open?.delivered, 0); + + // The same tunnel, now actually carrying a server frame. "Held open + // having delivered nothing" and "in use" are different facts about it. + rig.server.sendToolListChanged(); + await withDeadline(notified, 4_000, "no server notification reached the client"); + await until("the delivered frame was never attributed to the tunnel", () => (tunnels()?.delivered ?? 0) === 1); + assert.equal(tunnels()?.open[0]?.delivered, true); + assert.equal(tunnels()?.closed, 0, "a tunnel that delivered is still open"); + } finally { + await client.close(); + } + await until("the closed GET tunnel still reads as open", () => (tunnels()?.closed ?? 0) === 1); + assert.deepEqual(tunnels(), { opened: 1, closed: 1, delivered: 1, open: [] }); + } finally { + await rig.close(); + } +}); + test("the facade carries the session-closing DELETE, and the mount then refuses the stale session", async () => { const rig = await startRig(); try { diff --git a/src/runtime/engineBrokerMcpFacade.ts b/src/runtime/engineBrokerMcpFacade.ts index 25a741f..51e4e0e 100644 --- a/src/runtime/engineBrokerMcpFacade.ts +++ b/src/runtime/engineBrokerMcpFacade.ts @@ -1,7 +1,7 @@ import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import { Readable } from "node:stream"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; -import { EngineBrokerMcpCallLog, type EngineBrokerMcpCallObservation } from "./engineBrokerMcpCallLog.js"; +import { EngineBrokerMcpCallLog, type EngineBrokerMcpCallObservation, type EngineBrokerMcpTunnelHandle } from "./engineBrokerMcpCallLog.js"; /** * The brokered worker's only route to its own per-wake Daimon MCP mount. The @@ -90,6 +90,11 @@ export async function startEngineBrokerMcpFacade() { catch (error) { calls.undecodable(scope.turnId); throw error; } const payload = body === undefined ? undefined : (body.buffer.slice(body.byteOffset, body.byteOffset + body.byteLength) as ArrayBuffer); const call = calls.begin(scope.turnId, body); + // The GET SSE tunnel is the session's other channel and the one that + // outlives every request: it stays open until the worker or the mount ends + // it, so a turn sealed with one still open is a turn whose worker may be + // parked on it. Observed only — never closed, timed out or refused here. + const tunnel = method === "GET" ? calls.openTunnel(scope.turnId) : undefined; const controller = new AbortController(); const open = inflight.get(scope.turnId) ?? new Set(); @@ -101,9 +106,10 @@ export async function startEngineBrokerMcpFacade() { // Answered only on a relay that reached its own end: a tunnel torn down // by the worker's death must not mark the call it was blocked on as // finished. - if (await forward(target, method, headersFor(method, request), payload, controller.signal, response)) call.answer(); + if (await forward(target, method, headersFor(method, request), payload, controller.signal, response, tunnel)) call.answer(); } finally { call.close(); + tunnel?.close(); response.off("close", abort); open.delete(controller); if (open.size === 0) inflight.delete(scope.turnId); @@ -136,7 +142,8 @@ export async function startEngineBrokerMcpFacade() { calls.close(turnId); }, /** - * What the facade saw of this turn's tool calls, or `undefined` for a turn + * What the facade saw of this turn's tool calls and GET tunnels, or + * `undefined` for a turn * it never registered. Read on the failure path, before `revoke`. */ observe: (turnId: string): EngineBrokerMcpCallObservation | undefined => calls.observe(turnId), @@ -179,7 +186,8 @@ async function forward( headers: Record, body: ArrayBuffer | undefined, signal: AbortSignal, - response: ServerResponse + response: ServerResponse, + tunnel?: EngineBrokerMcpTunnelHandle ): Promise { const upstream = await fetch(target, { method, headers, body, signal, redirect: "manual" }); // MCP never redirects, and following one would let the mount aim the facade @@ -196,8 +204,12 @@ async function forward( response.writeHead(upstream.status, outbound); if (upstream.body === null) { response.end(); return true; } const stream = Readable.fromWeb(upstream.body as Parameters[0]); + let delivered = false; try { for await (const chunk of stream) { + // The first byte the mount pushes: a tunnel that carried frames is a + // different fact from one held open having delivered nothing. + if (!delivered) { delivered = true; tunnel?.deliver(); } if (response.destroyed || response.writableEnded) throw new Error("MCP tunnel closed"); if (!response.write(chunk as Uint8Array)) await awaitMcpTunnelDrain(response, signal); } diff --git a/src/runtime/engineBrokerProtocol.test.ts b/src/runtime/engineBrokerProtocol.test.ts index 6f57c10..835493c 100644 --- a/src/runtime/engineBrokerProtocol.test.ts +++ b/src/runtime/engineBrokerProtocol.test.ts @@ -101,4 +101,27 @@ test("a failed frame carries the broker's in-flight MCP tool-call observation, b ]) assert.throws(() => parseEngineBrokerResponse({ ...value, mcpCalls }), /invalid broker frame/u, JSON.stringify(mcpCalls)); // A v1 record predates the instrument; a v1 frame that carries it is forged. assert.throws(() => parseEngineBrokerV1TerminalResponse({ version: "noopolis.daimon.engine-broker.v1", kind: "failed", requestId: "request-1", turnId: "turn-1", code: "engine_failed", mcpCalls: value.mcpCalls }), /invalid broker frame/u); + + /** + * The session's standalone GET SSE tunnel rides the same member, under the + * same rules. It is optional for exactly one reason — a turn sealed before + * the facade observed that channel replays without it — so its absence means + * "not measured" and never zero, and a record that carries it must still be + * a measurement: nothing closes before it opens, nothing delivers without + * opening, and no more can be open than `opened - closed`. + */ + const tunnels = { opened: 2, closed: 1, delivered: 1, open: [{ openMs: 428_004, delivered: false }] } as const; + const observed = { ...value, mcpCalls: { ...value.mcpCalls, tunnels } } as const; + assert.deepEqual(parseEngineBrokerResponse(observed), observed); + assert.deepEqual(parseEngineBrokerResponse(value), value, "a frame sealed before the tunnel was observed still replays, without the member"); + for (const forged of [ + { ...tunnels, closed: 3 }, + { ...tunnels, delivered: 3 }, + { ...tunnels, opened: 1, closed: 1, open: [{ openMs: 1, delivered: false }] }, + { ...tunnels, open: Array.from({ length: 9 }, () => ({ openMs: 1, delivered: false })), opened: 12, closed: 0 }, + { ...tunnels, open: [{ openMs: -1, delivered: false }] }, + { ...tunnels, open: [{ openMs: 1, delivered: "yes" }] }, + { ...tunnels, open: [{ openMs: 1, delivered: false, sessionId: "mcp-session-0" }] }, + { opened: 1, closed: 0, open: [] } + ]) assert.throws(() => parseEngineBrokerResponse({ ...value, mcpCalls: { ...value.mcpCalls, tunnels: forged } }), /invalid broker frame/u, JSON.stringify(forged)); }); diff --git a/src/runtime/engineBrokerProtocol.ts b/src/runtime/engineBrokerProtocol.ts index 643e41c..201d005 100644 --- a/src/runtime/engineBrokerProtocol.ts +++ b/src/runtime/engineBrokerProtocol.ts @@ -1,5 +1,5 @@ import { isEngineBrokerInferenceRequestKind, isEngineBrokerInferenceResponseKind, parseEngineBrokerInferenceRequest, parseEngineBrokerInferenceResponse, type EngineBrokerInferenceRequest, type EngineBrokerInferenceResponse } from "./engineBrokerInferenceProtocol.js"; -import { ENGINE_BROKER_MCP_CALL_NAME, ENGINE_BROKER_MCP_OUTSTANDING_MAX, type EngineBrokerMcpCallObservation } from "./engineBrokerMcpCallLog.js"; +import { ENGINE_BROKER_MCP_CALL_NAME, ENGINE_BROKER_MCP_OUTSTANDING_MAX, ENGINE_BROKER_MCP_TUNNEL_MAX, type EngineBrokerMcpCallObservation, type EngineBrokerMcpTunnelObservation } from "./engineBrokerMcpCallLog.js"; import { parseEngineBrokerTurnAccounting, parseEngineBrokerTurnLimitOverrides, type EngineBrokerTurnAccounting, type EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; /** @@ -143,7 +143,10 @@ function parseTerminal(input: JsonRecord, expected: typeof VERSION | typeof V1): */ function parseMcpCallObservation(value: unknown): EngineBrokerMcpCallObservation { const input = record(value); - exact(input, ["started", "answered", "undecoded", "outstanding"]); + // `tunnels` is optional for one reason only: a turn sealed before the GET + // tunnel was observed carries no such member, and its record must still + // replay. Absence there means "the instrument did not exist", never zero. + exact(input, ["started", "answered", "undecoded", "outstanding", ...(input.tunnels === undefined ? [] : ["tunnels"])]); const started = input.started, answered = input.answered, undecoded = input.undecoded; if (![started, answered, undecoded].every((count) => Number.isSafeInteger(count) && (count as number) >= 0)) throw new TypeError("invalid broker frame"); if (!Array.isArray(input.outstanding) || input.outstanding.length > ENGINE_BROKER_MCP_OUTSTANDING_MAX) throw new TypeError("invalid broker frame"); @@ -155,7 +158,31 @@ function parseMcpCallObservation(value: unknown): EngineBrokerMcpCallObservation return { name: call.name, outstandingMs: call.outstandingMs as number }; }); if ((answered as number) > (started as number) || outstanding.length > (started as number) - (answered as number)) throw new TypeError("invalid broker frame"); - return { started: started as number, answered: answered as number, undecoded: undecoded as number, outstanding }; + const tunnels = input.tunnels === undefined ? undefined : parseMcpTunnelObservation(input.tunnels); + return { started: started as number, answered: answered as number, undecoded: undecoded as number, outstanding, ...(tunnels === undefined ? {} : { tunnels }) }; +} + +/** + * The GET SSE tunnels, under the call observation's rules: counts and elapsed + * milliseconds, bounded, and internally consistent. A tunnel cannot close + * before it opened, cannot deliver without having opened, and no more can be + * reported open than `opened - closed` — a report claiming otherwise is a + * frame, not a measurement. + */ +function parseMcpTunnelObservation(value: unknown): EngineBrokerMcpTunnelObservation { + const input = record(value); + exact(input, ["opened", "closed", "delivered", "open"]); + const opened = input.opened, closed = input.closed, delivered = input.delivered; + if (![opened, closed, delivered].every((count) => Number.isSafeInteger(count) && (count as number) >= 0)) throw new TypeError("invalid broker frame"); + if ((closed as number) > (opened as number) || (delivered as number) > (opened as number)) throw new TypeError("invalid broker frame"); + if (!Array.isArray(input.open) || input.open.length > ENGINE_BROKER_MCP_TUNNEL_MAX || input.open.length > (opened as number) - (closed as number)) throw new TypeError("invalid broker frame"); + const open = input.open.map((entry) => { + const tunnel = record(entry); + exact(tunnel, ["openMs", "delivered"]); + if (!Number.isSafeInteger(tunnel.openMs) || (tunnel.openMs as number) < 0 || typeof tunnel.delivered !== "boolean") throw new TypeError("invalid broker frame"); + return { openMs: tunnel.openMs as number, delivered: tunnel.delivered }; + }); + return { opened: opened as number, closed: closed as number, delivered: delivered as number, open }; } function closedDiagnostic(value:JsonRecord):boolean{ diff --git a/src/runtime/engineBrokerSealLedger.test.ts b/src/runtime/engineBrokerSealLedger.test.ts index 9b5ecc2..60754e3 100644 --- a/src/runtime/engineBrokerSealLedger.test.ts +++ b/src/runtime/engineBrokerSealLedger.test.ts @@ -120,3 +120,41 @@ test("a cancelled turn that called no tool is distinguishable from one the facad try { await assert.rejects(readFile(engineBrokerSealLedgerPathFor(path.join(root, "usage.jsonl")), "utf8")); } finally { await rm(root, { recursive: true, force: true }); } }); + +/** + * The channel the seal row could not see, on the same durable route. + * + * Seven live runs sealed with every provider request closed and every tool call + * answered, and still went idle to the deadline. The facade relays one more + * thing for the whole session — the standalone GET SSE tunnel — and recorded + * nothing about it, so a worker parked reading that stream and a worker doing + * nothing wrote identical rows. These three seal the boundary that separates + * them: still open at seal time, closed before it, and never opened at all. + * + * Mutation: drop the `tunnels` member from `renderBrokerTurnSealLine` and the + * first three go red; render it unconditionally as zeros when the observation + * carries none, and the fourth does — a zero nobody measured reads exactly + * like a zero somebody did. + */ +test("a cancelled turn's GET tunnel is sealed open with its age, closed, or never opened — three distinct rows", async () => { + const mcp = (tunnels: Record): EngineBrokerMcpCallObservation => + ({ started: 1, answered: 1, undecoded: 0, outstanding: [], ...tunnels } as EngineBrokerMcpCallObservation); + + const parked = await cancelledTurn(() => mcp({ tunnels: { opened: 1, closed: 0, delivered: 0, open: [{ openMs: 428_004, delivered: false }] } })); + assert.deepEqual((parked.seal?.mcp as Record).tunnels, { + opened: 1, closed: 0, delivered: 0, open: [{ open_ms: 428_004, delivered: false }] + }, "a turn sealed with a tunnel still open must say so, and say how long it had been open"); + + const ended = await cancelledTurn(() => mcp({ tunnels: { opened: 1, closed: 1, delivered: 2, open: [] } })); + assert.deepEqual((ended.seal?.mcp as Record).tunnels, { opened: 1, closed: 1, delivered: 2, open: [] }, + "a tunnel that closed before the seal is not an open one"); + + const never = await cancelledTurn(() => mcp({ tunnels: { opened: 0, closed: 0, delivered: 0, open: [] } })); + assert.deepEqual((never.seal?.mcp as Record).tunnels, { opened: 0, closed: 0, delivered: 0, open: [] }, + "a turn whose facade never relayed a GET measured zero, which is not the same as not having looked"); + + // And the fourth state, which is the absence: a turn sealed before this + // channel was observed at all carries no `tunnels` member. + const unobserved = await cancelledTurn(() => ({ started: 1, answered: 1, undecoded: 0, outstanding: [] })); + assert.equal(Object.hasOwn(unobserved.seal?.mcp as Record, "tunnels"), false); +}); diff --git a/src/runtime/engineBrokerSealLedger.ts b/src/runtime/engineBrokerSealLedger.ts index 52d38c1..d9fd3f4 100644 --- a/src/runtime/engineBrokerSealLedger.ts +++ b/src/runtime/engineBrokerSealLedger.ts @@ -1,4 +1,4 @@ -import { ENGINE_BROKER_MCP_CALL_NAME, ENGINE_BROKER_MCP_OUTSTANDING_MAX } from "./engineBrokerMcpCallLog.js"; +import { ENGINE_BROKER_MCP_CALL_NAME, ENGINE_BROKER_MCP_OUTSTANDING_MAX, ENGINE_BROKER_MCP_TUNNEL_MAX } from "./engineBrokerMcpCallLog.js"; import type { EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; import { TURN_USAGE_LEDGER, TURN_USAGE_MAX_IDENTIFIER_CHARS } from "./turnUsageLedger.js"; @@ -27,7 +27,8 @@ import { TURN_USAGE_LEDGER, TURN_USAGE_MAX_IDENTIFIER_CHARS } from "./turnUsageL * the accounting, the diagnostic's closed `status`/`stage`/`failure_class` * and its already-redacted, already-bounded, control-character-free `reason` * (`engineBrokerNativeClient.ts` produced it; nothing here re-derives it), - * plus tool-call names and elapsed milliseconds. Never a prompt, a body, a + * plus tool-call names, GET-tunnel counts and elapsed milliseconds. Never a + * prompt, a body, a * reply, a bearer, a capability or a session id — none of which the terminal * response carries in the first place. * - **Absence stays absence.** `mcp` is written only when the facade actually @@ -102,7 +103,21 @@ export const renderBrokerTurnSealLine = (terminal: EngineBrokerTerminalResponse, undecoded: terminal.mcpCalls.undecoded, outstanding: terminal.mcpCalls.outstanding .slice(0, ENGINE_BROKER_MCP_OUTSTANDING_MAX) - .map((call) => ({ name: ENGINE_BROKER_MCP_CALL_NAME.test(call.name) ? call.name : "", outstanding_ms: call.outstandingMs })) + .map((call) => ({ name: ENGINE_BROKER_MCP_CALL_NAME.test(call.name) ? call.name : "", outstanding_ms: call.outstandingMs })), + // The session's standalone GET tunnel, absent for a turn sealed before + // the facade observed that channel at all. + ...(terminal.mcpCalls.tunnels === undefined + ? {} + : { + tunnels: { + opened: terminal.mcpCalls.tunnels.opened, + closed: terminal.mcpCalls.tunnels.closed, + delivered: terminal.mcpCalls.tunnels.delivered, + open: terminal.mcpCalls.tunnels.open + .slice(0, ENGINE_BROKER_MCP_TUNNEL_MAX) + .map((tunnel) => ({ open_ms: tunnel.openMs, delivered: tunnel.delivered })) + } + }) } } : {}) From edbbb998d6185c1ecd40ed312709d60a5cb8d052 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 13:02:12 +0200 Subject: [PATCH 115/124] docs: record the MCP GET tunnel observation and what the real CLI does with it --- src/runtime/AGENTS.md | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index f4243e1..db88d50 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -314,6 +314,31 @@ answer, so the call it was blocked on stays outstanding with the elapsed time it had reached — otherwise the turn's death would erase the evidence the instrument exists to keep. +The facade relays one more thing, for the whole session, and until now wrote +nothing about it. A `tools/call` is a POST that answers; the standalone `GET` +SSE tunnel is the route a server notification or progress frame takes, and it +stays open from `initialize` to the worker's own shutdown. A worker parked +reading it was, in every artifact the broker wrote, identical to a worker doing +nothing: every provider request closed, every tool call answered, idle to the +deadline. `EngineBrokerMcpCallLog.openTunnel` records that lifecycle on the same +observation — `tunnels: {opened, closed, delivered, open: [{openMs, delivered}]}` +— so a turn sealed with one still open says so and says how long it had been +open, and `delivered` separates a tunnel actively carrying frames from one held +open having received nothing, which is the difference that decides whether it is +the blocker. Bounded at `ENGINE_BROKER_MCP_TUNNEL_MAX` open entries (a session +opens one), counts and elapsed milliseconds only, never a frame, an event +payload or a session id. It is *observation only*: nothing here closes, times +out or refuses a tunnel, because an instrument that tore the stream down would +destroy the evidence it exists to gather. The member is optional on the wire for +one reason — a turn sealed before it existed must still replay — so its absence +means "not measured" and never zero, exactly as `mcp`'s own absence does. +Measured against the real CLI (rig, grok 1.0.34, real facade and mount): the +tunnel opens ~3 ms after `initialize`, carries nothing for its whole life, and +**closes 16 ms before the worker exits** — the close is the worker's own +shutdown, not the facade's. A turn that never reaches that shutdown is the one +that seals with it open; a deliberately stalled `tools/call` sealed +`open: [{openMs: 14652, delivered: false}]` beside its outstanding call. + That seam is enough for a turn that *fails with a reply* and not for the turn the instrument was built for. A worker that crashes still produces a terminal response; a worker that HANGS is cancelled by its client's deadline, and a @@ -328,7 +353,8 @@ terminal turn (`turns.jsonl`, `engineBrokerSealLedgerPathFor`), rendered from the sealed response and nothing else. Its members are the accounting, the failure `code`, the diagnostic's closed `status`/`stage`/`failure_class` with the reason `engineBrokerNativeClient.ts` already redacted and bounded, and the -facade's `mcp` observation — names, counts and elapsed milliseconds. Never a +facade's `mcp` observation — names, counts, GET-tunnel lifecycle and elapsed +milliseconds. Never a prompt, body, reply, bearer, capability or session id; the terminal response carries none of those in the first place, and the projection is an allow-list rather than a spread, so a future additive member of the response cannot become From 1a5ba8dc13b30e5809a39de118bf950156874ba6 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 13:04:49 +0200 Subject: [PATCH 116/124] docs: name the tunnel timings in the seal line's own bound --- src/runtime/engineBrokerSealLedger.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/engineBrokerSealLedger.ts b/src/runtime/engineBrokerSealLedger.ts index d9fd3f4..c386838 100644 --- a/src/runtime/engineBrokerSealLedger.ts +++ b/src/runtime/engineBrokerSealLedger.ts @@ -56,7 +56,7 @@ export const TURN_SEAL_LEDGER = { fileMode: TURN_USAGE_LEDGER.fileMode } as const; -/** A rendered seal line is bounded by its own contents: a 768-byte reason plus 16 bounded names. */ +/** A rendered seal line is bounded by its own contents: a 768-byte reason, 16 bounded names and 8 tunnel timings. */ export const TURN_SEAL_MAX_LINE_BYTES = 8_192; export type BrokerTurnSealEntry = Readonly<{ agent: string; wake: string; at: string }>; From 130b548b5a64d6f4584a2fa66f040547d9896d3f Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 13:11:50 +0200 Subject: [PATCH 117/124] test: split the facade's observation suite and its rig out of the facade tests --- src/runtime/engineBrokerMcpFacade.test.ts | 268 +----------------- src/runtime/engineBrokerMcpFacadeRig.test.ts | 164 +++++++++++ .../engineBrokerMcpObservation.test.ts | 155 ++++++++++ 3 files changed, 322 insertions(+), 265 deletions(-) create mode 100644 src/runtime/engineBrokerMcpFacadeRig.test.ts create mode 100644 src/runtime/engineBrokerMcpObservation.test.ts diff --git a/src/runtime/engineBrokerMcpFacade.test.ts b/src/runtime/engineBrokerMcpFacade.test.ts index ff7d85e..6ad19ca 100644 --- a/src/runtime/engineBrokerMcpFacade.test.ts +++ b/src/runtime/engineBrokerMcpFacade.test.ts @@ -1,50 +1,13 @@ import assert from "node:assert/strict"; -import { createServer, type Server as HttpServer, type IncomingMessage, type ServerResponse } from "node:http"; +import { createServer } from "node:http"; -import { randomUUID } from "node:crypto"; import test from "node:test"; -import { Client } from "@modelcontextprotocol/sdk/client/index.js"; -import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; -import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; import { ToolListChangedNotificationSchema } from "@modelcontextprotocol/sdk/types.js"; -import { Type } from "@earendil-works/pi-ai"; -import { defineTool } from "@earendil-works/pi-coding-agent"; -import { createPiToolMcpServer } from "../mcp/toolServer.js"; -import type { EngineBrokerMcpTunnelObservation } from "./engineBrokerMcpCallLog.js"; -import { awaitMcpTunnelDrain, ENGINE_BROKER_MCP_FACADE_PORT, startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; +import { ENGINE_BROKER_MCP_FACADE_PORT } from "./engineBrokerMcpFacade.js"; -const FACADE_URL = `http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}/mcp`; -type Facade = Awaited>; - -/** - * One facade serves every turn of a broker, so the tests share one too. It - * also keeps the fixed port free: a facade per test would leave the HTTP - * client pooling a socket onto a server that no longer exists. - */ -let shared: Facade | undefined; -const sharedFacade = async (): Promise => (shared ??= await startFacade()); -const releaseShared = async (): Promise => { const facade = shared; shared = undefined; if (facade) await facade.close(); }; -test.after(releaseShared); - -/** - * Closing a facade destroys its sockets, and the port is fixed, so the HTTP - * client can still hold a pooled connection to the server that just went away. - * That is a test-harness artifact — one facade outlives a whole broker — so a - * fresh facade is probed until a refusal proves the route is live again. - */ -const startFacade = async (): Promise => { - const facade = await startEngineBrokerMcpFacade(); - for (let attempt = 0; attempt < 20; attempt += 1) { - try { - const probe = await fetch(FACADE_URL, { method: "PUT" }); - await probe.body?.cancel(); - if (probe.status === 403) return facade; - } catch { /* a pooled socket onto the previous facade: try the next one */ } - } - throw new Error("facade did not answer after starting"); -}; +import { connectClient, FACADE_URL, releaseShared, sharedFacade, startFacade, startRig, withDeadline } from "./engineBrokerMcpFacadeRig.test.js"; test("MCP facade routes only valid active capabilities to the registered mount", async () => { const facade=await sharedFacade(); @@ -53,177 +16,6 @@ test("MCP facade routes only valid active capabilities to the registered mount", try{assert.equal((await call("wrong-token-abcdefghijklmnopqrstuvwxyz0123456789")).status,403);assert.equal((await call(token)).status,200);assert.equal(calls,1);facade.revoke("turn-capabilities");assert.equal((await call(token)).status,403);assert.equal(calls,1);}finally{facade.revoke("turn-capabilities");await new Promise((resolve)=>target.close(()=>resolve()));} }); -/** - * Daimon writes a tool receipt only on completion, so a call that started and - * never returned reads exactly like a call that was never made — the one path - * a seven-minute live hang left unlit. The facade is where that difference is - * visible, and it has to survive the tear-down that ends the turn: a tunnel - * destroyed when the worker dies must not mark the call it was blocked on as - * answered, or the instrument erases the very evidence it exists to keep. - */ -test("MCP facade reports a tool call that started and never returned as outstanding, by name", async () => { - const facade = await sharedFacade(); - const held: ServerResponse[] = []; - // Two ways for a mount not to answer: never reply at all (`moltnet_read`), - // or open the stream and never deliver the result (`memory_recall`). - const target = createServer((request, response) => { - const chunks: Buffer[] = []; - request.on("data", (chunk: Buffer) => chunks.push(chunk)); - request.on("end", () => { - const asked = Buffer.concat(chunks).toString("utf8"); - if (asked.includes("moltnet_read")) { held.push(response); return; } - if (asked.includes("memory_recall")) { response.writeHead(200, { "content-type": "text/event-stream" }); response.write(": open\n\n"); held.push(response); return; } - response.writeHead(200, { "content-type": "application/json" }); response.end('{"jsonrpc":"2.0","id":1,"result":{}}'); - }); - }); - await new Promise((resolve) => target.listen(0, "127.0.0.1", resolve)); - const address = target.address(); if (address === null || typeof address === "string") throw new Error(); - const turnId = "turn-outstanding", token = facade.register("agent", turnId, `http://127.0.0.1:${address.port}/mcp`); - const post = (name: string, signal?: AbortSignal): Promise => fetch(FACADE_URL, { method: "POST", signal, headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name, arguments: { text: "argument-bytes-that-must-never-be-recorded" } } }) }); - const pending = new AbortController(); - /** - * A live call's elapsed time grows with every read, so two identical reads - * mean every relay has settled — the only moment at which "answered" is - * final. Polling for a name instead would read the log mid-teardown. - */ - const settled = async (): Promise> => { - for (let attempt = 0; attempt < 100; attempt += 1) { - const before = JSON.stringify(facade.observe(turnId)); - await new Promise((resolve) => setTimeout(resolve, 60)); - if (JSON.stringify(facade.observe(turnId)) === before) return facade.observe(turnId); - } - throw new Error("the facade's observation never settled"); - }; - const names = (observed: ReturnType): readonly string[] => (observed?.outstanding ?? []).map((call) => call.name); - try { - const answered = await post("daimon__moltnet_send"); - assert.equal(answered.status, 200); await answered.text(); - const hanging = post("daimon__moltnet_read", pending.signal).catch(() => undefined); - for (let attempt = 0; attempt < 200 && held.length === 0; attempt += 1) await new Promise((resolve) => setTimeout(resolve, 25)); - assert.equal(held.length, 1, "the mount never received the hung tool call"); - - const observed = facade.observe(turnId); - assert.deepEqual(names(observed), ["daimon__moltnet_read"], "the unanswered call must be outstanding, by name"); - assert.equal(observed?.started, 2); assert.equal(observed?.answered, 1, "the completed call must not be outstanding"); assert.equal(observed?.undecoded, 0); - assert.ok((observed?.outstanding[0]?.outstandingMs ?? -1) >= 0, "an outstanding call reports how long it has been waiting"); - assert.ok(!JSON.stringify(observed).includes("argument-bytes"), "names and timings only: no arguments"); - - // The turn's own death tears the tunnel down. The call was still never - // answered, and must still say so. - pending.abort(); await hanging; - const afterTeardown = await settled(); - assert.deepEqual(names(afterTeardown), ["daimon__moltnet_read"], "a torn-down relay is not an answer"); - assert.equal(afterTeardown?.answered, 1); - - // A stream the facade opened and never finished relaying is not an answer - // either. Awaiting the headers and one chunk puts the facade inside its own - // streaming relay before the client walks away, which is the branch that - // decides whether a half-written tunnel counts as an answer. - const halted = new AbortController(); - const half = await post("memory_recall", halted.signal); - assert.equal(half.status, 200); await half.body!.getReader().read(); halted.abort(); - const afterHalfRelay = await settled(); - assert.deepEqual(names(afterHalfRelay), ["daimon__moltnet_read", "memory_recall"], "a half-relayed stream is not an answer"); - assert.equal(afterHalfRelay?.answered, 1); - - facade.revoke(turnId); - assert.equal(facade.observe(turnId), undefined, "a turn the facade never registered observes as absence, not as zero"); - } finally { - pending.abort(); facade.revoke(turnId); - for (const response of held) response.destroy(); - target.closeAllConnections(); - await new Promise((resolve) => target.close(() => resolve())); - } -}); - -/** - * The brokered worker's real route: a real Daimon MCP mount behind a real - * Streamable HTTP transport, reached by a real MCP client through the facade. - * Asserting that a header is copied would pass while the route stayed broken, - * so every case below drives the transport end to end. - */ -type Rig = Readonly<{ - facade: Facade; - turnId: string; - server: ReturnType; - capability: string; - observed: IncomingMessage[]; - /** Resolves when the mount's standalone GET stream is torn down. */ - getStreamClosed: Promise; - close: () => Promise; -}>; - -const echoTool = defineTool({ - name: "moltnet_read", - label: "Read a scoped Moltnet surface", - description: "Reads the fixture room.", - parameters: Type.Object({ target: Type.String() }, { additionalProperties: false }), - async execute(_toolCallId: string, params: { target: string }) { - return { content: [{ type: "text" as const, text: `read ${params.target}` }], details: { target: params.target } }; - } -}); - -let turns = 0; - -const startRig = async (facade?: Facade): Promise => { - const host = facade ?? await sharedFacade(); - const turnId = `turn-${++turns}`; - const server = createPiToolMcpServer([echoTool], {}); - const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: () => randomUUID() }); - await server.connect(transport); - const observed: IncomingMessage[] = []; - let noteGetStreamClosed = (): void => undefined; - const getStreamClosed = new Promise((resolve) => { noteGetStreamClosed = resolve; }); - const mount = createServer((request, response) => { - observed.push(request); - if (request.method === "GET") response.on("close", () => noteGetStreamClosed()); - const chunks: Buffer[] = []; - request.on("data", (chunk: Buffer) => chunks.push(chunk)); - request.on("end", () => { - const raw = Buffer.concat(chunks); - let parsed: unknown; - try { parsed = raw.length === 0 ? undefined : JSON.parse(raw.toString("utf8")); } catch { parsed = undefined; } - void transport.handleRequest(request, response, parsed); - }); - }); - await new Promise((resolve) => mount.listen(0, "127.0.0.1", resolve)); - const address = mount.address(); - if (address === null || typeof address === "string") throw new Error("mount address unavailable"); - const capability = host.register("alpha", turnId, `http://127.0.0.1:${address.port}/mcp`); - return { - facade: host, turnId, server, capability, observed, getStreamClosed, - close: async () => { - host.revoke(turnId); - await closeMount(mount, transport, server); - } - }; -}; - -const closeMount = async (mount: HttpServer, transport: StreamableHTTPServerTransport, server: ReturnType): Promise => { - mount.closeAllConnections(); - await new Promise((resolve) => mount.close(() => resolve())); - await transport.close().catch(() => undefined); - await server.close().catch(() => undefined); -}; - -const connectClient = async (capability: string): Promise<{ client: Client; transport: StreamableHTTPClientTransport }> => { - const transport = new StreamableHTTPClientTransport(new URL(FACADE_URL), { - requestInit: { headers: { authorization: `Bearer ${capability}` } } - }); - const client = new Client({ name: "daimon-facade-test-client", version: "0.1.0" }); - await client.connect(transport); - return { client, transport }; -}; - -const withDeadline = async (work: Promise, ms: number, reason: string): Promise => { - let timer: NodeJS.Timeout | undefined; - try { - return await Promise.race([work, new Promise((_resolve, reject) => { timer = setTimeout(() => reject(new Error(reason)), ms); })]); - } finally { - if (timer) clearTimeout(timer); - } -}; - test("a brokered worker completes the whole MCP handshake through the facade and calls a mounted tool", async () => { const rig = await startRig(); try { @@ -266,60 +58,6 @@ test("the facade carries the mount's server-initiated SSE stream, which only the } }); -/** - * The channel the call log could not see. - * - * A brokered turn's tool calls all answer and its provider requests all close, - * and the worker can still sit idle to the deadline — parked on the standalone - * GET SSE tunnel, which stays open for the whole session and, until now, wrote - * nothing anywhere. This drives the real transport: the tunnel the real client - * opens must observe as open with an age while it is open, as *delivered* once - * the mount pushes a frame through it, and as closed once the client ends it. - * - * Mutation: remove `calls.openTunnel` from the facade's route and the first - * assertion goes red (an open tunnel reads as a turn that opened none); remove - * `tunnel?.close()` from the relay's `finally` and the last one does (a closed - * tunnel reads as still open, which is the reading the whole instrument is - * meant to make trustworthy). - */ -test("the facade observes the standalone GET tunnel: open with its age, whether it delivered, and closed when it ends", async () => { - const rig = await startRig(); - const tunnels = (): EngineBrokerMcpTunnelObservation | undefined => rig.facade.observe(rig.turnId)?.tunnels; - const until = async (reason: string, ready: () => boolean): Promise => { - for (let attempt = 0; attempt < 200 && !ready(); attempt += 1) await new Promise((resolve) => setTimeout(resolve, 25)); - assert.ok(ready(), reason); - }; - try { - // Before any request the turn is registered and has relayed nothing: a - // measured zero, which is not the same statement as an open tunnel. - assert.deepEqual(tunnels(), { opened: 0, closed: 0, delivered: 0, open: [] }); - const { client } = await connectClient(rig.capability); - const notified = new Promise((resolve) => client.setNotificationHandler(ToolListChangedNotificationSchema, () => resolve())); - try { - await until("the client never opened its GET tunnel", () => (tunnels()?.open.length ?? 0) > 0); - const open = tunnels(); - assert.equal(open?.opened, 1); assert.equal(open?.closed, 0); - assert.ok((open?.open[0]?.openMs ?? -1) >= 0, "an open tunnel reports how long it has been open"); - assert.equal(open?.open[0]?.delivered, false, "a tunnel the mount has not pushed through is held open in silence"); - assert.equal(open?.delivered, 0); - - // The same tunnel, now actually carrying a server frame. "Held open - // having delivered nothing" and "in use" are different facts about it. - rig.server.sendToolListChanged(); - await withDeadline(notified, 4_000, "no server notification reached the client"); - await until("the delivered frame was never attributed to the tunnel", () => (tunnels()?.delivered ?? 0) === 1); - assert.equal(tunnels()?.open[0]?.delivered, true); - assert.equal(tunnels()?.closed, 0, "a tunnel that delivered is still open"); - } finally { - await client.close(); - } - await until("the closed GET tunnel still reads as open", () => (tunnels()?.closed ?? 0) === 1); - assert.deepEqual(tunnels(), { opened: 1, closed: 1, delivered: 1, open: [] }); - } finally { - await rig.close(); - } -}); - test("the facade carries the session-closing DELETE, and the mount then refuses the stale session", async () => { const rig = await startRig(); try { diff --git a/src/runtime/engineBrokerMcpFacadeRig.test.ts b/src/runtime/engineBrokerMcpFacadeRig.test.ts new file mode 100644 index 0000000..fd336a4 --- /dev/null +++ b/src/runtime/engineBrokerMcpFacadeRig.test.ts @@ -0,0 +1,164 @@ +import { createServer, type Server as HttpServer, type IncomingMessage } from "node:http"; + +import { randomUUID } from "node:crypto"; +import test from "node:test"; + +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; +import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; +import { Type } from "@earendil-works/pi-ai"; +import { defineTool } from "@earendil-works/pi-coding-agent"; + +import { createPiToolMcpServer } from "../mcp/toolServer.js"; +import { ENGINE_BROKER_MCP_FACADE_PORT, startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; + +/** + * The facade's shared test rig, and not a suite of its own. + * + * Two suites drive the same boundary — the facade's routing and header + * contract, and the observations it records while relaying — and both need one + * facade on the protocol's fixed port plus a real Daimon MCP mount behind a + * real Streamable HTTP transport. Splitting them into one file each kept both + * readable; duplicating the rig into each would have left two copies of the + * thing every assertion depends on. It carries a `.test.ts` name so it never + * reaches production `dist` (`tsconfig.build.json` excludes exactly that), and + * running it on its own asserts nothing, which is what it is. + * + * Each suite is its own process, so each holds its own shared facade and each + * takes the fixed port for the length of its file. + */ +export const FACADE_URL = `http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}/mcp`; +export type Facade = Awaited>; + +/** + * One facade serves every turn of a broker, so the tests share one too. It + * also keeps the fixed port free: a facade per test would leave the HTTP + * client pooling a socket onto a server that no longer exists. + */ +let shared: Facade | undefined; +export const sharedFacade = async (): Promise => (shared ??= await startFacade()); +export const releaseShared = async (): Promise => { const facade = shared; shared = undefined; if (facade) await facade.close(); }; +test.after(releaseShared); + +/** + * Closing a facade destroys its sockets, and the port is fixed, so the HTTP + * client can still hold a pooled connection to the server that just went away. + * That is a test-harness artifact — one facade outlives a whole broker — so a + * fresh facade is probed until a refusal proves the route is live again. + */ +/** + * The facade's port is the control protocol's own, so the two suites that + * drive it cannot each hold one at the same time. Whichever binds first runs; + * the other waits for it to release the port rather than failing on the + * collision, which is the whole cost of splitting this boundary in two. + */ +const bindFacade = async (): Promise => { + for (let attempt = 0; attempt < 240; attempt += 1) { + try { return await startEngineBrokerMcpFacade(); } + catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EADDRINUSE") throw error; + await new Promise((resolve) => setTimeout(resolve, 250)); + } + } + throw new Error("the MCP facade port never came free"); +}; + +export const startFacade = async (): Promise => { + const facade = await bindFacade(); + for (let attempt = 0; attempt < 20; attempt += 1) { + try { + const probe = await fetch(FACADE_URL, { method: "PUT" }); + await probe.body?.cancel(); + if (probe.status === 403) return facade; + } catch { /* a pooled socket onto the previous facade: try the next one */ } + } + throw new Error("facade did not answer after starting"); +}; + +/** + * The brokered worker's real route: a real Daimon MCP mount behind a real + * Streamable HTTP transport, reached by a real MCP client through the facade. + * Asserting that a header is copied would pass while the route stayed broken, + * so every case below drives the transport end to end. + */ +export type Rig = Readonly<{ + facade: Facade; + turnId: string; + server: ReturnType; + capability: string; + observed: IncomingMessage[]; + /** Resolves when the mount's standalone GET stream is torn down. */ + getStreamClosed: Promise; + close: () => Promise; +}>; + +export const echoTool = defineTool({ + name: "moltnet_read", + label: "Read a scoped Moltnet surface", + description: "Reads the fixture room.", + parameters: Type.Object({ target: Type.String() }, { additionalProperties: false }), + async execute(_toolCallId: string, params: { target: string }) { + return { content: [{ type: "text" as const, text: `read ${params.target}` }], details: { target: params.target } }; + } +}); + +let turns = 0; + +export const startRig = async (facade?: Facade): Promise => { + const host = facade ?? await sharedFacade(); + const turnId = `turn-${++turns}`; + const server = createPiToolMcpServer([echoTool], {}); + const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: () => randomUUID() }); + await server.connect(transport); + const observed: IncomingMessage[] = []; + let noteGetStreamClosed = (): void => undefined; + const getStreamClosed = new Promise((resolve) => { noteGetStreamClosed = resolve; }); + const mount = createServer((request, response) => { + observed.push(request); + if (request.method === "GET") response.on("close", () => noteGetStreamClosed()); + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.on("end", () => { + const raw = Buffer.concat(chunks); + let parsed: unknown; + try { parsed = raw.length === 0 ? undefined : JSON.parse(raw.toString("utf8")); } catch { parsed = undefined; } + void transport.handleRequest(request, response, parsed); + }); + }); + await new Promise((resolve) => mount.listen(0, "127.0.0.1", resolve)); + const address = mount.address(); + if (address === null || typeof address === "string") throw new Error("mount address unavailable"); + const capability = host.register("alpha", turnId, `http://127.0.0.1:${address.port}/mcp`); + return { + facade: host, turnId, server, capability, observed, getStreamClosed, + close: async () => { + host.revoke(turnId); + await closeMount(mount, transport, server); + } + }; +}; + +export const closeMount = async (mount: HttpServer, transport: StreamableHTTPServerTransport, server: ReturnType): Promise => { + mount.closeAllConnections(); + await new Promise((resolve) => mount.close(() => resolve())); + await transport.close().catch(() => undefined); + await server.close().catch(() => undefined); +}; + +export const connectClient = async (capability: string): Promise<{ client: Client; transport: StreamableHTTPClientTransport }> => { + const transport = new StreamableHTTPClientTransport(new URL(FACADE_URL), { + requestInit: { headers: { authorization: `Bearer ${capability}` } } + }); + const client = new Client({ name: "daimon-facade-test-client", version: "0.1.0" }); + await client.connect(transport); + return { client, transport }; +}; + +export const withDeadline = async (work: Promise, ms: number, reason: string): Promise => { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([work, new Promise((_resolve, reject) => { timer = setTimeout(() => reject(new Error(reason)), ms); })]); + } finally { + if (timer) clearTimeout(timer); + } +}; diff --git a/src/runtime/engineBrokerMcpObservation.test.ts b/src/runtime/engineBrokerMcpObservation.test.ts new file mode 100644 index 0000000..b70a6f2 --- /dev/null +++ b/src/runtime/engineBrokerMcpObservation.test.ts @@ -0,0 +1,155 @@ +import assert from "node:assert/strict"; +import { createServer, type ServerResponse } from "node:http"; + +import test from "node:test"; + +import { ToolListChangedNotificationSchema } from "@modelcontextprotocol/sdk/types.js"; + +import type { EngineBrokerMcpTunnelObservation } from "./engineBrokerMcpCallLog.js"; +import { connectClient, FACADE_URL, sharedFacade, startRig, withDeadline } from "./engineBrokerMcpFacadeRig.test.js"; + +/** + * What the facade saw while it was relaying, which is the only place a call or + * a stream is observable *while it is still running*. + * + * Daimon writes a tool receipt on completion and nothing at all for the + * session's GET tunnel, so a call that never returned, a worker parked on an + * open stream, and a worker doing nothing were one indistinguishable silence. + * Both instruments are driven here through the real facade. + */ +/** + * Daimon writes a tool receipt only on completion, so a call that started and + * never returned reads exactly like a call that was never made — the one path + * a seven-minute live hang left unlit. The facade is where that difference is + * visible, and it has to survive the tear-down that ends the turn: a tunnel + * destroyed when the worker dies must not mark the call it was blocked on as + * answered, or the instrument erases the very evidence it exists to keep. + */ +test("MCP facade reports a tool call that started and never returned as outstanding, by name", async () => { + const facade = await sharedFacade(); + const held: ServerResponse[] = []; + // Two ways for a mount not to answer: never reply at all (`moltnet_read`), + // or open the stream and never deliver the result (`memory_recall`). + const target = createServer((request, response) => { + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.on("end", () => { + const asked = Buffer.concat(chunks).toString("utf8"); + if (asked.includes("moltnet_read")) { held.push(response); return; } + if (asked.includes("memory_recall")) { response.writeHead(200, { "content-type": "text/event-stream" }); response.write(": open\n\n"); held.push(response); return; } + response.writeHead(200, { "content-type": "application/json" }); response.end('{"jsonrpc":"2.0","id":1,"result":{}}'); + }); + }); + await new Promise((resolve) => target.listen(0, "127.0.0.1", resolve)); + const address = target.address(); if (address === null || typeof address === "string") throw new Error(); + const turnId = "turn-outstanding", token = facade.register("agent", turnId, `http://127.0.0.1:${address.port}/mcp`); + const post = (name: string, signal?: AbortSignal): Promise => fetch(FACADE_URL, { method: "POST", signal, headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name, arguments: { text: "argument-bytes-that-must-never-be-recorded" } } }) }); + const pending = new AbortController(); + /** + * A live call's elapsed time grows with every read, so two identical reads + * mean every relay has settled — the only moment at which "answered" is + * final. Polling for a name instead would read the log mid-teardown. + */ + const settled = async (): Promise> => { + for (let attempt = 0; attempt < 100; attempt += 1) { + const before = JSON.stringify(facade.observe(turnId)); + await new Promise((resolve) => setTimeout(resolve, 60)); + if (JSON.stringify(facade.observe(turnId)) === before) return facade.observe(turnId); + } + throw new Error("the facade's observation never settled"); + }; + const names = (observed: ReturnType): readonly string[] => (observed?.outstanding ?? []).map((call) => call.name); + try { + const answered = await post("daimon__moltnet_send"); + assert.equal(answered.status, 200); await answered.text(); + const hanging = post("daimon__moltnet_read", pending.signal).catch(() => undefined); + for (let attempt = 0; attempt < 200 && held.length === 0; attempt += 1) await new Promise((resolve) => setTimeout(resolve, 25)); + assert.equal(held.length, 1, "the mount never received the hung tool call"); + + const observed = facade.observe(turnId); + assert.deepEqual(names(observed), ["daimon__moltnet_read"], "the unanswered call must be outstanding, by name"); + assert.equal(observed?.started, 2); assert.equal(observed?.answered, 1, "the completed call must not be outstanding"); assert.equal(observed?.undecoded, 0); + assert.ok((observed?.outstanding[0]?.outstandingMs ?? -1) >= 0, "an outstanding call reports how long it has been waiting"); + assert.ok(!JSON.stringify(observed).includes("argument-bytes"), "names and timings only: no arguments"); + + // The turn's own death tears the tunnel down. The call was still never + // answered, and must still say so. + pending.abort(); await hanging; + const afterTeardown = await settled(); + assert.deepEqual(names(afterTeardown), ["daimon__moltnet_read"], "a torn-down relay is not an answer"); + assert.equal(afterTeardown?.answered, 1); + + // A stream the facade opened and never finished relaying is not an answer + // either. Awaiting the headers and one chunk puts the facade inside its own + // streaming relay before the client walks away, which is the branch that + // decides whether a half-written tunnel counts as an answer. + const halted = new AbortController(); + const half = await post("memory_recall", halted.signal); + assert.equal(half.status, 200); await half.body!.getReader().read(); halted.abort(); + const afterHalfRelay = await settled(); + assert.deepEqual(names(afterHalfRelay), ["daimon__moltnet_read", "memory_recall"], "a half-relayed stream is not an answer"); + assert.equal(afterHalfRelay?.answered, 1); + + facade.revoke(turnId); + assert.equal(facade.observe(turnId), undefined, "a turn the facade never registered observes as absence, not as zero"); + } finally { + pending.abort(); facade.revoke(turnId); + for (const response of held) response.destroy(); + target.closeAllConnections(); + await new Promise((resolve) => target.close(() => resolve())); + } +}); + +/** + * The channel the call log could not see. + * + * A brokered turn's tool calls all answer and its provider requests all close, + * and the worker can still sit idle to the deadline — parked on the standalone + * GET SSE tunnel, which stays open for the whole session and, until now, wrote + * nothing anywhere. This drives the real transport: the tunnel the real client + * opens must observe as open with an age while it is open, as *delivered* once + * the mount pushes a frame through it, and as closed once the client ends it. + * + * Mutation: remove `calls.openTunnel` from the facade's route and the first + * assertion goes red (an open tunnel reads as a turn that opened none); remove + * `tunnel?.close()` from the relay's `finally` and the last one does (a closed + * tunnel reads as still open, which is the reading the whole instrument is + * meant to make trustworthy). + */ +test("the facade observes the standalone GET tunnel: open with its age, whether it delivered, and closed when it ends", async () => { + const rig = await startRig(); + const tunnels = (): EngineBrokerMcpTunnelObservation | undefined => rig.facade.observe(rig.turnId)?.tunnels; + const until = async (reason: string, ready: () => boolean): Promise => { + for (let attempt = 0; attempt < 200 && !ready(); attempt += 1) await new Promise((resolve) => setTimeout(resolve, 25)); + assert.ok(ready(), reason); + }; + try { + // Before any request the turn is registered and has relayed nothing: a + // measured zero, which is not the same statement as an open tunnel. + assert.deepEqual(tunnels(), { opened: 0, closed: 0, delivered: 0, open: [] }); + const { client } = await connectClient(rig.capability); + const notified = new Promise((resolve) => client.setNotificationHandler(ToolListChangedNotificationSchema, () => resolve())); + try { + await until("the client never opened its GET tunnel", () => (tunnels()?.open.length ?? 0) > 0); + const open = tunnels(); + assert.equal(open?.opened, 1); assert.equal(open?.closed, 0); + assert.ok((open?.open[0]?.openMs ?? -1) >= 0, "an open tunnel reports how long it has been open"); + assert.equal(open?.open[0]?.delivered, false, "a tunnel the mount has not pushed through is held open in silence"); + assert.equal(open?.delivered, 0); + + // The same tunnel, now actually carrying a server frame. "Held open + // having delivered nothing" and "in use" are different facts about it. + rig.server.sendToolListChanged(); + await withDeadline(notified, 4_000, "no server notification reached the client"); + await until("the delivered frame was never attributed to the tunnel", () => (tunnels()?.delivered ?? 0) === 1); + assert.equal(tunnels()?.open[0]?.delivered, true); + assert.equal(tunnels()?.closed, 0, "a tunnel that delivered is still open"); + } finally { + await client.close(); + } + await until("the closed GET tunnel still reads as open", () => (tunnels()?.closed ?? 0) === 1); + assert.deepEqual(tunnels(), { opened: 1, closed: 1, delivered: 1, open: [] }); + } finally { + await rig.close(); + } +}); From 4f4922a1212298af2743dd0ba1ae2bf88b46a112 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 15:00:25 +0200 Subject: [PATCH 118/124] feat: count and seal the MCP facade's refused requests by reason class --- src/runtime/AGENTS.md | 27 +++++++- src/runtime/engineBrokerCapabilities.ts | 22 ++++++ src/runtime/engineBrokerControlClient.ts | 6 +- src/runtime/engineBrokerMcpCallLog.test.ts | 50 +++++++++++++- src/runtime/engineBrokerMcpCallLog.ts | 42 +++++++++++- src/runtime/engineBrokerMcpFacade.ts | 62 ++++++++++++++--- .../engineBrokerMcpObservation.test.ts | 68 +++++++++++++++++++ src/runtime/engineBrokerProtocol.test.ts | 19 ++++++ src/runtime/engineBrokerProtocol.ts | 23 ++++++- src/runtime/engineBrokerSealLedger.test.ts | 27 ++++++++ src/runtime/engineBrokerSealLedger.ts | 9 ++- 11 files changed, 331 insertions(+), 24 deletions(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index db88d50..20b4e62 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -332,6 +332,25 @@ out or refuses a tunnel, because an instrument that tore the stream down would destroy the evidence it exists to gather. The member is optional on the wire for one reason — a turn sealed before it existed must still replay — so its absence means "not measured" and never zero, exactly as `mcp`'s own absence does. +A request the facade *refuses* is the sharpest form of the same silence, and +it used to observe as nothing at all: `route()` threw before `calls.begin`, so +a turn 403'd on every request sealed `answered == started, outstanding: []` — +byte-identical to a healthy turn. `EngineBrokerMcpCallLog.refuse` now counts +each one by a closed reason class (`route`, `expired`, `exhausted`, +`unrouted`, `oversized`), because the classes call for opposite fixes: an +exhausted per-turn capability is a budget, an unserved route is a worker +asking for something that does not exist. The budget is reachable rather than +theoretical — `ENGINE_BROKER_MCP_CAPABILITY_REQUESTS` (128) covers every POST, +the GET tunnel and the DELETE, and a `search_tool`+`use_tool` round spends two, +so a 48-round wake asks for ~96 plus its handshake. Attribution comes from +`EngineBrokerCapabilities.classifyToken`, which names the token's turn and why +it would be refused *without spending its budget*; a bearer no grant matches +names no turn and stays unattributed, because guessing an owner would be +inventing the measurement. Counts only: never the token, the capability, the +URL or the body. The member is optional on the wire for `tunnels`' one reason, +and reaches the operator as `mcp_refused=exhausted:41` and the seal row's +`mcp.refusals`. + Measured against the real CLI (rig, grok 1.0.34, real facade and mount): the tunnel opens ~3 ms after `initialize`, carries nothing for its whole life, and **closes 16 ms before the worker exits** — the close is the worker's own @@ -353,8 +372,12 @@ terminal turn (`turns.jsonl`, `engineBrokerSealLedgerPathFor`), rendered from the sealed response and nothing else. Its members are the accounting, the failure `code`, the diagnostic's closed `status`/`stage`/`failure_class` with the reason `engineBrokerNativeClient.ts` already redacted and bounded, and the -facade's `mcp` observation — names, counts, GET-tunnel lifecycle and elapsed -milliseconds. Never a +facade's `mcp` observation — names, counts, refusals by reason class, +GET-tunnel lifecycle and elapsed milliseconds. That projection is a closed +allow-list and `engineBrokerSealLedger.test.ts` asserts the *exact key set* of +a written row for a completed turn: replacing it with `...terminal` writes +`usage`, `diagnostic`, `mcpCalls` and the model's entire reply into the +ledger, and that mutation is what the assertion exists to catch. Never a prompt, body, reply, bearer, capability or session id; the terminal response carries none of those in the first place, and the projection is an allow-list rather than a spread, so a future additive member of the response cannot become diff --git a/src/runtime/engineBrokerCapabilities.ts b/src/runtime/engineBrokerCapabilities.ts index 0bbd102..1f99f8a 100644 --- a/src/runtime/engineBrokerCapabilities.ts +++ b/src/runtime/engineBrokerCapabilities.ts @@ -23,6 +23,28 @@ export class EngineBrokerCapabilities { } return undefined; } + /** + * Which grant a token names and why it would be refused, *without* spending + * it. + * + * The facade needs this on its refusal path alone. A 403 it cannot attribute + * to a turn is a 403 that seals as nothing at all, and a turn whose every + * request was refused then reads exactly like a healthy one — the silence + * `engineBrokerMcpCallLog.ts` exists to end. A token no grant matches names + * no turn and stays unattributed; nothing here returns the token, the grant + * or the agent's capability, only the turn id and a closed reason. + */ + classifyToken(token: string): Readonly<{ turnId: string; state: "live" | "expired" | "exhausted" }> | undefined { + const candidate = hash(token); + for (const grant of this.grants.values()) { + if (!timingSafeEqual(grant.digest, candidate)) continue; + // Budget before expiry: the TTL outlives every declared turn limit, so an + // exhausted grant is the reachable refusal and the actionable answer. + if (grant.requests >= grant.maxRequests) return { turnId: grant.turnId, state: "exhausted" }; + return { turnId: grant.turnId, state: grant.expiresAt <= Date.now() ? "expired" : "live" }; + } + return undefined; + } inspectToken(token:string):Readonly<{agentId:string;turnId:string}>|undefined{const candidate=hash(token);for(const grant of this.grants.values()){if(grant.expiresAt>Date.now()&&grant.requestscalls===undefined?"":`; mcp=${calls.answered}/${calls.started} answered${calls.undecoded===0?"":`; mcp_undecoded=${calls.undecoded}`}${calls.outstanding.length===0?"":`; mcp_outstanding=${calls.outstanding.map((call)=>`${call.name}@${call.outstandingMs}ms`).join(",")}`}${renderMcpTunnels(calls.tunnels)}`; +const renderMcpCalls=(calls:EngineBrokerMcpCallObservation|undefined):string=>calls===undefined?"":`; mcp=${calls.answered}/${calls.started} answered${calls.undecoded===0?"":`; mcp_undecoded=${calls.undecoded}`}${calls.outstanding.length===0?"":`; mcp_outstanding=${calls.outstanding.map((call)=>`${call.name}@${call.outstandingMs}ms`).join(",")}`}${renderMcpRefusals(calls.refusals)}${renderMcpTunnels(calls.tunnels)}`; +/** The refusals the facade never relayed, by reason class, and only the classes that happened: a turn refused 403 must not read as a turn that was served. */ +const renderMcpRefusals=(refusals:EngineBrokerMcpRefusalObservation|undefined):string=>{if(refusals===undefined)return"";const named=ENGINE_BROKER_MCP_REFUSAL_REASONS.filter((reason)=>refusals[reason]>0);return named.length===0?"":`; mcp_refused=${named.map((reason)=>`${reason}:${refusals[reason]}`).join(",")}`;}; /** The session's GET SSE tunnels: how many closed of how many opened, and each one still open with its age and whether the mount ever pushed through it. */ const renderMcpTunnels=(tunnels:EngineBrokerMcpTunnelObservation|undefined):string=>tunnels===undefined?"":`; mcp_get=${tunnels.closed}/${tunnels.opened} closed, ${tunnels.delivered} delivered${tunnels.open.length===0?"":`; mcp_get_open=${tunnels.open.map((tunnel)=>`${tunnel.openMs}ms/${tunnel.delivered?"delivered":"silent"}`).join(",")}`}`; diff --git a/src/runtime/engineBrokerMcpCallLog.test.ts b/src/runtime/engineBrokerMcpCallLog.test.ts index 7b0ab26..dbf51ac 100644 --- a/src/runtime/engineBrokerMcpCallLog.test.ts +++ b/src/runtime/engineBrokerMcpCallLog.test.ts @@ -9,6 +9,8 @@ const call = (name: unknown, id = 1): Buffer => body({ jsonrpc: "2.0", id, metho /** A controllable clock: an outstanding call's whole value is how long it has been outstanding. */ /** An observed turn whose facade never relayed a GET tunnel — a measurement, not an absence. */ const NO_TUNNEL = { opened: 0, closed: 0, delivered: 0, open: [] }; +/** An observed turn the facade never refused — also a measurement, and not the same statement as a turn it never observed. */ +const NO_REFUSALS = { route: 0, expired: 0, exhausted: 0, unrouted: 0, oversized: 0 }; const clock = (): { log: EngineBrokerMcpCallLog; advance: (ms: number) => void } => { let at = 1_000; @@ -36,12 +38,12 @@ test("absence stays absence: an unopened turn observes undefined, a turn that ca const { log } = clock(); assert.equal(log.observe("turn"), undefined); log.open("turn"); - assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 0, outstanding: [], tunnels: NO_TUNNEL }); + assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 0, outstanding: [], refusals: NO_REFUSALS, tunnels: NO_TUNNEL }); // Everything that is not a tool call records nothing at all, so `started` // stays a count of tool calls and not of traffic. for (const value of [body({ jsonrpc: "2.0", id: 1, method: "tools/list" }), body({ jsonrpc: "2.0", method: "notifications/initialized" }), Buffer.alloc(0)]) log.begin("turn", value).answer(); log.begin("turn", undefined).answer(); - assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 0, outstanding: [], tunnels: NO_TUNNEL }); + assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 0, outstanding: [], refusals: NO_REFUSALS, tunnels: NO_TUNNEL }); log.close("turn"); assert.equal(log.observe("turn"), undefined, "a revoked turn keeps nothing"); }); @@ -51,7 +53,7 @@ test("a body the facade could not read counts as undecoded, never as a call with log.open("turn"); log.begin("turn", Buffer.from("{not json", "utf8")); log.undecodable("turn"); - assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 2, outstanding: [], tunnels: NO_TUNNEL }); + assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 2, outstanding: [], refusals: NO_REFUSALS, tunnels: NO_TUNNEL }); log.undecodable("absent"); }); @@ -144,3 +146,45 @@ test("the open-tunnel list is bounded, and the counts still name every tunnel be assert.equal(tunnels?.open.length, ENGINE_BROKER_MCP_TUNNEL_MAX); assert.equal(tunnels?.open[0]?.openMs, ENGINE_BROKER_MCP_TUNNEL_MAX + 3, "the earliest tunnels are the ones kept"); }); + +/** + * The refusal is the sharpest form of the silence this log exists to end. + * + * A refused request never reaches the relay, so a turn every one of whose + * requests was 403'd observed as `started: 0, answered: 0, outstanding: []` — + * the same three numbers a turn that simply had nothing to call observes. The + * boundary these assertions straddle is that one: a turn refused against a + * turn served, and an exhausted capability budget against a route the facade + * does not serve, because the two call for opposite fixes. + * + * Mutation: drop the `refusals` member from `observe`, or make `refuse` a + * no-op, and a refused turn reads as an idle one again. + */ +test("a refused request is counted by reason class, and refusing is not calling", () => { + const { log } = clock(); + log.open("turn"); + assert.deepEqual(log.observe("turn")?.refusals, NO_REFUSALS, "an observed turn that was never refused measured zero"); + log.refuse("turn", "exhausted"); log.refuse("turn", "exhausted"); log.refuse("turn", "route"); + const observed = log.observe("turn"); + assert.deepEqual(observed?.refusals, { ...NO_REFUSALS, exhausted: 2, route: 1 }); + // The reading the instrument has to keep honest: a refused turn is not a + // turn that called nothing and answered everything. + assert.deepEqual([observed?.started, observed?.answered, observed?.outstanding], [0, 0, []]); + // Every reason class is its own count, and the observation is a copy: a + // later refusal cannot rewrite a report already handed out. + log.refuse("turn", "unrouted"); + assert.deepEqual(observed?.refusals, { ...NO_REFUSALS, exhausted: 2, route: 1 }); + assert.deepEqual(log.observe("turn")?.refusals, { ...NO_REFUSALS, exhausted: 2, route: 1, unrouted: 1 }); +}); + +test("a refusal the facade cannot attribute is recorded against no turn at all", () => { + const { log } = clock(); + log.refuse("absent", "expired"); + assert.equal(log.observe("absent"), undefined, "an unopened turn keeps nothing"); + log.open("absent"); + assert.deepEqual(log.observe("absent")?.refusals, NO_REFUSALS, "a refusal before the turn existed is not this turn's"); + log.refuse("absent", "oversized"); + log.close("absent"); + log.open("absent"); + assert.deepEqual(log.observe("absent")?.refusals, NO_REFUSALS, "re-opening an id resets its refusals with everything else"); +}); diff --git a/src/runtime/engineBrokerMcpCallLog.ts b/src/runtime/engineBrokerMcpCallLog.ts index b680c48..6654102 100644 --- a/src/runtime/engineBrokerMcpCallLog.ts +++ b/src/runtime/engineBrokerMcpCallLog.ts @@ -55,6 +55,21 @@ * the turn's death must not retroactively mark the call it was blocked on as * finished. */ +/** + * The same map carries the facade's *refusals*, for the sharpest form of the + * same problem. A refused request never reaches the relay at all, so a turn + * every one of whose requests was 403'd sealed as `answered == started, + * outstanding: []` — byte-identical to a healthy turn, which is precisely the + * reading this instrument exists to make trustworthy. The reason class is what + * makes it actionable: an exhausted per-turn capability budget (a worker's + * `search_tool`+`use_tool` pair per round is two requests of the facade's + * `ENGINE_BROKER_MCP_CAPABILITY_REQUESTS`) is a different fault from a + * mount that was never registered, and both differ from a worker asking for a + * route the facade does not serve. Counts only, keyed by a closed vocabulary: + * never the token, the capability, the URL or the body. A refusal the facade + * cannot attribute to a turn — a bearer no live grant matches — is recorded + * nowhere, because attributing it to a turn would be inventing the fact. + */ export const ENGINE_BROKER_MCP_OUTSTANDING_MAX = 16; export const ENGINE_BROKER_MCP_CALL_INVALID = ""; export const ENGINE_BROKER_MCP_CALL_TRUNCATED = ""; @@ -65,6 +80,19 @@ const TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/u; /** Open GET tunnels reported: a session opens one, so more than a handful is already the anomaly. */ export const ENGINE_BROKER_MCP_TUNNEL_MAX = 8; +/** + * Why the facade refused one relayed request, as a closed vocabulary: + * - `route`: a path or method the facade does not serve. + * - `expired`: the turn capability's TTL had passed. + * - `exhausted`: the turn capability's request budget was spent. + * - `unrouted`: a live capability whose turn has no registered mount. + * - `oversized`: a POST body past the facade's own request bound. + */ +export const ENGINE_BROKER_MCP_REFUSAL_REASONS = ["route", "expired", "exhausted", "unrouted", "oversized"] as const; +export type EngineBrokerMcpRefusalReason = (typeof ENGINE_BROKER_MCP_REFUSAL_REASONS)[number]; +/** How many requests of this turn the facade refused, by reason class. Every member is a measurement; the whole member is absent only where it was never measured. */ +export type EngineBrokerMcpRefusalObservation = Readonly>; + export type EngineBrokerOutstandingMcpCall = Readonly<{ name: string; outstandingMs: number }>; /** One GET SSE tunnel still open at observation: how long it has been open, and whether the mount ever pushed through it. */ export type EngineBrokerOpenMcpTunnel = Readonly<{ openMs: number; delivered: boolean }>; @@ -79,7 +107,7 @@ export type EngineBrokerMcpTunnelObservation = Readonly<{ opened: number; closed * facade answered, how many POST bodies it could not read, and the ones still * unanswered with the time each has been outstanding. */ -export type EngineBrokerMcpCallObservation = Readonly<{ started: number; answered: number; undecoded: number; outstanding: readonly EngineBrokerOutstandingMcpCall[]; tunnels?: EngineBrokerMcpTunnelObservation }>; +export type EngineBrokerMcpCallObservation = Readonly<{ started: number; answered: number; undecoded: number; outstanding: readonly EngineBrokerOutstandingMcpCall[]; tunnels?: EngineBrokerMcpTunnelObservation; refusals?: EngineBrokerMcpRefusalObservation }>; /** One relayed POST's calls. `answer` marks a complete relay; `close` ends it unanswered. Both are idempotent. */ export interface EngineBrokerMcpCallHandle { answer(): void; close(): void } @@ -93,14 +121,16 @@ type TunnelRecord = { readonly openedAt: number; delivered: boolean }; type TurnLog = { started: number; answered: number; undecoded: number; readonly live: Set; readonly ended: CallRecord[]; tunnelsOpened: number; tunnelsClosed: number; tunnelsDelivered: number; readonly openTunnels: Set; + readonly refusals: Record; }; +const noRefusals = (): Record => ({ route: 0, expired: 0, exhausted: 0, unrouted: 0, oversized: 0 }); export class EngineBrokerMcpCallLog { private readonly turns = new Map(); constructor(private readonly now: () => number = Date.now) {} /** A turn the facade registered. Re-opening an id resets it: a turn id is unique per turn. */ - open(turnId: string): void { this.turns.set(turnId, { started: 0, answered: 0, undecoded: 0, live: new Set(), ended: [], tunnelsOpened: 0, tunnelsClosed: 0, tunnelsDelivered: 0, openTunnels: new Set() }); } + open(turnId: string): void { this.turns.set(turnId, { started: 0, answered: 0, undecoded: 0, live: new Set(), ended: [], tunnelsOpened: 0, tunnelsClosed: 0, tunnelsDelivered: 0, openTunnels: new Set(), refusals: noRefusals() }); } close(turnId: string): void { this.turns.delete(turnId); } /** A POST body the facade is about to relay. Anything that is not a `tools/call` records nothing. */ @@ -152,6 +182,13 @@ export class EngineBrokerMcpCallLog { }; } + /** + * A request the facade refused before it could ever be relayed. A refusal it + * cannot attribute to a turn is never recorded against one, so an unknown + * turn is a no-op here exactly as every other operation is. + */ + refuse(turnId: string, reason: EngineBrokerMcpRefusalReason): void { const log = this.turns.get(turnId); if (log !== undefined) log.refusals[reason] += 1; } + /** A POST whose body the facade refused to read (over its own bound), which is a call it cannot name. */ undecodable(turnId: string): void { const log = this.turns.get(turnId); if (log !== undefined) log.undecoded += 1; } @@ -167,6 +204,7 @@ export class EngineBrokerMcpCallLog { .map((record): EngineBrokerOpenMcpTunnel => ({ openMs: Math.max(0, at - record.openedAt), delivered: record.delivered })); return { started: log.started, answered: log.answered, undecoded: log.undecoded, + refusals: { ...log.refusals }, tunnels: { opened: log.tunnelsOpened, closed: log.tunnelsClosed, delivered: log.tunnelsDelivered, open }, outstanding: outstanding.length > ENGINE_BROKER_MCP_OUTSTANDING_MAX ? [...outstanding.slice(0, ENGINE_BROKER_MCP_OUTSTANDING_MAX - 1), { name: ENGINE_BROKER_MCP_CALL_TRUNCATED, outstandingMs: 0 }] diff --git a/src/runtime/engineBrokerMcpFacade.ts b/src/runtime/engineBrokerMcpFacade.ts index 51e4e0e..44eccfe 100644 --- a/src/runtime/engineBrokerMcpFacade.ts +++ b/src/runtime/engineBrokerMcpFacade.ts @@ -1,7 +1,7 @@ import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import { Readable } from "node:stream"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; -import { EngineBrokerMcpCallLog, type EngineBrokerMcpCallObservation, type EngineBrokerMcpTunnelHandle } from "./engineBrokerMcpCallLog.js"; +import { EngineBrokerMcpCallLog, type EngineBrokerMcpCallObservation, type EngineBrokerMcpRefusalReason, type EngineBrokerMcpTunnelHandle } from "./engineBrokerMcpCallLog.js"; /** * The brokered worker's only route to its own per-wake Daimon MCP mount. The @@ -46,6 +46,18 @@ const FORWARDED_RESPONSE_HEADERS = ["content-type", "mcp-session-id", "mcp-proto const FORWARDED_METHODS = new Set(["POST", "GET", "DELETE"]); const MAX_REQUEST_BYTES = 1024 * 1024; export const ENGINE_BROKER_MCP_FACADE_PORT = 43_124; +/** + * Requests one turn capability may spend, across all three methods. + * + * A worker's round is a `search_tool` and a `use_tool`, so a 48-turn wake is + * ~96 POSTs plus the handshake, the standalone GET tunnel and the closing + * DELETE: exhaustion is reachable rather than theoretical, and every request + * past it is a 403 the worker cannot explain. That is why the refusal is + * counted and sealed (`engineBrokerMcpCallLog.ts`) rather than being an + * absence in the turn's row. + */ +export const ENGINE_BROKER_MCP_CAPABILITY_REQUESTS = 128; +export const ENGINE_BROKER_MCP_CAPABILITY_TTL_MS = 15 * 60_000; class FacadeRefusal extends Error {} @@ -71,15 +83,37 @@ export async function startEngineBrokerMcpFacade() { }); }); + /** + * A 403 the call log can read, whenever the bearer names a turn. + * + * The refusal itself is unchanged — same status, same body, same silence + * towards the worker — but it is attributed first, through a lookup that + * does not spend the capability's budget. A bearer no grant matches names no + * turn and stays unattributed, because guessing whose it was would be + * inventing the measurement. A capability that is *also* spent or expired + * reports that instead of the route it asked for: it is the fault the + * operator can act on. + */ + function refuse(bearer: string | undefined, live: EngineBrokerMcpRefusalReason): FacadeRefusal { + const classified = bearer === undefined ? undefined : capabilities.classifyToken(bearer); + if (classified !== undefined) calls.refuse(classified.turnId, classified.state === "live" ? live : classified.state); + return new FacadeRefusal(); + } + async function route(request: IncomingMessage, response: ServerResponse): Promise { const method = request.method ?? ""; - if (request.url !== "/mcp" || !FORWARDED_METHODS.has(method)) throw new FacadeRefusal(); - const match = request.headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u); - if (!match) throw new FacadeRefusal(); - const scope = capabilities.authorizeToken(match[1]!); - if (!scope) throw new FacadeRefusal(); + const bearer = request.headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u)?.[1]; + if (request.url !== "/mcp" || !FORWARDED_METHODS.has(method)) throw refuse(bearer, "route"); + if (bearer === undefined) throw new FacadeRefusal(); + const scope = capabilities.authorizeToken(bearer); + // A live grant that authorizes is the only way past here; anything else is + // classified so an exhausted budget and an expired TTL reach the turn's + // sealed row as themselves. `live` cannot be the answer on this path — + // `authorizeToken` read the same grant a moment ago, expiry only moves + // forward and a budget only spends — so it is the unreachable arm. + if (!scope) throw refuse(bearer, "expired"); const target = targets.get(scope.turnId); - if (target === undefined) throw new FacadeRefusal(); + if (target === undefined) { calls.refuse(scope.turnId, "unrouted"); throw new FacadeRefusal(); } // Only POST carries a JSON-RPC body; drain anything else so the socket // never stalls waiting for a body the facade will not forward. @@ -87,7 +121,13 @@ export async function startEngineBrokerMcpFacade() { // A body refused for size is a call the log can never name, and counting // it keeps "no tool call started" an honest reading rather than a gap. try { body = method === "POST" ? await bounded(request) : (request.resume(), undefined); } - catch (error) { calls.undecodable(scope.turnId); throw error; } + catch (error) { + calls.undecodable(scope.turnId); + // A body past the bound is refused, not merely unreadable: the worker + // gets a 403 for it, so it is counted as one too. + if (error instanceof FacadeRefusal) calls.refuse(scope.turnId, "oversized"); + throw error; + } const payload = body === undefined ? undefined : (body.buffer.slice(body.byteOffset, body.byteOffset + body.byteLength) as ArrayBuffer); const call = calls.begin(scope.turnId, body); // The GET SSE tunnel is the session's other channel and the one that @@ -133,7 +173,7 @@ export async function startEngineBrokerMcpFacade() { if (targets.has(turnId)) throw new Error("MCP turn already registered"); targets.set(turnId, url.href); calls.open(turnId); - return capabilities.issue(agentId, turnId, 15 * 60_000, 128); + return capabilities.issue(agentId, turnId, ENGINE_BROKER_MCP_CAPABILITY_TTL_MS, ENGINE_BROKER_MCP_CAPABILITY_REQUESTS); }, revoke(turnId: string): void { targets.delete(turnId); @@ -142,8 +182,8 @@ export async function startEngineBrokerMcpFacade() { calls.close(turnId); }, /** - * What the facade saw of this turn's tool calls and GET tunnels, or - * `undefined` for a turn + * What the facade saw of this turn's tool calls, GET tunnels and refusals, + * or `undefined` for a turn * it never registered. Read on the failure path, before `revoke`. */ observe: (turnId: string): EngineBrokerMcpCallObservation | undefined => calls.observe(turnId), diff --git a/src/runtime/engineBrokerMcpObservation.test.ts b/src/runtime/engineBrokerMcpObservation.test.ts index b70a6f2..fc2b42d 100644 --- a/src/runtime/engineBrokerMcpObservation.test.ts +++ b/src/runtime/engineBrokerMcpObservation.test.ts @@ -6,6 +6,7 @@ import test from "node:test"; import { ToolListChangedNotificationSchema } from "@modelcontextprotocol/sdk/types.js"; import type { EngineBrokerMcpTunnelObservation } from "./engineBrokerMcpCallLog.js"; +import { ENGINE_BROKER_MCP_CAPABILITY_REQUESTS, ENGINE_BROKER_MCP_FACADE_PORT } from "./engineBrokerMcpFacade.js"; import { connectClient, FACADE_URL, sharedFacade, startRig, withDeadline } from "./engineBrokerMcpFacadeRig.test.js"; /** @@ -153,3 +154,70 @@ test("the facade observes the standalone GET tunnel: open with its age, whether await rig.close(); } }); + +/** + * The refusal, which was the one relay outcome that observed as nothing. + * + * `route()` refuses before the call log is ever touched, so a request the + * facade 403'd never reached `started`, `undecoded` or `outstanding`: a turn + * whose every request was refused sealed as `answered == started, + * outstanding: []`, which is exactly what a healthy turn seals as. The budget + * makes that reachable rather than theoretical — one capability buys + * {@link ENGINE_BROKER_MCP_CAPABILITY_REQUESTS} requests across all three + * methods, and a worker spends two of them per round. + * + * The boundary these assertions straddle: a turn served against a turn + * refused, and within the refusals, a spent capability against a route the + * facade does not serve — the first is a budget to raise, the second is a + * worker asking for something that does not exist. + * + * Mutation: restore `throw new FacadeRefusal()` in place of either `refuse` + * call in `route()`, and a 403'd turn reads as an idle one again. + */ +test("a request the facade refused is counted against its turn, by reason, and is never a call it served", async () => { + const facade = await sharedFacade(); + let served = 0; + const target = createServer((request, response) => { + served += 1; + request.resume(); + request.on("end", () => { response.writeHead(200, { "content-type": "application/json" }); response.end('{"jsonrpc":"2.0","id":1,"result":{}}'); }); + }); + await new Promise((resolve) => target.listen(0, "127.0.0.1", resolve)); + const address = target.address(); if (address === null || typeof address === "string") throw new Error(); + const turnId = "turn-refused", token = facade.register("agent", turnId, `http://127.0.0.1:${address.port}/mcp`); + const listed = JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list", params: {} }); + const send = async (url: string, bearer: string, method = "POST"): Promise => { + const response = await fetch(url, { method, headers: { authorization: `Bearer ${bearer}`, "content-type": "application/json" }, ...(method === "POST" ? { body: listed } : {}) }); + await response.text(); + return response.status; + }; + try { + // A route the facade does not serve, asked for with a live capability. + assert.equal(await send(`http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}/nope`, token), 403); + assert.equal(await send(FACADE_URL, token, "PUT"), 403); + assert.deepEqual(facade.observe(turnId)?.refusals, { route: 2, expired: 0, exhausted: 0, unrouted: 0, oversized: 0 }); + + // A bearer no live grant matches names no turn, so it is recorded against + // none: inventing an owner would be worse than the silence. + assert.equal(await send(FACADE_URL, "wrong-token-abcdefghijklmnopqrstuvwxyz0123456789"), 403); + assert.equal(facade.observe(turnId)?.refusals?.route, 2, "an unattributable refusal belongs to no turn"); + + // Neither refusal spent the capability, so the budget is exactly what was + // issued — and spending it all is reachable: a 48-round worker asks for + // ~96 of these plus its handshake, tunnel and DELETE. + for (let spent = 0; spent < ENGINE_BROKER_MCP_CAPABILITY_REQUESTS; spent += 1) assert.equal(await send(FACADE_URL, token), 200); + assert.equal(served, ENGINE_BROKER_MCP_CAPABILITY_REQUESTS); + assert.equal(await send(FACADE_URL, token), 403, "the capability's budget is spent"); + assert.equal(served, ENGINE_BROKER_MCP_CAPABILITY_REQUESTS, "an exhausted capability never reaches the mount"); + + const observed = facade.observe(turnId); + assert.deepEqual(observed?.refusals, { route: 2, expired: 0, exhausted: 1, unrouted: 0, oversized: 0 }); + // And the reading the seal row used to publish for all of it: nothing. + assert.deepEqual([observed?.started, observed?.answered, observed?.undecoded, observed?.outstanding], [0, 0, 0, []]); + assert.ok(!JSON.stringify(observed).includes(token), "counts and reason classes only: never the capability"); + } finally { + facade.revoke(turnId); + target.closeAllConnections(); + await new Promise((resolve) => target.close(() => resolve())); + } +}); diff --git a/src/runtime/engineBrokerProtocol.test.ts b/src/runtime/engineBrokerProtocol.test.ts index 835493c..8d59256 100644 --- a/src/runtime/engineBrokerProtocol.test.ts +++ b/src/runtime/engineBrokerProtocol.test.ts @@ -124,4 +124,23 @@ test("a failed frame carries the broker's in-flight MCP tool-call observation, b { ...tunnels, open: [{ openMs: 1, delivered: false, sessionId: "mcp-session-0" }] }, { opened: 1, closed: 0, open: [] } ]) assert.throws(() => parseEngineBrokerResponse({ ...value, mcpCalls: { ...value.mcpCalls, tunnels: forged } }), /invalid broker frame/u, JSON.stringify(forged)); + + /** + * The refusals ride the same member under the same rules, and they are the + * counts that make a 403'd turn readable: without them a turn the facade + * refused every request of publishes `started: 0, answered: 0` — an idle + * turn's numbers. Every reason class is its own measurement, so a partial + * member is refused rather than zero-filled, and the whole member is optional + * for the one reason `tunnels` is. + */ + const refusals = { route: 1, expired: 0, exhausted: 41, unrouted: 0, oversized: 2 } as const; + const refused = { ...value, mcpCalls: { ...value.mcpCalls, refusals } } as const; + assert.deepEqual(parseEngineBrokerResponse(refused), refused); + for (const forged of [ + { ...refusals, exhausted: -1 }, + { ...refusals, exhausted: 1.5 }, + { ...refusals, exhausted: "41" }, + { route: 1, expired: 0, unrouted: 0, oversized: 0 }, + { ...refusals, capability: 3 } + ]) assert.throws(() => parseEngineBrokerResponse({ ...value, mcpCalls: { ...value.mcpCalls, refusals: forged } }), /invalid broker frame/u, JSON.stringify(forged)); }); diff --git a/src/runtime/engineBrokerProtocol.ts b/src/runtime/engineBrokerProtocol.ts index 201d005..f5e37b4 100644 --- a/src/runtime/engineBrokerProtocol.ts +++ b/src/runtime/engineBrokerProtocol.ts @@ -1,5 +1,5 @@ import { isEngineBrokerInferenceRequestKind, isEngineBrokerInferenceResponseKind, parseEngineBrokerInferenceRequest, parseEngineBrokerInferenceResponse, type EngineBrokerInferenceRequest, type EngineBrokerInferenceResponse } from "./engineBrokerInferenceProtocol.js"; -import { ENGINE_BROKER_MCP_CALL_NAME, ENGINE_BROKER_MCP_OUTSTANDING_MAX, ENGINE_BROKER_MCP_TUNNEL_MAX, type EngineBrokerMcpCallObservation, type EngineBrokerMcpTunnelObservation } from "./engineBrokerMcpCallLog.js"; +import { ENGINE_BROKER_MCP_CALL_NAME, ENGINE_BROKER_MCP_OUTSTANDING_MAX, ENGINE_BROKER_MCP_REFUSAL_REASONS, ENGINE_BROKER_MCP_TUNNEL_MAX, type EngineBrokerMcpCallObservation, type EngineBrokerMcpRefusalObservation, type EngineBrokerMcpTunnelObservation } from "./engineBrokerMcpCallLog.js"; import { parseEngineBrokerTurnAccounting, parseEngineBrokerTurnLimitOverrides, type EngineBrokerTurnAccounting, type EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; /** @@ -146,7 +146,7 @@ function parseMcpCallObservation(value: unknown): EngineBrokerMcpCallObservation // `tunnels` is optional for one reason only: a turn sealed before the GET // tunnel was observed carries no such member, and its record must still // replay. Absence there means "the instrument did not exist", never zero. - exact(input, ["started", "answered", "undecoded", "outstanding", ...(input.tunnels === undefined ? [] : ["tunnels"])]); + exact(input, ["started", "answered", "undecoded", "outstanding", ...(input.tunnels === undefined ? [] : ["tunnels"]), ...(input.refusals === undefined ? [] : ["refusals"])]); const started = input.started, answered = input.answered, undecoded = input.undecoded; if (![started, answered, undecoded].every((count) => Number.isSafeInteger(count) && (count as number) >= 0)) throw new TypeError("invalid broker frame"); if (!Array.isArray(input.outstanding) || input.outstanding.length > ENGINE_BROKER_MCP_OUTSTANDING_MAX) throw new TypeError("invalid broker frame"); @@ -159,7 +159,24 @@ function parseMcpCallObservation(value: unknown): EngineBrokerMcpCallObservation }); if ((answered as number) > (started as number) || outstanding.length > (started as number) - (answered as number)) throw new TypeError("invalid broker frame"); const tunnels = input.tunnels === undefined ? undefined : parseMcpTunnelObservation(input.tunnels); - return { started: started as number, answered: answered as number, undecoded: undecoded as number, outstanding, ...(tunnels === undefined ? {} : { tunnels }) }; + const refusals = input.refusals === undefined ? undefined : parseMcpRefusalObservation(input.refusals); + return { started: started as number, answered: answered as number, undecoded: undecoded as number, outstanding, ...(tunnels === undefined ? {} : { tunnels }), ...(refusals === undefined ? {} : { refusals }) }; +} + +/** + * The refused requests, by reason class: one count per closed reason, all of + * them required once the member is present. Optional for the same single + * reason `tunnels` is — a turn sealed before the facade counted its refusals + * must still replay — so its absence means "not measured" and never zero, and + * a partial member is refused rather than zero-filled. + */ +function parseMcpRefusalObservation(value: unknown): EngineBrokerMcpRefusalObservation { + const input = record(value); + exact(input, ENGINE_BROKER_MCP_REFUSAL_REASONS); + for (const reason of ENGINE_BROKER_MCP_REFUSAL_REASONS) { + if (!Number.isSafeInteger(input[reason]) || (input[reason] as number) < 0) throw new TypeError("invalid broker frame"); + } + return Object.fromEntries(ENGINE_BROKER_MCP_REFUSAL_REASONS.map((reason) => [reason, input[reason] as number])) as EngineBrokerMcpRefusalObservation; } /** diff --git a/src/runtime/engineBrokerSealLedger.test.ts b/src/runtime/engineBrokerSealLedger.test.ts index 60754e3..a609601 100644 --- a/src/runtime/engineBrokerSealLedger.test.ts +++ b/src/runtime/engineBrokerSealLedger.test.ts @@ -158,3 +158,30 @@ test("a cancelled turn's GET tunnel is sealed open with its age, closed, or neve const unobserved = await cancelledTurn(() => ({ started: 1, answered: 1, undecoded: 0, outstanding: [] })); assert.equal(Object.hasOwn(unobserved.seal?.mcp as Record, "tunnels"), false); }); + +/** + * The refusal, on the same durable route as the hang it looks like. + * + * A request the facade 403'd never reached the relay, so before it was counted + * a turn whose capability was spent — 128 requests, two per worker round — + * sealed `answered == started, outstanding: []`, which is exactly what a + * healthy turn seals. The row has to carry the reason class, because an + * exhausted budget and an unserved route are opposite fixes. + * + * Mutation: drop the `refusals` member from `renderBrokerTurnSealLine` and the + * first assertion goes red; render it unconditionally as zeros for an + * observation that carries none, and the second does — a zero nobody measured + * reads exactly like a zero somebody did. + */ +test("a turn whose MCP requests were refused seals the refusals by reason, and a turn sealed before they were counted seals none", async () => { + const refused = await cancelledTurn(() => ({ + started: 0, answered: 0, undecoded: 0, outstanding: [], + refusals: { route: 0, expired: 0, exhausted: 41, unrouted: 1, oversized: 0 } + })); + assert.deepEqual((refused.seal?.mcp as Record).refusals, { route: 0, expired: 0, exhausted: 41, unrouted: 1, oversized: 0 }); + // Without it this row is `started: 0, answered: 0` — an idle turn's row. + assert.deepEqual([(refused.seal?.mcp as Record).started, (refused.seal?.mcp as Record).answered], [0, 0]); + + const unobserved = await cancelledTurn(() => ({ started: 1, answered: 1, undecoded: 0, outstanding: [] })); + assert.equal(Object.hasOwn(unobserved.seal?.mcp as Record, "refusals"), false); +}); diff --git a/src/runtime/engineBrokerSealLedger.ts b/src/runtime/engineBrokerSealLedger.ts index c386838..a1d119a 100644 --- a/src/runtime/engineBrokerSealLedger.ts +++ b/src/runtime/engineBrokerSealLedger.ts @@ -1,4 +1,4 @@ -import { ENGINE_BROKER_MCP_CALL_NAME, ENGINE_BROKER_MCP_OUTSTANDING_MAX, ENGINE_BROKER_MCP_TUNNEL_MAX } from "./engineBrokerMcpCallLog.js"; +import { ENGINE_BROKER_MCP_CALL_NAME, ENGINE_BROKER_MCP_OUTSTANDING_MAX, ENGINE_BROKER_MCP_REFUSAL_REASONS, ENGINE_BROKER_MCP_TUNNEL_MAX } from "./engineBrokerMcpCallLog.js"; import type { EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; import { TURN_USAGE_LEDGER, TURN_USAGE_MAX_IDENTIFIER_CHARS } from "./turnUsageLedger.js"; @@ -104,6 +104,13 @@ export const renderBrokerTurnSealLine = (terminal: EngineBrokerTerminalResponse, outstanding: terminal.mcpCalls.outstanding .slice(0, ENGINE_BROKER_MCP_OUTSTANDING_MAX) .map((call) => ({ name: ENGINE_BROKER_MCP_CALL_NAME.test(call.name) ? call.name : "", outstanding_ms: call.outstandingMs })), + // Every request the facade refused before it could relay it, by reason + // class. Without it a turn whose capability was spent — 128 requests, + // two per worker round — seals as `answered == started, outstanding: + // []`, which is what a healthy turn seals as. + ...(terminal.mcpCalls.refusals === undefined + ? {} + : { refusals: Object.fromEntries(ENGINE_BROKER_MCP_REFUSAL_REASONS.map((reason) => [reason, terminal.mcpCalls!.refusals![reason]])) }), // The session's standalone GET tunnel, absent for a turn sealed before // the facade observed that channel at all. ...(terminal.mcpCalls.tunnels === undefined From 35c9bd588fcd3d0d20d52aca1a40c54a3204be6b Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 15:01:04 +0200 Subject: [PATCH 119/124] test: pin the seal row's exact field set for a completed turn --- src/runtime/engineBrokerSealLedger.test.ts | 60 ++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/src/runtime/engineBrokerSealLedger.test.ts b/src/runtime/engineBrokerSealLedger.test.ts index a609601..d2a299f 100644 --- a/src/runtime/engineBrokerSealLedger.test.ts +++ b/src/runtime/engineBrokerSealLedger.test.ts @@ -185,3 +185,63 @@ test("a turn whose MCP requests were refused seals the refusals by reason, and a const unobserved = await cancelledTurn(() => ({ started: 1, answered: 1, undecoded: 0, outstanding: [] })); assert.equal(Object.hasOwn(unobserved.seal?.mcp as Record, "refusals"), false); }); + +/** + * The projection is an allow-list, and this is the assertion that makes it one. + * + * `renderBrokerTurnSealLine` copies a closed field set out of the sealed + * terminal response. Replacing that copy with `...terminal` passed every other + * test in this suite while writing `usage`, `diagnostic`, `mcpCalls` — and, for + * a completed turn, `text`: the model's entire reply, into a ledger whose whole + * rule is that it carries no prompt, body or reply. Nothing sealed a completed + * turn and read the file back, so nothing was watching the one row that + * carries a reply at all. + * + * The boundary: the exact key set of a written row, for the turn kind that has + * the most to leak. + * + * Mutation: spread the terminal into the row (`...terminal, v: ..., agent: ...`) + * and this goes red on both halves — the key set gains `text`, `kind`, + * `version`, `requestId`, `workerPid`, `workerUid`, `workerStartTime` and + * `usage`, and the reply itself appears in the file's bytes. + */ +const reply = "TANGERINE-7-IS-THE-MODELS-OWN-REPLY"; +const answered = (text: string): string => { + const session = "01a0ad21-a90f-7f71-8054-93fdb4334d6a"; + const usage = { input_tokens: 2_677, output_tokens: 92, cache_read_input_tokens: 2_816, cache_creation_input_tokens: 0 }; + return [ + { type: "system", subtype: "init", session_id: session }, + { type: "assistant", message: { id: "msg_0", type: "message", role: "assistant", model: "daimon-broker-grok", content: [{ type: "text", text }], stop_reason: "end_turn", usage }, parent_tool_use_id: null, session_id: session }, + { type: "result", subtype: "success", is_error: false, num_turns: 1, result: text, stop_reason: "end_turn", total_cost_usd: 0.0024, usage, modelUsage: { "grok-4.6-build": {} }, session_id: session } + ].map((frame) => JSON.stringify(frame)).join("\n"); +}; + +test("a completed turn's seal row carries exactly its declared fields, and never the model's reply", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-seal-completed-")); + try { + const usageLedgerPath = path.join(root, "usage.jsonl"); + const deps: GrokEngineBrokerTurnDependencies = { + turns: new EngineBrokerTurnRegistry(path.join(root, "turns")), proxy, credentialStale: () => false, + mcp: { register: () => "mcp-capability-0123456789abcdef", revoke: () => undefined, observe: () => ({ started: 1, answered: 1, undecoded: 0, outstanding: [] }) }, + prepareIsolation: async () => async () => undefined, + runNative: async () => ({ text: answered(reply), workerPid: 4_242, workerUid: 2_200, startTicks: 99n }) + }; + const result = await runGrokEngineBrokerTurn(deps, registration(usageLedgerPath), "wake-done", "prompt", "http://127.0.0.1:43124/mcp"); + assert.equal(result.text, reply, "the turn itself still answers with the model's reply"); + + const bytes = await readFile(engineBrokerSealLedgerPathFor(usageLedgerPath), "utf8"); + const lines = bytes.split("\n").filter((line) => line.length > 0); + assert.equal(lines.length, 1); + const row = JSON.parse(lines[0]!) as Record; + assert.deepEqual(Object.keys(row).sort(), ["agent", "at", "engine", "limit_reason", "model", "outcome", "requests", "turn", "v", "wake"]); + assert.deepEqual([row.v, row.agent, row.wake, row.engine, row.outcome, row.model, row.limit_reason, row.requests], [TURN_SEAL_LEDGER_VERSION, "foreman", "wake-done", "grok", "completed", "grok-4.6", "none", 1]); + // The second half of the same guarantee, on the bytes rather than the keys: + // a reply that reached the ledger under any name is the failure. + assert.ok(!bytes.includes(reply), "the model's reply must never reach the ledger"); + // A completed turn carries no failure members at all, and the facade's + // observation is a failed turn's member: neither may appear here. + for (const absent of ["text", "code", "diagnostic", "mcp", "usage", "workerPid", "workerUid", "kind", "version"]) { + assert.equal(Object.hasOwn(row, absent), false, absent); + } + } finally { await rm(root, { recursive: true, force: true }); } +}); From 3c7e74f65e51e0cdafe1283ec791acfcfbe96fc1 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 15:02:53 +0200 Subject: [PATCH 120/124] test: type the sealed completed turn's native result frame --- src/runtime/engineBrokerSealLedger.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/engineBrokerSealLedger.test.ts b/src/runtime/engineBrokerSealLedger.test.ts index d2a299f..b3d60c6 100644 --- a/src/runtime/engineBrokerSealLedger.test.ts +++ b/src/runtime/engineBrokerSealLedger.test.ts @@ -224,7 +224,7 @@ test("a completed turn's seal row carries exactly its declared fields, and never turns: new EngineBrokerTurnRegistry(path.join(root, "turns")), proxy, credentialStale: () => false, mcp: { register: () => "mcp-capability-0123456789abcdef", revoke: () => undefined, observe: () => ({ started: 1, answered: 1, undecoded: 0, outstanding: [] }) }, prepareIsolation: async () => async () => undefined, - runNative: async () => ({ text: answered(reply), workerPid: 4_242, workerUid: 2_200, startTicks: 99n }) + runNative: async () => ({ text: answered(reply), workerPid: 4_242, workerUid: 2_200, startTicks: 99n, diagnostic: { status: "ok", stage: "output", failureClass: "none", profileApplied: false, exitCode: 0, termSignal: 0, workerPid: 4_242, workerUid: 2_200, startTicks: "99" } }) }; const result = await runGrokEngineBrokerTurn(deps, registration(usageLedgerPath), "wake-done", "prompt", "http://127.0.0.1:43124/mcp"); assert.equal(result.text, reply, "the turn itself still answers with the model's reply"); From 074c75c924410981857b0abe77adf5645797e796 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 15:04:38 +0200 Subject: [PATCH 121/124] fix: assert and correct the mode of existing runtime-home subdirectories --- src/observability/causalEvents.ts | 11 +-- src/observability/orgObserver.ts | 7 +- src/pi/cliSession.ts | 14 +-- src/pi/piHarness.ts | 16 +--- src/pi/turnTrace.ts | 7 +- src/pi/worldTrajectory.ts | 7 +- src/runtime/AGENTS.md | 15 ++- src/runtime/runtimeHomeLayout.test.ts | 130 ++++++++++++++++++++++++-- src/runtime/runtimeHomeLayout.ts | 71 ++++++++++++++ 9 files changed, 228 insertions(+), 50 deletions(-) diff --git a/src/observability/causalEvents.ts b/src/observability/causalEvents.ts index 9a29dd5..b594157 100644 --- a/src/observability/causalEvents.ts +++ b/src/observability/causalEvents.ts @@ -1,8 +1,8 @@ import { createHash, randomUUID } from "node:crypto"; import { constants } from "node:fs"; -import { appendFile, mkdir, open, readFile, rename, stat, unlink } from "node:fs/promises"; +import { appendFile, open, readFile, rename, stat, unlink } from "node:fs/promises"; import path from "node:path"; -import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "../runtime/runtimeHomeLayout.js"; +import { ensureRuntimeHomeDirectory } from "../runtime/runtimeHomeLayout.js"; /** * Daimon's own copy of the `noopolis.causal-event.v1` wire envelope. Field- @@ -107,8 +107,7 @@ const readSeqStore = async (runtimeHomePath: string): Promise => }; const writeSeqStore = async (runtimeHomePath: string, store: CausalSeqStore): Promise => { - const directory = telemetryDir(runtimeHomePath); - await mkdir(directory, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); + const directory = await ensureRuntimeHomeDirectory(runtimeHomePath, "telemetry"); const file = seqFilePath(runtimeHomePath); const temporary = `${file}.${randomUUID()}.tmp`; const handle = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); @@ -183,7 +182,7 @@ export const nextCausalSeq = async (input: { const lockPath = path.resolve(telemetryDir(input.runtimeHomePath), "causal.seq.lock"); const previous = seqAllocationQueues.get(lockPath) ?? Promise.resolve(); const allocation = previous.catch(() => undefined).then(async () => { - await mkdir(telemetryDir(input.runtimeHomePath), { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); + await ensureRuntimeHomeDirectory(input.runtimeHomePath, "telemetry"); await acquireSeqLock(lockPath); try { const store = await readSeqStore(input.runtimeHomePath); @@ -208,7 +207,7 @@ export const nextCausalSeq = async (input: { /** Appends one CausalEvent record as a line of `runtimeHome/telemetry/causal.jsonl`. */ export const appendCausalEvent = async (runtimeHomePath: string, event: CausalEvent): Promise => { - await mkdir(telemetryDir(runtimeHomePath), { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); + await ensureRuntimeHomeDirectory(runtimeHomePath, "telemetry"); await appendFile(jsonlFilePath(runtimeHomePath), `${JSON.stringify(event)}\n`, "utf8"); }; diff --git a/src/observability/orgObserver.ts b/src/observability/orgObserver.ts index 2d21ef8..bc55516 100644 --- a/src/observability/orgObserver.ts +++ b/src/observability/orgObserver.ts @@ -1,8 +1,8 @@ -import { mkdir, writeFile } from "node:fs/promises"; +import { writeFile } from "node:fs/promises"; import path from "node:path"; import type { MemoryRecallAudit } from "@noopolis/mneme"; -import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "../runtime/runtimeHomeLayout.js"; +import { ensureRuntimeHomeDirectory } from "../runtime/runtimeHomeLayout.js"; export interface WakeBenchRow { agent: string; @@ -207,8 +207,7 @@ export class OrgObserver { } async write(runtimeRoot: string): Promise { - const telemetryDir = path.join(runtimeRoot, "telemetry"); - await mkdir(telemetryDir, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); + const telemetryDir = await ensureRuntimeHomeDirectory(runtimeRoot, "telemetry"); const summaryRecord = { assertions: this.assertions, behavior: this.behaviorSummary(), diff --git a/src/pi/cliSession.ts b/src/pi/cliSession.ts index f161009..b045f4b 100644 --- a/src/pi/cliSession.ts +++ b/src/pi/cliSession.ts @@ -1,7 +1,6 @@ import { randomUUID } from "node:crypto"; import { createServer, type Server } from "node:http"; import { spawn, type ChildProcess } from "node:child_process"; -import { mkdir } from "node:fs/promises"; import type { ToolDefinition } from "@earendil-works/pi-coding-agent"; import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; @@ -35,7 +34,7 @@ import { decodeGrokHeadlessResult } from "./grokHeadlessResult.js"; import { terminateChild, trackCliChild } from "./cliProcess.js"; import type { PiSessionLike } from "./piAgentHandle.js"; import type { PiSessionFactoryInput } from "./piHarness.js"; -import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "../runtime/runtimeHomeLayout.js"; +import { ensureRuntimeHome, ensureRuntimeHomeDirectory } from "../runtime/runtimeHomeLayout.js"; export type CliEngineKind = "agy" | "codex" | "grok"; @@ -139,14 +138,9 @@ type CliTurnEnd = Extract; export const prepareCliRuntimeHome = async (runtimeHomePath: string | undefined): Promise => { if (runtimeHomePath === undefined) return; - await Promise.all([ - runtimeHomePath, - `${runtimeHomePath}/.config`, - `${runtimeHomePath}/.local/share`, - `${runtimeHomePath}/.local/state`, - `${runtimeHomePath}/.cache`, - `${runtimeHomePath}/.tmp` - ].map((directory) => mkdir(directory, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }))); + await ensureRuntimeHome(runtimeHomePath); + await Promise.all([".config", ".local/share", ".local/state", ".cache", ".tmp"] + .map((relative) => ensureRuntimeHomeDirectory(runtimeHomePath, relative))); }; const childSecretValues = (redactedNames: readonly string[]): readonly string[] => diff --git a/src/pi/piHarness.ts b/src/pi/piHarness.ts index 69096d9..cc4e235 100644 --- a/src/pi/piHarness.ts +++ b/src/pi/piHarness.ts @@ -23,7 +23,7 @@ import { type PiWakeEnvironmentContextRef } from "./piAgentWakeSupport.js"; import { DAIMON_WAKE_ID_ENV } from "./cliEnvironment.js"; import { createPiWorldTools, piWorldToolNames, type PiWorldBinding } from "./worldTools.js"; import type { PiWorldToolContextRef } from "./worldNudge.js"; -import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "../runtime/runtimeHomeLayout.js"; +import { ensureRuntimeHome, ensureRuntimeHomeDirectory } from "../runtime/runtimeHomeLayout.js"; import { bindPiRawTrainingCapture, validatePiRawTrainingCaptureOptions, @@ -98,18 +98,12 @@ export class PiHarnessAdapter implements AgentHarnessAdapter { async startAgent(input: AgentStartInput): Promise { validatePiRawTrainingCaptureOptions(this.options.rawTrainingCapture); - await Promise.all([ - input.runtimeHomePath, - `${input.runtimeHomePath}/.config`, - `${input.runtimeHomePath}/.local/share`, - `${input.runtimeHomePath}/.local/state`, - `${input.runtimeHomePath}/.cache`, - `${input.runtimeHomePath}/.tmp`, - `${input.runtimeHomePath}/tool-state` - ].map((directory) => mkdir(directory, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }))); + await ensureRuntimeHome(input.runtimeHomePath); + await Promise.all([".config", ".local/share", ".local/state", ".cache", ".tmp", "tool-state"] + .map((relative) => ensureRuntimeHomeDirectory(input.runtimeHomePath, relative))); await mkdir(input.workspacePath, { recursive: true }); const memoryRuntimeHomePath = this.options.memory?.runtimeHomePath ?? input.runtimeHomePath; - await mkdir(memoryRuntimeHomePath, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); + await ensureRuntimeHome(memoryRuntimeHomePath); const modelSpec = this.options.model ?? { auth: { method: "codex" as const }, provider: "openai", diff --git a/src/pi/turnTrace.ts b/src/pi/turnTrace.ts index 5ac198b..1833ed0 100644 --- a/src/pi/turnTrace.ts +++ b/src/pi/turnTrace.ts @@ -1,12 +1,12 @@ import { createHash } from "node:crypto"; -import { appendFile, mkdir, writeFile } from "node:fs/promises"; +import { appendFile, writeFile } from "node:fs/promises"; import path from "node:path"; import type { MemoryPrepareTurnResult, MemoryWakeMode } from "@noopolis/mneme"; import type { HarnessModelSpec, WakeEvent } from "../core/types.js"; import { redactCredentialText } from "../core/credentialRedaction.js"; -import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "../runtime/runtimeHomeLayout.js"; +import { ensureRuntimeHomeDirectory } from "../runtime/runtimeHomeLayout.js"; export interface PiTurnTraceModel { authMethod: NonNullable["method"]; @@ -269,8 +269,7 @@ export const writeTurnTraceRecord = async ( record: PiTurnTraceRecord ): Promise => { const telemetryPath = path.join(runtimeHomePath, "telemetry"); - const turnsPath = path.join(telemetryPath, "turns"); - await mkdir(turnsPath, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); + const turnsPath = await ensureRuntimeHomeDirectory(runtimeHomePath, "telemetry/turns"); const body = `${JSON.stringify(record, null, 2)}\n`; await writeFile(path.join(turnsPath, `${sanitizeTraceFileId(record.turn_id)}.json`), body, "utf8"); await appendFile(path.join(telemetryPath, "turns.ndjson"), `${JSON.stringify(record)}\n`, "utf8"); diff --git a/src/pi/worldTrajectory.ts b/src/pi/worldTrajectory.ts index 6883ed0..9ee6f8b 100644 --- a/src/pi/worldTrajectory.ts +++ b/src/pi/worldTrajectory.ts @@ -1,11 +1,11 @@ import { createHash } from "node:crypto"; -import { appendFile, mkdir, writeFile } from "node:fs/promises"; +import { appendFile, writeFile } from "node:fs/promises"; import path from "node:path"; import type { PiTurnTraceModel } from "./turnTrace.js"; import { redactTraceText, sanitizeTraceFileId } from "./turnTrace.js"; import type { PiWorldTurnContext } from "./worldNudge.js"; -import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "../runtime/runtimeHomeLayout.js"; +import { ensureRuntimeHomeDirectory } from "../runtime/runtimeHomeLayout.js"; export const WORLD_TRAJECTORY_SCHEMA = "daimon.world_trajectory.v1" as const; @@ -189,8 +189,7 @@ export const persistPiWorldTrajectory = async ( } }; const telemetryPath = path.join(input.runtimeHomePath, "telemetry"); - const trajectoriesPath = path.join(telemetryPath, "world-trajectories"); - await mkdir(trajectoriesPath, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); + const trajectoriesPath = await ensureRuntimeHomeDirectory(input.runtimeHomePath, "telemetry/world-trajectories"); const bytes = `${JSON.stringify(record, null, 2)}\n`; await writeFile( path.join(trajectoriesPath, `${sanitizeTraceFileId(input.turnId)}.json`), diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 20b4e62..a84e495 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -462,7 +462,20 @@ in `deny` (verified: `/tmp`, `/var/tmp`, `/run`, `/etc`, `sessions` all fail; (`runtimeHomeLayout.ts`: telemetry, turn traces, world trajectories, `tool-state`, the engine XDG directories, `.tmp`), so a traversable home still exposes nothing but `tool-output/`. A deployment-provisioned memory - home under that runtime home must stay `0700` for the same reason; + home under that runtime home must stay `0700` for the same reason. That mode + is *asserted and corrected*, not merely passed to `mkdir`, because `mkdir`'s + `mode` decides nothing for a directory that already exists: a `telemetry/` + left at `0755` by a pre-branch Daimon or pre-created by a deployment stayed + `0755` forever, and under a `0710` home that is the worker reading its own + agent's prompts, replies and causal history. `ensureRuntimeHomeDirectory` + walks every level below the home, opens each through + `O_DIRECTORY|O_NOFOLLOW` and `fchmod`s the directory it stat'd; one owned by + another uid is **refused**, never widened, and a symlink planted where a + directory belongs is refused rather than followed. The home itself is + create-only (`ensureRuntimeHome`) — whether it should be `0700` or a Grok + agent's `0710` is `physicalReadiness.ts`'s judgement, not the layout's. The + mode constant lives only in that module, and a test fails the build if any + writer imports it again; - spills (`toolResultSpill.ts`) are written `0640`; provision `/tool-output` as `2000: 2750` (setgid) under a runtime home the worker can traverse, so each spill carries that agent's diff --git a/src/runtime/runtimeHomeLayout.test.ts b/src/runtime/runtimeHomeLayout.test.ts index 6c4363d..7be7c9b 100644 --- a/src/runtime/runtimeHomeLayout.test.ts +++ b/src/runtime/runtimeHomeLayout.test.ts @@ -1,12 +1,12 @@ import assert from "node:assert/strict"; -import { mkdir, mkdtemp, readFile, readdir, rm, stat } from "node:fs/promises"; +import { chmod, mkdir, mkdtemp, readFile, readdir, rm, stat, symlink } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import test from "node:test"; import { appendCausalEvent, CAUSAL_EVENT_VERSION, nextCausalSeq } from "../observability/causalEvents.js"; import { summarizePrompt, writeTurnTraceRecord } from "../pi/turnTrace.js"; -import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "./runtimeHomeLayout.js"; +import { ensureRuntimeHome, ensureRuntimeHomeDirectory, RUNTIME_HOME_SUBDIRECTORY_MODE } from "./runtimeHomeLayout.js"; /** * A brokered Grok runtime home is traversable by its worker uid (0710), so @@ -22,6 +22,15 @@ const withTraversableHome = async (body: (home: string) => Promise): Promi }; const mode = async (target: string): Promise => (await stat(target)).mode & 0o7777; +const traceRecord = { + agent_id: "mapper", completed_at: "2026-01-01T00:00:01.000Z", + engine: { auth_method: "none", kind: "pi", model: "llama3.2", provider: "local" }, + memory: { enabled: false }, prompt: summarizePrompt("hi"), reply: { output_chars: 2, reply_given: true }, + schema: "daimon.turn_trace.v1", session: { dispose_after_wake: false, mode: "awake", thread_id: "t" }, + started_at: "2026-01-01T00:00:00.000Z", status: "completed", timings_ms: { total: 1 }, tools: [], + turn_id: "turn-1", wake: { event_id: "w", kind: "message" } +} as unknown as Parameters[1]; + test("the runtime-home subdirectory mode grants nobody but the runtime user", () => { assert.equal(RUNTIME_HOME_SUBDIRECTORY_MODE, 0o700); }); @@ -39,18 +48,119 @@ test("telemetry directories Daimon creates in a traversable runtime home are pri await nextCausalSeq({ runtimeHomePath: home, agentId: "a", turnId: "t1", count: 1 } as never); assert.equal(await mode(path.join(home, "telemetry")), RUNTIME_HOME_SUBDIRECTORY_MODE); - await writeTurnTraceRecord(home, { - agent_id: "mapper", completed_at: "2026-01-01T00:00:01.000Z", - engine: { auth_method: "none", kind: "pi", model: "llama3.2", provider: "local" }, - memory: { enabled: false }, prompt: summarizePrompt("hi"), reply: { output_chars: 2, reply_given: true }, - schema: "daimon.turn_trace.v1", session: { dispose_after_wake: false, mode: "awake", thread_id: "t" }, - started_at: "2026-01-01T00:00:00.000Z", status: "completed", timings_ms: { total: 1 }, tools: [], - turn_id: "turn-1", wake: { event_id: "w", kind: "message" } - }); + await writeTurnTraceRecord(home, traceRecord); + assert.equal(await mode(path.join(home, "telemetry", "turns")), RUNTIME_HOME_SUBDIRECTORY_MODE); + }); +}); + +/** + * The half the mode argument never covered. + * + * `mkdir(..., { mode })` decides nothing for a directory that already exists, + * and `assertRuntimeDirectory` checks the home and not what Daimon creates + * inside it. So a `telemetry/` left at 0755 by a pre-branch Daimon — or + * pre-created by a deployment — stayed 0755 under a Grok agent's deliberately + * traversable 0710 home, where it is the sandboxed worker reading its own + * agent's prompts, replies and causal history. + * + * The boundary these assertions straddle: a fresh install against an existing + * one. Every writer below is reached through its real entry point, because the + * hole was never in the mode constant — it was in what the call sites did with + * it. + * + * Mutation: restore `mkdir(directory, { recursive: true, mode })` in + * `ensureRuntimeHomeDirectory` and every assertion here goes red while the + * fresh-install test above stays green, which is exactly how this shipped. + */ +test("a runtime-home subdirectory that already exists is made private, not left as it was found", async () => { + await withTraversableHome(async (home) => { + // Pre-created by a deployment, or by a Daimon that predates the mode. + await mkdir(path.join(home, "telemetry"), { mode: 0o755 }); + await appendCausalEvent(home, { + version: CAUSAL_EVENT_VERSION, id: "daimon:t1:turn.output.completed", type: "turn.output.completed", + occurred_at: "2026-01-01T00:00:00.000Z", actor: { kind: "agent", id: "a" }, subject: { kind: "turn", id: "t1" }, + causes: [], run_id: "run", seq: 1, payload: {} + } as never); + assert.equal(await mode(path.join(home, "telemetry")), RUNTIME_HOME_SUBDIRECTORY_MODE); + + // An existing ancestor is the same hole one level up: `telemetry/turns` can + // be created privately under a `telemetry/` that stays world-readable. + await chmod(path.join(home, "telemetry"), 0o755); + await mkdir(path.join(home, "telemetry", "turns"), { mode: 0o755 }); + await writeTurnTraceRecord(home, traceRecord); + assert.equal(await mode(path.join(home, "telemetry")), RUNTIME_HOME_SUBDIRECTORY_MODE, "the ancestor is corrected too"); assert.equal(await mode(path.join(home, "telemetry", "turns")), RUNTIME_HOME_SUBDIRECTORY_MODE); + + // And the home itself is never touched: which mode it should carry is + // `physicalReadiness.ts`'s judgement, and for a Grok agent it is 0710. + assert.equal(await mode(home), 0o710); + }); +}); + +/** + * Refuse rather than widen, and never follow a link to do it. + * + * A directory the runtime does not own cannot be made private by it, and + * writing an agent's telemetry into it anyway is the failure the correction + * exists to prevent. A symlink planted where a directory belongs is the same + * fault with an attacker attached, so the correction goes through an + * `O_DIRECTORY|O_NOFOLLOW` handle and the `fchmod` lands on the directory that + * was stat'd. + * + * Mutation: drop the owner check and the first case silently proceeds; drop + * `O_NOFOLLOW` and the second chmods the link's target instead of refusing. + */ +test("a runtime-home subdirectory owned by another user, or replaced by a symlink, is refused", async () => { + await withTraversableHome(async (home) => { + await mkdir(path.join(home, "telemetry"), { mode: 0o755 }); + const foreign = (process.getuid?.() ?? 0) + 4_242; + await assert.rejects(ensureRuntimeHomeDirectory(home, "telemetry", foreign), /owned by the runtime user/u); + assert.equal(await mode(path.join(home, "telemetry")), 0o755, "a refusal corrects nothing and widens nothing"); + + const elsewhere = path.join(home, "elsewhere"); + await mkdir(elsewhere, { mode: 0o755 }); + await symlink(elsewhere, path.join(home, "tool-state")); + await assert.rejects(ensureRuntimeHomeDirectory(home, "tool-state")); + assert.equal(await mode(elsewhere), 0o755, "the link's target must not be chmod'ed through it"); + }); +}); + +test("the home itself is created when absent and never re-moded when present", async () => { + await withTraversableHome(async (home) => { + const fresh = path.join(home, "fresh-home"); + assert.equal(await ensureRuntimeHome(fresh), fresh); + assert.equal(await mode(fresh), RUNTIME_HOME_SUBDIRECTORY_MODE); + await chmod(fresh, 0o710); + await ensureRuntimeHome(fresh); + assert.equal(await mode(fresh), 0o710, "a Grok agent's traversable home is not this helper's judgement"); }); }); +/** + * The rule that keeps the correction from being reintroducible. + * + * `mkdir(..., { mode })` reads like a guarantee and is one only for a + * directory that does not exist yet, so the mode constant stays private to + * this module: a call site that wants a private directory under a runtime home + * asks `ensureRuntimeHomeDirectory` for one and gets the assertion with it. + * + * Mutation: import the constant into any writer and pass it to `mkdir` again, + * and this goes red — which is the shape the 0755 hole had. + */ +test("the private mode is used only where it is also asserted", async () => { + const offenders: string[] = []; + const walk = async (directory: string): Promise => { + for (const entry of await readdir(directory, { withFileTypes: true })) { + const target = path.join(directory, entry.name); + if (entry.isDirectory()) { if (entry.name !== "fixtures" && entry.name !== "artifacts") await walk(target); continue; } + if (!entry.name.endsWith(".ts") || entry.name.endsWith(".test.ts") || target === "src/runtime/runtimeHomeLayout.ts") continue; + if ((await readFile(target, "utf8")).includes("RUNTIME_HOME_SUBDIRECTORY_MODE")) offenders.push(target); + } + }; + await Promise.all(["src/pi", "src/observability", "src/runtime", "src/mcp", "src/core"].map(walk)); + assert.deepEqual(offenders, []); +}); + // Creations that are not inside an agent's runtime home. const OUTSIDE_RUNTIME_HOME = [ "src/runtime/native/copyArtifact.ts", "src/observability/emitCausalFixture.ts", "src/pi/auth.ts", "src/runtime/cli.ts" diff --git a/src/runtime/runtimeHomeLayout.ts b/src/runtime/runtimeHomeLayout.ts index f8a280d..d1b5c1f 100644 --- a/src/runtime/runtimeHomeLayout.ts +++ b/src/runtime/runtimeHomeLayout.ts @@ -1,3 +1,7 @@ +import { constants } from "node:fs"; +import { mkdir, open } from "node:fs/promises"; +import path from "node:path"; + /** * Mode for every directory Daimon creates inside an agent's runtime home. * @@ -11,3 +15,70 @@ * readable one. */ export const RUNTIME_HOME_SUBDIRECTORY_MODE = 0o700; + +/** + * The home itself, created if it is absent and otherwise left exactly as it is. + * + * Create-only is the whole contract here. A brokered Grok agent's home is + * deliberately `0710` and an organization's may be `0700`; which of the two is + * correct is `physicalReadiness.ts`'s judgement, made against the agent's + * declared engine, and a layout helper that "corrected" a traversable home to + * `0700` would break the worker's only route to its own spills. + */ +export const ensureRuntimeHome = async (runtimeHomePath: string): Promise => { + await mkdir(runtimeHomePath, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); + return runtimeHomePath; +}; + +/** + * One directory Daimon owns *below* a runtime home, private on every install. + * + * `mkdir(..., { mode })` decides nothing for a directory that already exists, + * and that is the common case rather than the exotic one: a `telemetry/` left + * at `0755` by a pre-branch Daimon, or pre-created by a deployment, stayed + * `0755` forever. Under a Grok agent's traversable `0710` home that is the + * worker reading its own agent's prompts, replies and causal history — the + * home is traverse-only precisely so that nothing but `tool-output/` is + * readable. `assertRuntimeDirectory` checks the home, and nothing checked what + * Daimon created inside it. + * + * So every level below the home is asserted and corrected on the way down, + * through a handle rather than a path: `O_DIRECTORY|O_NOFOLLOW` refuses a + * symlink planted where a directory belongs, and the `fchmod` that follows + * lands on the directory that was stat'd. A directory owned by anyone but the + * runtime user is **refused**, never widened and never silently accepted — the + * runtime cannot make someone else's directory private, and proceeding would + * write an agent's telemetry into it anyway. + * + * `owner` is a seam so both refusals are testable unprivileged, exactly as + * `physicalReadiness.ts`'s `RuntimeIdentity` is. + */ +export async function ensureRuntimeHomeDirectory(runtimeHomePath: string, relative: string, owner: number = process.getuid?.() ?? -1): Promise { + const segments = relative.split("/").filter((segment) => segment.length > 0); + if (segments.length === 0 || segments.some((segment) => segment === "." || segment === "..")) { + throw new Error(`runtime home subdirectory must name a path below the home: ${relative}`); + } + let current = await ensureRuntimeHome(runtimeHomePath); + for (const segment of segments) { + current = path.join(current, segment); + await mkdir(current, { mode: RUNTIME_HOME_SUBDIRECTORY_MODE }).catch((error: NodeJS.ErrnoException) => { + if (error.code !== "EEXIST") throw error; + }); + await assertPrivateDirectory(current, owner); + } + return current; +} + +const flag = (name: "O_DIRECTORY" | "O_NOFOLLOW"): number => (constants as typeof constants & Partial>)[name] ?? 0; + +async function assertPrivateDirectory(directory: string, owner: number): Promise { + const handle = await open(directory, constants.O_RDONLY | flag("O_DIRECTORY") | flag("O_NOFOLLOW")); + try { + const entry = await handle.stat(); + if (!entry.isDirectory()) throw new Error(`runtime home path is not a directory: ${directory}`); + if (entry.uid !== owner) throw new Error(`runtime home subdirectory must be owned by the runtime user: ${directory}`); + if ((entry.mode & 0o7777) !== RUNTIME_HOME_SUBDIRECTORY_MODE) await handle.chmod(RUNTIME_HOME_SUBDIRECTORY_MODE); + } finally { + await handle.close(); + } +} From f81c55ed69d743a8d1cf22bbd74e2ac0616168cf Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 15:08:13 +0200 Subject: [PATCH 122/124] fix: ensure the agent tool-state directory through the runtime-home layout --- src/runtime/AGENTS.md | 8 ++++++-- src/runtime/productionAgentTools.ts | 5 +++-- src/runtime/runtimeHomeLayout.test.ts | 22 ++++++++++++++++++++-- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index a84e495..57a657e 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -474,8 +474,12 @@ in `deny` (verified: `/tmp`, `/var/tmp`, `/run`, `/etc`, `sessions` all fail; directory belongs is refused rather than followed. The home itself is create-only (`ensureRuntimeHome`) — whether it should be `0700` or a Grok agent's `0710` is `physicalReadiness.ts`'s judgement, not the layout's. The - mode constant lives only in that module, and a test fails the build if any - writer imports it again; + mode constant lives only in that module, a test fails the build if any writer + imports it again, and the same test refuses any `mkdir` that names a runtime + home outside the layout — a `mode:` argument covers only the install where + the directory is new. `wakeAcceptanceFs.ts` is the one exception and closes + the hole the other way, by asserting the directory it found and refusing a + wider one; - spills (`toolResultSpill.ts`) are written `0640`; provision `/tool-output` as `2000: 2750` (setgid) under a runtime home the worker can traverse, so each spill carries that agent's diff --git a/src/runtime/productionAgentTools.ts b/src/runtime/productionAgentTools.ts index a23f304..1b7c702 100644 --- a/src/runtime/productionAgentTools.ts +++ b/src/runtime/productionAgentTools.ts @@ -1,7 +1,7 @@ import { spawn } from "node:child_process"; import { createHash } from "node:crypto"; import { constants } from "node:fs"; -import { lstat, mkdir, open, readdir, rename, unlink } from "node:fs/promises"; +import { lstat, open, readdir, rename, unlink } from "node:fs/promises"; import path from "node:path"; import type { ToolDefinition } from "@earendil-works/pi-coding-agent"; @@ -13,6 +13,7 @@ import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/ import type { OrganizationRuntimeAgentConfig, OrganizationRuntimeMcpServer } from "./organizationRuntime.js"; import { moltnetOperationResult, readMoltnetPages } from "./moltnetMachineRead.js"; import { McpToolCallError, MCP_TOOL_RESULT_MAX_BYTES, renderMcpToolResult, replayMcpReceipt, type McpUpstreamResult } from "./mcpToolResult.js"; +import { ensureRuntimeHomeDirectory } from "./runtimeHomeLayout.js"; import { capToolResult, resolveExemptToolNames, resolveToolResultMaxBytes, TOOL_OUTPUT_DIRECTORY_NAME } from "./toolResultSpill.js"; import { cliChildEnvironment } from "../pi/cliEnvironment.js"; import type { PiWakeEnvironmentContextRef } from "../pi/piAgentWakeSupport.js"; @@ -31,7 +32,7 @@ const MAX_RESULT = 65_536; const TIMEOUT = 10_000; const DAIMON_ACTION_ID_PREFIX = "daimon-"; export async function createProductionAgentTools(agent: OrganizationRuntimeAgentConfig, wakeContext: PiWakeEnvironmentContextRef = {}): Promise { - await mkdir(path.join(agent.runtimeHomePath, "tool-state"), { recursive: true, mode: 0o700 }); + await ensureRuntimeHomeDirectory(agent.runtimeHomePath, "tool-state"); // Resolved once, at agent start: a malformed bound is a configuration error // that should refuse the agent, not a surprise thrown from the middle of a // tool call the model is waiting on. diff --git a/src/runtime/runtimeHomeLayout.test.ts b/src/runtime/runtimeHomeLayout.test.ts index 7be7c9b..8b1a867 100644 --- a/src/runtime/runtimeHomeLayout.test.ts +++ b/src/runtime/runtimeHomeLayout.test.ts @@ -166,6 +166,16 @@ const OUTSIDE_RUNTIME_HOME = [ "src/runtime/native/copyArtifact.ts", "src/observability/emitCausalFixture.ts", "src/pi/auth.ts", "src/runtime/cli.ts" ]; +/** + * Files that may name a runtime home in a `mkdir` of their own. + * + * `runtimeHomeLayout.ts` is the correction itself. `wakeAcceptanceFs.ts` + * creates the home and its store directory and then *asserts* each one — + * refusing a directory it finds wider rather than correcting it — which closes + * the same hole the other way and is its own documented contract. + */ +const MAY_MKDIR_A_RUNTIME_HOME = ["src/runtime/runtimeHomeLayout.ts", "src/pi/wakeAcceptanceFs.ts"]; + const mkdirCalls = (source: string): string[] => { const calls: string[] = []; for (let index = source.indexOf("mkdir("); index !== -1; index = source.indexOf("mkdir(", index + 1)) { @@ -178,9 +188,10 @@ const mkdirCalls = (source: string): string[] => { return calls; }; -test("no runtime-home directory is created without an explicit private mode", async () => { +test("no runtime-home directory is created without an explicit private mode, and none is created outside the layout", async () => { // Source policy: a default `mkdir` under an agent's runtime home would be 0755, - // and the home of a brokered Grok agent is traversable by its worker uid. + // the home of a brokered Grok agent is traversable by its worker uid, and a + // `mode:` argument only covers the install where the directory is new. const offenders: string[] = []; const walk = async (directory: string): Promise => { for (const entry of await readdir(directory, { withFileTypes: true })) { @@ -189,6 +200,13 @@ test("no runtime-home directory is created without an explicit private mode", as if (!entry.name.endsWith(".ts") || entry.name.endsWith(".test.ts") || OUTSIDE_RUNTIME_HOME.includes(target)) continue; const source = await readFile(target, "utf8"); for (const call of mkdirCalls(source)) { + // A `mode:` argument is a create-only mode: it decides nothing for a + // directory that already exists, so naming a runtime home in a `mkdir` + // is the defect whether or not a mode is passed. + if (/runtimeHome/iu.test(call) && !MAY_MKDIR_A_RUNTIME_HOME.includes(target)) { + offenders.push(`${target}: ${call.replace(/\s+/gu, " ").slice(0, 90)}`); + continue; + } // The workspace is a caller-prepared root with its own contract (group-readable for Grok). if (call.includes("mode:") || call.includes("{ mode }") || call.includes("workspacePath")) continue; offenders.push(`${target}: ${call.replace(/\s+/gu, " ").slice(0, 90)}`); From 734372a50e30fcb4014b7403e6c53a21bd6f475e Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 15:12:34 +0200 Subject: [PATCH 123/124] fix: derive the MCP capability budget from the compiled worker turn bound --- src/runtime/AGENTS.md | 19 +++++-- src/runtime/engineBrokerMcpCallLog.ts | 10 ++-- src/runtime/engineBrokerMcpFacade.ts | 31 +++++++++--- .../engineBrokerMcpObservation.test.ts | 50 +++++++++++++------ src/runtime/engineBrokerSealLedger.test.ts | 5 +- src/runtime/engineBrokerSealLedger.ts | 6 +-- 6 files changed, 83 insertions(+), 38 deletions(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 57a657e..0b91035 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -339,10 +339,21 @@ byte-identical to a healthy turn. `EngineBrokerMcpCallLog.refuse` now counts each one by a closed reason class (`route`, `expired`, `exhausted`, `unrouted`, `oversized`), because the classes call for opposite fixes: an exhausted per-turn capability is a budget, an unserved route is a worker -asking for something that does not exist. The budget is reachable rather than -theoretical — `ENGINE_BROKER_MCP_CAPABILITY_REQUESTS` (128) covers every POST, -the GET tunnel and the DELETE, and a `search_tool`+`use_tool` round spends two, -so a 48-round wake asks for ~96 plus its handshake. Attribution comes from +asking for something that does not exist. That budget is *derived*, not +picked: `ENGINE_BROKER_MCP_CAPABILITY_REQUESTS` is `GROK_WORKER_MAX_TURNS` +times `ENGINE_BROKER_MCP_ROUND_REQUESTS` (3 — a round's `search_tool`, its +`use_tool`, and one spare for a retry or a second discovery) plus +`ENGINE_BROKER_MCP_SESSION_REQUESTS` (5 — `initialize`, +`notifications/initialized`, `tools/list`, the GET tunnel, the DELETE). It was +a literal 128 against a bound of 48 rounds whose legitimate traffic is ~101, so +the first round that also retried met a mid-turn 403 storm; the two numbers +that must agree now live in one place, and raising the turn bound can no longer +silently exhaust the budget. It stays a bound rather than a comfortable number +because the derivation is exact: the request *after* the worst-case legitimate +session is refused, so a compromised worker gets three MCP calls per round it +was compiled to take and not one more. `engineBrokerMcpObservation.test.ts` +drives that worst case through the real facade, computed from the turn bound +alone. Attribution comes from `EngineBrokerCapabilities.classifyToken`, which names the token's turn and why it would be refused *without spending its budget*; a bearer no grant matches names no turn and stays unattributed, because guessing an owner would be diff --git a/src/runtime/engineBrokerMcpCallLog.ts b/src/runtime/engineBrokerMcpCallLog.ts index 6654102..f8d8b8d 100644 --- a/src/runtime/engineBrokerMcpCallLog.ts +++ b/src/runtime/engineBrokerMcpCallLog.ts @@ -61,11 +61,11 @@ * every one of whose requests was 403'd sealed as `answered == started, * outstanding: []` — byte-identical to a healthy turn, which is precisely the * reading this instrument exists to make trustworthy. The reason class is what - * makes it actionable: an exhausted per-turn capability budget (a worker's - * `search_tool`+`use_tool` pair per round is two requests of the facade's - * `ENGINE_BROKER_MCP_CAPABILITY_REQUESTS`) is a different fault from a - * mount that was never registered, and both differ from a worker asking for a - * route the facade does not serve. Counts only, keyed by a closed vocabulary: + * makes it actionable: an exhausted per-turn capability budget — the facade's + * `ENGINE_BROKER_MCP_CAPABILITY_REQUESTS`, derived from the compiled turn + * bound times what a round may spend — is a different fault from a mount that + * was never registered, and both differ from a worker asking for a route the + * facade does not serve. Counts only, keyed by a closed vocabulary: * never the token, the capability, the URL or the body. A refusal the facade * cannot attribute to a turn — a bearer no live grant matches — is recorded * nowhere, because attributing it to a turn would be inventing the fact. diff --git a/src/runtime/engineBrokerMcpFacade.ts b/src/runtime/engineBrokerMcpFacade.ts index 44eccfe..68cd0d3 100644 --- a/src/runtime/engineBrokerMcpFacade.ts +++ b/src/runtime/engineBrokerMcpFacade.ts @@ -1,5 +1,6 @@ import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import { Readable } from "node:stream"; +import { GROK_WORKER_MAX_TURNS } from "../contracts/grokWorkerContract.js"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; import { EngineBrokerMcpCallLog, type EngineBrokerMcpCallObservation, type EngineBrokerMcpRefusalReason, type EngineBrokerMcpTunnelHandle } from "./engineBrokerMcpCallLog.js"; @@ -46,17 +47,31 @@ const FORWARDED_RESPONSE_HEADERS = ["content-type", "mcp-session-id", "mcp-proto const FORWARDED_METHODS = new Set(["POST", "GET", "DELETE"]); const MAX_REQUEST_BYTES = 1024 * 1024; export const ENGINE_BROKER_MCP_FACADE_PORT = 43_124; + +/** Requests one worker round may legitimately make: its `search_tool`, its `use_tool`, and one spare for a retry or a second discovery. */ +export const ENGINE_BROKER_MCP_ROUND_REQUESTS = 3; +/** The session's fixed cost, once per turn: `initialize`, `notifications/initialized`, `tools/list`, the standalone GET tunnel, the closing DELETE. */ +export const ENGINE_BROKER_MCP_SESSION_REQUESTS = 5; /** - * Requests one turn capability may spend, across all three methods. + * Requests one turn capability may spend, across all three methods — *derived* + * from the compiled turn bound rather than chosen. + * + * It was 128, and a legitimate 48-round wake needs ~101 of them: a worker's + * round is a `search_tool` and a `use_tool`, so the first round that also + * retries, or looks something up twice, eats the margin. A bound that can be + * predicted to bite mid-turn is not a bound, it is a 403 storm waiting for a + * real wake — and a number raised until it feels comfortable is not one + * either, because the budget exists to cap a *compromised* worker. * - * A worker's round is a `search_tool` and a `use_tool`, so a 48-turn wake is - * ~96 POSTs plus the handshake, the standalone GET tunnel and the closing - * DELETE: exhaustion is reachable rather than theoretical, and every request - * past it is a 403 the worker cannot explain. That is why the refusal is - * counted and sealed (`engineBrokerMcpCallLog.ts`) rather than being an - * absence in the turn's row. + * So the two numbers that must agree are kept in one place: the launcher's + * `--max-turns` backstop ({@link GROK_WORKER_MAX_TURNS}) times what a round + * may legitimately spend, plus the session's fixed cost. Raising the turn + * bound raises this with it, and a compromised worker still gets exactly three + * MCP calls per round it was compiled to take and not one more. The arithmetic + * is spelled out rather than folded into a literal so it can be audited: the + * two multiplicands above each say what they are. */ -export const ENGINE_BROKER_MCP_CAPABILITY_REQUESTS = 128; +export const ENGINE_BROKER_MCP_CAPABILITY_REQUESTS = GROK_WORKER_MAX_TURNS * ENGINE_BROKER_MCP_ROUND_REQUESTS + ENGINE_BROKER_MCP_SESSION_REQUESTS; export const ENGINE_BROKER_MCP_CAPABILITY_TTL_MS = 15 * 60_000; class FacadeRefusal extends Error {} diff --git a/src/runtime/engineBrokerMcpObservation.test.ts b/src/runtime/engineBrokerMcpObservation.test.ts index fc2b42d..86fdfeb 100644 --- a/src/runtime/engineBrokerMcpObservation.test.ts +++ b/src/runtime/engineBrokerMcpObservation.test.ts @@ -6,7 +6,8 @@ import test from "node:test"; import { ToolListChangedNotificationSchema } from "@modelcontextprotocol/sdk/types.js"; import type { EngineBrokerMcpTunnelObservation } from "./engineBrokerMcpCallLog.js"; -import { ENGINE_BROKER_MCP_CAPABILITY_REQUESTS, ENGINE_BROKER_MCP_FACADE_PORT } from "./engineBrokerMcpFacade.js"; +import { GROK_WORKER_MAX_TURNS } from "../contracts/grokWorkerContract.js"; +import { ENGINE_BROKER_MCP_FACADE_PORT, ENGINE_BROKER_MCP_ROUND_REQUESTS, ENGINE_BROKER_MCP_SESSION_REQUESTS } from "./engineBrokerMcpFacade.js"; import { connectClient, FACADE_URL, sharedFacade, startRig, withDeadline } from "./engineBrokerMcpFacadeRig.test.js"; /** @@ -161,18 +162,32 @@ test("the facade observes the standalone GET tunnel: open with its age, whether * `route()` refuses before the call log is ever touched, so a request the * facade 403'd never reached `started`, `undecoded` or `outstanding`: a turn * whose every request was refused sealed as `answered == started, - * outstanding: []`, which is exactly what a healthy turn seals as. The budget - * makes that reachable rather than theoretical — one capability buys - * {@link ENGINE_BROKER_MCP_CAPABILITY_REQUESTS} requests across all three - * methods, and a worker spends two of them per round. + * outstanding: []`, which is exactly what a healthy turn seals as. + * + * The middle of this test is a second guarantee and it is a *relationship*, + * not a number. The demand side is computed from the launcher's compiled + * `--max-turns` backstop alone — every round spending its full MCP allowance, + * plus the session's fixed cost — and the whole of it must be served on one + * capability, with the very next request refused. Two numbers that must agree + * and live apart drift: the budget was a literal 128 against a bound of 48 + * rounds, ~101 requests of legitimate traffic, and the first round that also + * retried would have met a mid-turn 403 storm. Deriving one from the other is + * what makes raising `GROK_WORKER_MAX_TURNS` unable to silently exhaust the + * budget — and asserting the refusal one past the worst case is what keeps the + * derivation a bound rather than a comfortable number. * * The boundary these assertions straddle: a turn served against a turn - * refused, and within the refusals, a spent capability against a route the - * facade does not serve — the first is a budget to raise, the second is a - * worker asking for something that does not exist. + * refused; within the refusals, a spent capability against a route the facade + * does not serve; and, for the budget, legitimate worst-case traffic against + * the first request beyond it. * * Mutation: restore `throw new FacadeRefusal()` in place of either `refuse` - * call in `route()`, and a 403'd turn reads as an idle one again. + * call in `route()`, and a 403'd turn reads as an idle one again. Pin + * `ENGINE_BROKER_MCP_CAPABILITY_REQUESTS` back to a literal `128` and this is + * already red at today's turn bound; raise `GROK_WORKER_MAX_TURNS` beside that + * literal and it stays red, which is the drift the derivation removes. Raising + * `GROK_WORKER_MAX_TURNS` *with* the derivation keeps it green, which is the + * guarantee itself. */ test("a request the facade refused is counted against its turn, by reason, and is never a call it served", async () => { const facade = await sharedFacade(); @@ -202,13 +217,18 @@ test("a request the facade refused is counted against its turn, by reason, and i assert.equal(await send(FACADE_URL, "wrong-token-abcdefghijklmnopqrstuvwxyz0123456789"), 403); assert.equal(facade.observe(turnId)?.refusals?.route, 2, "an unattributable refusal belongs to no turn"); - // Neither refusal spent the capability, so the budget is exactly what was - // issued — and spending it all is reachable: a 48-round worker asks for - // ~96 of these plus its handshake, tunnel and DELETE. - for (let spent = 0; spent < ENGINE_BROKER_MCP_CAPABILITY_REQUESTS; spent += 1) assert.equal(await send(FACADE_URL, token), 200); - assert.equal(served, ENGINE_BROKER_MCP_CAPABILITY_REQUESTS); + // Neither refusal spent the capability, so what follows is the whole of it. + // The demand is read off the compiled turn bound and nothing else: every + // round the launcher admits, each spending its full MCP allowance, plus the + // one-off session cost. All of it must be served. + const legitimate = GROK_WORKER_MAX_TURNS * ENGINE_BROKER_MCP_ROUND_REQUESTS + ENGINE_BROKER_MCP_SESSION_REQUESTS; + for (let spent = 0; spent < legitimate; spent += 1) assert.equal(await send(FACADE_URL, token), 200, `request ${spent + 1} of ${legitimate} was refused`); + assert.equal(served, legitimate, "every legitimate request must reach the mount"); + // And it is still a bound: the first request past the worst case is refused, + // so the budget caps a compromised worker at exactly the traffic the turn + // bound compiles for. assert.equal(await send(FACADE_URL, token), 403, "the capability's budget is spent"); - assert.equal(served, ENGINE_BROKER_MCP_CAPABILITY_REQUESTS, "an exhausted capability never reaches the mount"); + assert.equal(served, legitimate, "an exhausted capability never reaches the mount"); const observed = facade.observe(turnId); assert.deepEqual(observed?.refusals, { route: 2, expired: 0, exhausted: 1, unrouted: 0, oversized: 0 }); diff --git a/src/runtime/engineBrokerSealLedger.test.ts b/src/runtime/engineBrokerSealLedger.test.ts index b3d60c6..5639d94 100644 --- a/src/runtime/engineBrokerSealLedger.test.ts +++ b/src/runtime/engineBrokerSealLedger.test.ts @@ -163,9 +163,8 @@ test("a cancelled turn's GET tunnel is sealed open with its age, closed, or neve * The refusal, on the same durable route as the hang it looks like. * * A request the facade 403'd never reached the relay, so before it was counted - * a turn whose capability was spent — 128 requests, two per worker round — - * sealed `answered == started, outstanding: []`, which is exactly what a - * healthy turn seals. The row has to carry the reason class, because an + * a turn whose capability was spent sealed `answered == started, + * outstanding: []`, which is exactly what a healthy turn seals. The row has to carry the reason class, because an * exhausted budget and an unserved route are opposite fixes. * * Mutation: drop the `refusals` member from `renderBrokerTurnSealLine` and the diff --git a/src/runtime/engineBrokerSealLedger.ts b/src/runtime/engineBrokerSealLedger.ts index a1d119a..4d64893 100644 --- a/src/runtime/engineBrokerSealLedger.ts +++ b/src/runtime/engineBrokerSealLedger.ts @@ -105,9 +105,9 @@ export const renderBrokerTurnSealLine = (terminal: EngineBrokerTerminalResponse, .slice(0, ENGINE_BROKER_MCP_OUTSTANDING_MAX) .map((call) => ({ name: ENGINE_BROKER_MCP_CALL_NAME.test(call.name) ? call.name : "", outstanding_ms: call.outstandingMs })), // Every request the facade refused before it could relay it, by reason - // class. Without it a turn whose capability was spent — 128 requests, - // two per worker round — seals as `answered == started, outstanding: - // []`, which is what a healthy turn seals as. + // class. Without it a turn whose capability was spent seals as + // `answered == started, outstanding: []`, which is what a healthy turn + // seals as. ...(terminal.mcpCalls.refusals === undefined ? {} : { refusals: Object.fromEntries(ENGINE_BROKER_MCP_REFUSAL_REASONS.map((reason) => [reason, terminal.mcpCalls!.refusals![reason]])) }), From 36ce9f06246f32314bb4e7daa51b3897e734f36a Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 16:30:52 +0200 Subject: [PATCH 124/124] fix: keep the v2 activity closure query answerable after a control host stop --- src/contracts/runtimeContractManifest.ts | 2 +- src/runtime/AGENTS.md | 21 +++++ .../organizationRuntimeClosure.test.ts | 89 +++++++++++++++++++ src/runtime/organizationRuntimeControl.ts | 31 ++++++- src/runtime/wakeAcceptanceTypes.ts | 6 ++ 5 files changed, 144 insertions(+), 5 deletions(-) create mode 100644 src/runtime/organizationRuntimeClosure.test.ts diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 95b0dfc..687b104 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -185,5 +185,5 @@ export const RUNTIME_CONTRACT_MANIFEST = { ] }, healthResponseSchema: { type: "object", additionalProperties: false, required: ["version", "state", "agents"], properties: { version: { const: "noopolis.daimon.organization-runtime-health.v1" }, state: { enum: ["starting", "running", "stopping", "stopped"] }, agents: { type: "array", maxItems: ORGANIZATION_RUNTIME_MAX_AGENTS, items: { type: "object", additionalProperties: false, required: ["agentId", "state"], properties: { agentId: text, state: { enum: ["starting", "running", "stopping", "stopped", "idle", "failed"] } } } } } }, activityResponseSchema: { type: "object", additionalProperties: false, required: ["version", "items"], properties: { version: { const: "noopolis.daimon.organization-runtime-activity.v1" }, items: { type: "array", maxItems: 100, items: activityItem }, nextCursor: { type: "string", minLength: 1, maxLength: 16, pattern: "^(0|[1-9][0-9]{0,15})$" } } }, - activityV2ResponseSchema: { type: "object", additionalProperties: false, required: ["version", "items"], properties: { version: { const: ORGANIZATION_RUNTIME_ACTIVITY_V2_VERSION }, executions: { type: "array", maxItems: ORGANIZATION_RUNTIME_MAX_AGENTS, items: { type: "object", additionalProperties: false, required: ["agent_id", "execution_id", "state", "delivery_ids"], properties: { agent_id: text, execution_id: { type: "string" }, state: { const: "running" }, delivery_ids: { type: "array", maxItems: 32, items: text } } } }, items: { type: "array", maxItems: 2_112, items: { type: "object", additionalProperties: false, required: ["version", "acceptance_id", "agent_id", "delivery_id", "request_digest", "state", "accepted_at", "updated_at", "active"], properties: { version: { const: "noopolis.daimon.wake-receipt-status.v2" }, acceptance_id: { type: "string" }, agent_id: text, delivery_id: text, request_digest: { type: "string" }, state: { enum: ["accepted", "running", "completed", "failed", "stopped"] }, accepted_at: timestamp, updated_at: timestamp, active: { type: "boolean" }, execution_id: { type: "string" }, deferred: { type: "boolean" }, text: { type: "string", maxLength: 16384 }, queue_position: { type: "integer", minimum: 1 }, code: { enum: ["engine_failed", "host_stopped", "host_stopping", "queue_full", "unknown_agent"] } } } } } } + activityV2ResponseSchema: { type: "object", additionalProperties: false, required: ["version", "items"], properties: { version: { const: ORGANIZATION_RUNTIME_ACTIVITY_V2_VERSION }, state: { enum: ["running", "stopped"] }, executions: { type: "array", maxItems: ORGANIZATION_RUNTIME_MAX_AGENTS, items: { type: "object", additionalProperties: false, required: ["agent_id", "execution_id", "state", "delivery_ids"], properties: { agent_id: text, execution_id: { type: "string" }, state: { const: "running" }, delivery_ids: { type: "array", maxItems: 32, items: text } } } }, items: { type: "array", maxItems: 2_112, items: { type: "object", additionalProperties: false, required: ["version", "acceptance_id", "agent_id", "delivery_id", "request_digest", "state", "accepted_at", "updated_at", "active"], properties: { version: { const: "noopolis.daimon.wake-receipt-status.v2" }, acceptance_id: { type: "string" }, agent_id: text, delivery_id: text, request_digest: { type: "string" }, state: { enum: ["accepted", "running", "completed", "failed", "stopped"] }, accepted_at: timestamp, updated_at: timestamp, active: { type: "boolean" }, execution_id: { type: "string" }, deferred: { type: "boolean" }, text: { type: "string", maxLength: 16384 }, queue_position: { type: "integer", minimum: 1 }, code: { enum: ["engine_failed", "host_stopped", "host_stopping", "queue_full", "unknown_agent"] } } } } } } } as const; diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 0b91035..f53a605 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -725,3 +725,24 @@ unmarked/deferred deliveries wait for new input without a self-wake loop. Live turn authority is `activity.executions`, independent of receipt completion; its execution id must equal the engine wake id. Budget pauses retain acceptance, and operator stop remains a hard latch. + +That authority has to outlive the host, because the caller who needs it reads it +last. `activityV2` used to answer `undefined` once `stop()` closed the acceptance +store — HTTP 503 `native_host_unavailable` through a caller's route — and the one +caller that must prove an execution closed asks *after* the runtime stopped: a +harness worker stops its host the moment a delivery closes its execution and +stays deferred awaiting external input. So a trial whose subject really ran, +spent its budget and simply did not do the work could not be told from a hung or +crashed one, and reported as an unscorable infrastructure failure. `stop()` now +seals the projection between the dispatcher's own shutdown — which awaits every +admitted turn, so `executions` is settled rather than momentary — and the store's +close, and `activityV2` serves that seal afterwards with `state: "stopped"`. A +stopped host has *more* certainty about quiescence than a live poll, not less, +because nothing can be admitted after the seal. Three things it is not: a bypass +of the control token, a fabricated idle runtime (a host that never started and +one whose seal could not be read both still answer nothing, because absence must +stay absence), and a claim about the store-backed routes beside it — +`availability` and `wakeReceipt` keep answering `undefined` after a stop, since +neither settles a closure proof. `state` is optional on the wire for the reason +every additive member here is: a projection published before the seal existed +must still parse, and its absence means "not stated", never "running". diff --git a/src/runtime/organizationRuntimeClosure.test.ts b/src/runtime/organizationRuntimeClosure.test.ts new file mode 100644 index 0000000..8ec4178 --- /dev/null +++ b/src/runtime/organizationRuntimeClosure.test.ts @@ -0,0 +1,89 @@ +import assert from "node:assert/strict"; +import { chmod, mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { ORGANIZATION_RUNTIME_VERSION, type OrganizationRuntimeHost, type OrganizationRuntimeWakeRequest } from "./organizationRuntime.js"; +import { createOrganizationRuntimeControlHostWithCoreForTest } from "./organizationRuntimeControl.js"; +import { ACTIVITY_V2_VERSION } from "./wakeAcceptanceTypes.js"; + +const token = "control-secret"; +const config = { + version: ORGANIZATION_RUNTIME_VERSION, + host: { bindHost: "127.0.0.1", port: 4318, controlTokenEnv: "DAIMON_CONTROL_CLOSURE_TOKEN" }, + agents: [{ id: "alpha", name: "Alpha", instructions: "Act.", workspacePath: "/runtime/workspace", runtimeHomePath: "/runtime/home", engine: { kind: "codex" as const } }] +}; +const storeOptions = { processIdentity: async () => ({ pid: 1, process_start: "test-start", boot_id: "test-boot", pid_namespace_dev: 1, pid_namespace_ino: 1 }), ownerLiveness: async () => true }; +const delivery = (deliveryId = "delivery-1") => ({ token, agent_id: "alpha", delivery_id: deliveryId, event: { version: "noopolis.daimon.wake.v2", kind: "manual" as const, text: "hello", occurred_at: "2026-09-18T00:00:00.000Z" } }); + +const core = { + async start(): Promise {}, + async wake(request: OrganizationRuntimeWakeRequest) { return { version: "noopolis.daimon.wake-result.v1", status: "completed", agentId: request.agentId, wakeId: request.event.id, text: "private", durationMs: 1 } as const; }, + async health() { return { version: "noopolis.daimon.organization-runtime-health.v1" as const, state: "running" as const, agents: [{ agentId: "alpha", engine: "codex" as const, state: "idle" as const }] }; }, + async activity() { return { version: "noopolis.daimon.organization-runtime-activity.v1" as const, items: [] }; }, + async stop() { return { version: "noopolis.daimon.organization-runtime-stop.v1" as const, state: "stopped" as const }; } +} as unknown as OrganizationRuntimeHost; + +/** + * A caller proving that a native execution closed has exactly one authority to + * read — the v2 activity projection — and the runtime that owns it is stopped by + * the time the proof is taken: the worker stops its own host as soon as a delivery + * closes its execution and stays deferred. Before this, `activityV2` answered + * `undefined` there (HTTP 503 `native_host_unavailable` through the caller's + * worker route), so a trial whose subject really ran, spent its budget and simply + * did not do the work reported as an unscorable infrastructure failure. + */ +test("a stopped control host still answers the closure query it alone can settle", async () => { + const root = await privateRoot(); + const control = createOrganizationRuntimeControlHostWithCoreForTest(config, core, { acceptanceStorePath: root, controlToken: token, storeOptions }); + try { + // Nothing has started: there is no projection to seal and none is invented. + assert.equal(await control.activityV2(token), undefined); + await control.start(); + const accepted = await control.accept(delivery()); + assert.equal(accepted.state, "accepted"); + const live = await control.activityV2(token); + assert.equal(live?.state, "running"); + assert.equal(live?.items.length, 1); + + assert.equal((await control.stop()).state, "stopped"); + const sealed = await control.activityV2(token); + assert.equal(sealed?.version, ACTIVITY_V2_VERSION); + // The same projection, said to be final: nothing can be admitted after it, so + // an empty execution list is a stronger quiescence statement than a live poll. + assert.equal(sealed?.state, "stopped"); + assert.deepEqual(sealed?.executions, []); + assert.equal(sealed?.items.length, 1); + assert.equal(sealed?.items[0]?.delivery_id, "delivery-1"); + assert.equal(sealed?.items[0]?.active, false); + // Repeating the query repeats the seal rather than draining it. + assert.deepEqual(await control.activityV2(token), sealed); + // The seal is not a bypass of authentication, and the store-backed routes that + // have no post-stop answer still report absence instead of an empty runtime. + assert.equal(await control.activityV2("wrong-token"), undefined); + assert.equal(await control.availability(token), undefined); + assert.equal(await control.wakeReceipt(token, accepted.state === "accepted" ? accepted.acceptance_id : ""), undefined); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +/** A host that was never started cannot attest anything, and a second stop keeps the seal. */ +test("an unstarted host seals nothing and a repeated stop does not erase the seal", async () => { + const root = await privateRoot(); + const unstarted = createOrganizationRuntimeControlHostWithCoreForTest(config, core, { acceptanceStorePath: root, controlToken: token, storeOptions }); + try { + assert.equal((await unstarted.stop()).state, "stopped"); + assert.equal(await unstarted.activityV2(token), undefined); + + const control = createOrganizationRuntimeControlHostWithCoreForTest(config, core, { acceptanceStorePath: root, controlToken: token, storeOptions }); + await control.start(); + await control.accept(delivery("delivery-2")); + await control.stop(); + const sealed = await control.activityV2(token); + assert.equal(sealed?.state, "stopped"); + await control.stop(); + assert.deepEqual(await control.activityV2(token), sealed); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +async function privateRoot(): Promise { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-closure-")); await chmod(root, 0o700); return root; } diff --git a/src/runtime/organizationRuntimeControl.ts b/src/runtime/organizationRuntimeControl.ts index e4d9c5c..386b6c5 100644 --- a/src/runtime/organizationRuntimeControl.ts +++ b/src/runtime/organizationRuntimeControl.ts @@ -47,6 +47,19 @@ function createControl(config: OrganizationRuntimeConfig, host: OrganizationRunt let fusePoll: ReturnType | undefined; let started = false; let stopping = false; + let sealedActivity: OrganizationRuntimeActivityV2 | undefined; + + /** + * One projection, read the same way live and at shutdown. `active` is decided by + * the dispatcher's own execution authority rather than the record's flag alone, + * so a stopped host — whose dispatcher has already awaited every in-flight turn + * — reports exactly the executions that were still admitted when it stopped. + */ + const projectActivity = async (current: WakeAcceptanceStore, state: "running" | "stopped"): Promise => { + const executions = dispatcher?.activeExecutions() ?? []; + const items = (await current.activity()).map((item) => ({ ...item, active: item.active && executions.some((execution) => execution.agent_id === item.agent_id && execution.delivery_ids.includes(item.delivery_id)) })); + return { version: ACTIVITY_V2_VERSION, state, items, executions }; + }; const hardReason = (): BlockReason | undefined => { if (!started || stopping) return stopping ? "host_stopping" : "host_stopped"; @@ -132,10 +145,15 @@ function createControl(config: OrganizationRuntimeConfig, host: OrganizationRunt return await store.status(acceptanceId); }, async activityV2(token) { - if (!tokensEqual(expectedToken, token) || store === undefined) return undefined; - const executions = dispatcher?.activeExecutions() ?? []; - const items = (await store.activity()).map((item) => ({ ...item, active: item.active && executions.some((execution) => execution.agent_id === item.agent_id && execution.delivery_ids.includes(item.delivery_id)) })); - return { version: ACTIVITY_V2_VERSION, items, executions }; + if (!tokensEqual(expectedToken, token)) return undefined; + // A stopped host is not an unanswerable one. Its sealed projection is a + // *stronger* statement about quiescence than a live poll, because nothing + // can be admitted after it, and a caller proving that an execution closed + // has no other authority to read. A host that never started, or one whose + // seal could not be taken, still answers nothing: absence stays absence + // rather than becoming a fabricated idle runtime. + if (store === undefined) return sealedActivity; + return await projectActivity(store, "running"); }, async availability(token) { if (!tokensEqual(expectedToken, token) || !store || !fuse) return undefined; @@ -161,6 +179,11 @@ function createControl(config: OrganizationRuntimeConfig, host: OrganizationRunt await Promise.allSettled(persistence); const result = await host.stop(); await dispatcher?.stop(); + // The last moment the store can be read, and the only one at which the + // dispatcher has finished every admitted turn. Seal the projection here so + // the closure query keeps an accurate answer once the store is closed; a + // fault leaves it absent instead of inventing one. + if (store) { try { sealedActivity = await projectActivity(store, "stopped"); } catch { /* an unreadable final state stays absent */ } } await store?.close(); await fuse?.close(); store = undefined; fuse = undefined; schedules = undefined; started = false; return result; diff --git a/src/runtime/wakeAcceptanceTypes.ts b/src/runtime/wakeAcceptanceTypes.ts index 1bc8e40..e21de59 100644 --- a/src/runtime/wakeAcceptanceTypes.ts +++ b/src/runtime/wakeAcceptanceTypes.ts @@ -71,6 +71,12 @@ export type OrganizationRuntimeActivityV2Item = OrganizationRuntimeWakeReceiptSt }>; export type OrganizationRuntimeActivityV2 = Readonly<{ version: typeof ACTIVITY_V2_VERSION; + /** + * Whether this projection was read from a live runtime or sealed as the host + * stopped. Optional on the wire because a projection published before the seal + * existed must still parse; its absence means "not stated", never "running". + */ + state?: "running" | "stopped"; items: readonly OrganizationRuntimeActivityV2Item[]; executions?: readonly Readonly<{ agent_id: string; execution_id: string; state: "running"; delivery_ids: readonly string[] }>[]; }>;