From 024e1026eabd2a5b5d835addcbe8dca27cad6d22 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:39:29 +0200 Subject: [PATCH 01/69] feat: give each Grok worker a private TMPDIR under its registered home --- src/contracts/runtimeContractManifest.ts | 11 ++++++++++- src/runtime/native/engineBrokerLauncherCore.inc | 6 +++++- .../engineBrokerLauncherIntegrationLauncher.inc | 2 +- src/runtime/native/fixtureWorker.c | 2 +- src/runtime/native/launcherArgv.test.ts | 9 ++++++++- 5 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index a9eb1be..2235701 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -64,7 +64,16 @@ export const GROK_ENGINE_BROKER = { directory: { uid: 0, group: "worker", mode: 0o1771 }, sessionsDirectory: { relativePath: "sessions", uid: 0, group: "worker", mode: 0o1771 }, readOnlyFiles: { names: ["config.toml", "managed_config.toml", "requirements.toml", "sandbox.toml", "trusted_folders.toml"], uid: 0, gid: 0, mode: 0o444 }, - sandboxEvents: { relativePath: "sessions/sandbox-events.jsonl", owner: "worker", group: "broker", mode: 0o640 } + sandboxEvents: { relativePath: "sessions/sandbox-events.jsonl", owner: "worker", group: "broker", mode: 0o640 }, + // The launcher exports TMPDIR=/tmp; Grok's strict profile grants TMPDIR read-write. + privateTmp: { relativeToWorkerHome: "tmp", owner: "worker", mode: 0o700 }, + // Strict also grants shared /tmp and /var/tmp read-write and refuses to start if either is + // denied, so the deployment keeps them from every worker by mode: root-owned, a non-worker + // group (< 2200), others read-only (Grok needs to open the directory) and no search/write. + sharedTmp: { paths: ["/tmp", "/var/tmp"], uid: 0, maxGroupExclusive: 2_200, otherMode: 0o4, mode: 0o1774 }, + // Spilled tool output the worker reads with read_file: setgid directory in the worker's group, + // files written 0640 by the runtime, never other-readable. + spillDirectory: { relativeToRuntimeHome: "tool-output", owner: "organization", group: "worker", mode: 0o2750, fileMode: 0o640 } } }, bounds: { promptBytes: 65_536, capabilityBytes: 4_096, capabilityBundleBytes: 8_196, outputBytes: 65_536 }, diff --git a/src/runtime/native/engineBrokerLauncherCore.inc b/src/runtime/native/engineBrokerLauncherCore.inc index cb2b9ec..41cc947 100644 --- a/src/runtime/native/engineBrokerLauncherCore.inc +++ b/src/runtime/native/engineBrokerLauncherCore.inc @@ -361,10 +361,13 @@ static pid_t launch(const struct dbl_registration *r, int executable, "--model", "daimon-broker-grok", NULL}; - char home[300], grok[300], mcp_env[DBL_MAX_TOKEN + 24], + char home[300], grok[300], tmp[300], mcp_env[DBL_MAX_TOKEN + 24], provider_env[DBL_MAX_TOKEN + 32]; snprintf(home, sizeof(home), "HOME=%s", r->home); snprintf(grok, sizeof(grok), "GROK_HOME=%s/.grok", r->home); + /* Worker-private temp: strict grants TMPDIR read-write, and shared /tmp is + kept from the worker by the deployment's modes (attested by the broker). */ + snprintf(tmp, sizeof(tmp), "TMPDIR=%s/tmp", r->home); snprintf(mcp_env, sizeof(mcp_env), "DAIMON_MCP_CAPABILITY=%s", mcp); /* Grok 1.0.34 never runs `[auth_provider.*]` helpers for a custom model, so the worker's `[model.daimon-broker-grok] env_key` reads the turn-scoped @@ -375,6 +378,7 @@ static pid_t launch(const struct dbl_registration *r, int executable, erase(mcp, sizeof(mcp)); char *const envp[] = {home, grok, + tmp, mcp_env, provider_env, "DAIMON_CAPABILITY_FD=4", diff --git a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc index ee189d9..1512c47 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc @@ -110,7 +110,7 @@ static void org_cases(void) { char *out = calloc(1, r.output_length + 1); check(read(s, out, r.output_length) == (ssize_t)r.output_length && strstr(out, "uid=2200") && strstr(out, "--always-approve") && - strstr(out, " prompt=prompt fds=0,1,2,3,4 stdin=/dev/null\n") && + strstr(out, " prompt=prompt fds=0,1,2,3,4 stdin=/dev/null tmpdir=/tmp/worker-home/tmp\n") && !strstr(out, "EVIL"), "fixed worker boundary"); check(strstr(out, "=--verbatim\n") && strstr(out, "=--no-plan\n") && diff --git a/src/runtime/native/fixtureWorker.c b/src/runtime/native/fixtureWorker.c index b26a4ae..cb92d4e 100644 --- a/src/runtime/native/fixtureWorker.c +++ b/src/runtime/native/fixtureWorker.c @@ -6,4 +6,4 @@ #include #include #include "engineBrokerLauncher.h" -int main(int argc,char**argv){unsigned char bundle[8200]={0};ssize_t size=pread(4,bundle,sizeof(bundle),0);if(size<6)return 21;size_t offset=0;unsigned provider_length=bundle[0]|(bundle[1]<<8);offset=2;if(offset+provider_length+2>(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output");if(!strcmp(provider,"sleep"))sleep(30);if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;id_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target);for(int i=0;i(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output");if(!strcmp(provider,"sleep"))sleep(30);if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;id_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s tmpdir=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target,getenv("TMPDIR")?getenv("TMPDIR"):"unset");for(int i=0;i { const source = read("engineBrokerLauncherCore.inc"); assert.match(source, new RegExp(`"${GROK_BROKER_PROVIDER_CAPABILITY_ENV}=%s"`, "u")); - assert.match(source, /char \*const envp\[\] = \{home,\s*grok,\s*mcp_env,\s*provider_env,/u); + assert.match(source, /char \*const envp\[\] = \{home,\s*grok,\s*tmp,\s*mcp_env,\s*provider_env,/u); assert.match(renderGrokBrokerWorkerConfig(), new RegExp(`\\nenv_key = "${GROK_BROKER_PROVIDER_CAPABILITY_ENV}"\\n`, "u")); }); + +test("the launcher gives every worker a private TMPDIR under its registered home", () => { + const source = read("engineBrokerLauncherCore.inc"); + assert.match(source, /snprintf\(tmp, sizeof\(tmp\), "TMPDIR=%s\/tmp", r->home\);/u); + assert.match(source, /char \*const envp\[\] = \{home,\s*grok,\s*tmp,/u); + assert.equal(GROK_ENGINE_BROKER.worker.home.privateTmp.relativeToWorkerHome, "tmp"); +}); From aa1aef0f1a1a887501838172cc76e96a2aeb8d5c Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:39:29 +0200 Subject: [PATCH 02/69] feat: refuse Grok turns unless shared temp is closed to workers and the private temp is worker-only --- src/runtime/grokWorkerAttestation.ts | 6 ++- .../grokWorkerAttestationChecks.test.ts | 37 ++++++++++--- src/runtime/grokWorkerTmpAttestation.test.ts | 53 +++++++++++++++++++ src/runtime/grokWorkerTmpAttestation.ts | 48 +++++++++++++++++ 4 files changed, 136 insertions(+), 8 deletions(-) create mode 100644 src/runtime/grokWorkerTmpAttestation.test.ts create mode 100644 src/runtime/grokWorkerTmpAttestation.ts diff --git a/src/runtime/grokWorkerAttestation.ts b/src/runtime/grokWorkerAttestation.ts index adfa3f5..d10d01f 100644 --- a/src/runtime/grokWorkerAttestation.ts +++ b/src/runtime/grokWorkerAttestation.ts @@ -4,6 +4,7 @@ import { lstat,open } from "node:fs/promises"; import path from "node:path"; import { verifyGrokWorkerHome } from "./grokWorkerHomeAttestation.js"; +import { verifyGrokWorkerTmp } from "./grokWorkerTmpAttestation.js"; import { grokWorkerEventsPathFor } from "./grokWorkerSandboxProfile.js"; /** @@ -61,9 +62,12 @@ export function parseGrokWorkerSandboxProfile(bytes:Uint8Array,profileSha256:str * 1.0.34), and a root-owned read-only worker home whose `config.toml` hashes to * the declared renderer output (`grokWorkerHomeAttestation.ts`). */ -export async function prepareGrokWorkerAttestation(input:Readonly<{profilePath:string;eventsPath:string;profileSha256:string;workerUid:number;brokerGid:number;configSha256:string}>,profileOwner:Readonly<{uid:number;gid:number}>={uid:0,gid:0}):Promise{ +export async function prepareGrokWorkerAttestation(input:Readonly<{profilePath:string;eventsPath:string;profileSha256:string;workerUid:number;brokerGid:number;configSha256:string}>,profileOwner:Readonly<{uid:number;gid:number;sharedTmpRoots?:readonly string[]}>={uid:0,gid:0}):Promise{ if(input.eventsPath!==grokWorkerEventsPathFor(input.profilePath))throw new Error("Grok worker sandbox events must be read from $GROK_HOME/sessions/sandbox-events.jsonl"); const profile=await secureOpen(input.profilePath,profileOwner.uid,profileOwner.gid,0o444,65_536);let bytes:Buffer|undefined;let denyPaths:readonly string[]=[];try{bytes=await profile.readFile();denyPaths=parseGrokWorkerSandboxProfile(bytes,input.profileSha256);}catch{throw new Error("Grok worker isolation attestation unavailable");}finally{bytes?.fill(0);await profile.close();} + // The launcher exports TMPDIR=/tmp; the profile lives at /.grok/sandbox.toml. + if(path.basename(path.dirname(input.profilePath))!==".grok")throw new Error("Grok worker isolation attestation unavailable"); + await verifyGrokWorkerTmp(path.dirname(path.dirname(input.profilePath)),input.workerUid,profileOwner.sharedTmpRoots); await verifyGrokWorkerHome(path.dirname(input.profilePath),input.configSha256); const events=await secureOpen(input.eventsPath,input.workerUid,input.brokerGid,0o640,16*1024*1024);try{const stat=await events.stat();return{dev:Number(stat.dev),ino:Number(stat.ino),size:Number(stat.size),denyPaths};}finally{await events.close();} } diff --git a/src/runtime/grokWorkerAttestationChecks.test.ts b/src/runtime/grokWorkerAttestationChecks.test.ts index 694391c..b5568a6 100644 --- a/src/runtime/grokWorkerAttestationChecks.test.ts +++ b/src/runtime/grokWorkerAttestationChecks.test.ts @@ -61,17 +61,40 @@ test("refuses events that change while they are being read", async (t) => { assert.ok(reading.mock.callCount() >= 1); }); -test("prepare refuses a worker home that fails attestation even when profile and events are valid", async (t) => { - // Run as a non-root owner so the profile and events legs pass; the home leg - // (root-owned, read-only config) cannot, and must be what refuses. - const home = await mkdtemp(path.join(os.tmpdir(), "daimon-guard-home-")); - t.after(() => rm(home, { recursive: true, force: true })); +test("prepare refuses a worker home that fails attestation even when profile, temp and events are valid", async (t) => { + // Run as a non-root owner so the profile, temp and events legs pass; the home + // leg (root-owned, read-only config) cannot, and must be what refuses. + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-guard-home-")); + t.after(() => rm(root, { recursive: true, force: true })); + const home = path.join(root, ".grok"); + await mkdir(path.join(home, "sessions"), { recursive: true }); + await mkdir(path.join(root, "tmp"), { mode: 0o700 }); const profile = path.join(home, "sandbox.toml"); const text = '[profiles.daimon-strict]\nextends = "strict"\nrestrict_network = true\ndeny = []\n'; await writeFile(profile, text); await chmod(profile, 0o444); - await mkdir(path.join(home, "sessions")); const events = path.join(home, "sessions", "sandbox-events.jsonl"); await writeFile(events, ""); await chmod(events, 0o640); const input = { profilePath: profile, eventsPath: events, profileSha256: createHash("sha256").update(text).digest("hex"), workerUid: self.uid, brokerGid: self.gid, configSha256: "0".repeat(64) }; - await assert.rejects(prepareGrokWorkerAttestation(input, { uid: self.uid, gid: Number((await stat(profile)).gid) }), /attestation unavailable/u); + await assert.rejects(prepareGrokWorkerAttestation(input, { uid: self.uid, gid: Number((await stat(profile)).gid), sharedTmpRoots: [] }), (error: Error) => error.message === "Grok worker isolation attestation unavailable"); +}); + +test("prepare refuses a worker without a private temp directory before the home check", async (t) => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-guard-tmp-")); + t.after(() => rm(root, { recursive: true, force: true })); + const grokHome = path.join(root, ".grok"); + await mkdir(path.join(grokHome, "sessions"), { recursive: true }); + const profile = path.join(grokHome, "sandbox.toml"); + const text = '[profiles.daimon-strict]\nextends = "strict"\nrestrict_network = true\ndeny = []\n'; + await writeFile(profile, text); await chmod(profile, 0o444); + const events = path.join(grokHome, "sessions", "sandbox-events.jsonl"); + await writeFile(events, ""); await chmod(events, 0o640); + const input = { profilePath: profile, eventsPath: events, profileSha256: createHash("sha256").update(text).digest("hex"), workerUid: self.uid, brokerGid: self.gid, configSha256: "0".repeat(64) }; + const owner = { uid: self.uid, gid: Number((await stat(profile)).gid) }; + // No /tmp: the temp leg refuses (the home leg would refuse too, with a different message). + await assert.rejects(prepareGrokWorkerAttestation(input, owner), /temp isolation attestation unavailable/u); + // A profile outside /.grok cannot name the launcher's TMPDIR home. + await mkdir(path.join(root, "elsewhere", "sessions"), { recursive: true }); + const stray = { ...input, profilePath: path.join(root, "elsewhere", "sandbox.toml"), eventsPath: path.join(root, "elsewhere", "sessions", "sandbox-events.jsonl") }; + await writeFile(stray.profilePath, text); await chmod(stray.profilePath, 0o444); await writeFile(stray.eventsPath, ""); await chmod(stray.eventsPath, 0o640); + await assert.rejects(prepareGrokWorkerAttestation(stray, owner), (error: Error) => /attestation unavailable/u.test(error.message) && !/temp/u.test(error.message)); }); diff --git a/src/runtime/grokWorkerTmpAttestation.test.ts b/src/runtime/grokWorkerTmpAttestation.test.ts new file mode 100644 index 0000000..e05016a --- /dev/null +++ b/src/runtime/grokWorkerTmpAttestation.test.ts @@ -0,0 +1,53 @@ +import assert from "node:assert/strict"; +import { chmod, mkdir, mkdtemp, rm, symlink } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { assertGrokWorkerTmpEntries, verifyGrokWorkerTmp } from "./grokWorkerTmpAttestation.js"; + +const worker = 2200; +const dir = (mode: number, uid: number, gid: number, kind: "dir" | "link" | "file" = "dir") => ({ + uid, gid, mode: (kind === "dir" ? 0o040000 : kind === "link" ? 0o120000 : 0o100000) | mode, + isDirectory: () => kind === "dir", isSymbolicLink: () => kind === "link" +}); +type Entry = ReturnType; +const good = (): { privateTmp: Entry | undefined; shared: (Entry | undefined)[] } => ({ privateTmp: dir(0o700, worker, worker), shared: [dir(0o1774, 0, 2000), dir(0o1774, 0, 2000)] }); + +test("accepts private worker temp and shared temp roots the worker cannot open or write", () => { + assert.doesNotThrow(() => assertGrokWorkerTmpEntries(good(), worker)); + assert.doesNotThrow(() => assertGrokWorkerTmpEntries({ ...good(), shared: [dir(0o1770, 0, 2000), dir(0o700, 0, 0)] }, worker)); +}); + +test("refuses shared /tmp or /var/tmp a worker could traverse, write, or own through its group", () => { + const refusals: Record = { + "shared 1777 (default /tmp)": { ...good(), shared: [dir(0o1777, 0, 0), dir(0o1774, 0, 2000)] }, + "shared other search": { ...good(), shared: [dir(0o1774, 0, 2000), dir(0o1775, 0, 2000)] }, + "shared other write": { ...good(), shared: [dir(0o1776, 0, 2000), dir(0o1774, 0, 2000)] }, + "shared owned by a worker group": { ...good(), shared: [dir(0o1774, 0, worker), dir(0o1774, 0, 2000)] }, + "shared owned by the org user": { ...good(), shared: [dir(0o1774, 2000, 2000), dir(0o1774, 0, 2000)] }, + "shared missing": { ...good(), shared: [undefined, dir(0o1774, 0, 2000)] }, + "shared symlink": { ...good(), shared: [dir(0o777, 0, 0, "link"), dir(0o1774, 0, 2000)] }, + "private missing": { ...good(), privateTmp: undefined }, + "private owned by another worker": { ...good(), privateTmp: dir(0o700, worker + 1, worker + 1) }, + "private group readable": { ...good(), privateTmp: dir(0o750, worker, worker) }, + "private symlink": { ...good(), privateTmp: dir(0o700, worker, worker, "link") }, + "private is a file": { ...good(), privateTmp: dir(0o600, worker, worker, "file") } + }; + for (const [label, entries] of Object.entries(refusals)) assert.throws(() => assertGrokWorkerTmpEntries(entries, worker), /temp isolation attestation unavailable/u, label); +}); + +test("checks the real private temp directory under the worker home and the given shared roots", async (t) => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-grok-tmp-")); + t.after(() => rm(root, { recursive: true, force: true })); + const uid = process.getuid?.() ?? 0; + const home = path.join(root, "home"), shared = path.join(root, "shared"); + await mkdir(path.join(home, "tmp"), { recursive: true }); await mkdir(shared); + await chmod(path.join(home, "tmp"), 0o700); + // A shared root owned by the test user is refused (not root-owned) — as is a symlinked private temp. + await assert.rejects(verifyGrokWorkerTmp(home, uid, [shared]), /temp isolation attestation unavailable/u); + await rm(path.join(home, "tmp"), { recursive: true }); await symlink(shared, path.join(home, "tmp")); + await assert.rejects(verifyGrokWorkerTmp(home, uid, []), /temp isolation attestation unavailable/u); + await rm(path.join(home, "tmp")); await mkdir(path.join(home, "tmp"), { mode: 0o700 }); + await verifyGrokWorkerTmp(home, uid, []); +}); diff --git a/src/runtime/grokWorkerTmpAttestation.ts b/src/runtime/grokWorkerTmpAttestation.ts new file mode 100644 index 0000000..4d2cf7b --- /dev/null +++ b/src/runtime/grokWorkerTmpAttestation.ts @@ -0,0 +1,48 @@ +import type { Stats } from "node:fs"; +import { lstat } from "node:fs/promises"; +import path from "node:path"; + +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; + +type Entry = Pick & Readonly<{ isDirectory(): boolean; isSymbolicLink(): boolean }>; +const HOME = GROK_ENGINE_BROKER.worker.home; + +/** + * Temp-directory isolation for one worker, checked before every turn. + * + * Grok 1.0.34's strict profile grants shared `/tmp` and `/var/tmp` + * read-write, and it refuses to start when either (or any ancestor of a + * granted path) is in `deny` — verified live: `deny = ["/tmp"]`, + * `["/var/tmp"]`, `["/run"]`, `["/etc"]` all fail with "could not apply the + * sandbox profile", while `["/tmp/sub"]` works. So the kernel profile cannot + * keep evaluator temp files away from the worker. Unix modes can, because the + * launcher drops the worker to its own uid/gid with no supplementary groups: + * + * - shared temp roots are root-owned, owned by a group below the worker range, + * and give "other" at most read (Grok opens the directory; without search + * the worker can list names but cannot open, stat, or create anything); a + * deployment that lets workers traverse or write them is refused; + * - the worker's own `/tmp` (the launcher's compiled `TMPDIR`, which + * strict grants read-write) is a real directory owned by the worker with no + * group or other access. + * + * Pure so every refusal is testable without root. + */ +export function assertGrokWorkerTmpEntries(entries: Readonly<{ privateTmp: Entry | undefined; shared: readonly (Entry | undefined)[] }>, workerUid: number): void { + const shared = HOME.sharedTmp; + for (const entry of entries.shared) { + if (entry === undefined || !entry.isDirectory() || entry.isSymbolicLink() || entry.uid !== shared.uid || entry.gid >= shared.maxGroupExclusive || (Number(entry.mode) & 0o007 & ~shared.otherMode) !== 0) throw unavailable(); + } + const own = entries.privateTmp; + if (own === undefined || !own.isDirectory() || own.isSymbolicLink() || own.uid !== workerUid || (Number(own.mode) & 0o077) !== 0) throw unavailable(); +} + +export async function verifyGrokWorkerTmp(workerHome: string, workerUid: number, sharedRoots: readonly string[] = HOME.sharedTmp.paths): Promise { + const inspect = async (file: string): Promise => { try { return await lstat(file); } catch { return undefined; } }; + assertGrokWorkerTmpEntries({ + privateTmp: await inspect(path.join(workerHome, HOME.privateTmp.relativeToWorkerHome)), + shared: await Promise.all(sharedRoots.map(inspect)) + }, workerUid); +} + +const unavailable = (): Error => new Error("Grok worker temp isolation attestation unavailable"); From faae4cdd2a760d25ebdef7e0a17c9171f09e295d Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:39:47 +0200 Subject: [PATCH 03/69] build: rebuild native engine broker artifacts with the worker TMPDIR --- src/contracts/runtimeContractManifest.ts | 6 +++--- .../artifacts/daimon-engine-broker-arm64 | Bin 67456 -> 67456 bytes ...daimon-engine-broker-arm64.provenance.json | 2 +- .../native/artifacts/daimon-engine-broker-x64 | Bin 55392 -> 55392 bytes .../daimon-engine-broker-x64.provenance.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 2235701..e56f6ae 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -129,9 +129,9 @@ export const GROK_ENGINE_BROKER = { projectionVersion: "noopolis.daimon.grok-broker-projection.v1", slotPreflightVersion: "noopolis.daimon.grok-slot-preflight.v2", artifacts: { - sourceSha256: "36f60689f0a8af0e3108f5f53d78ed52b7d4b6f934c75b6184606dfa82bc741e", - x64Sha256: "36dc76b134eb59cf5a6720b6f94228eb279108e20ea3343fa6efd9ffcb60a4d3", - arm64Sha256: "c93216cc6fa4ca50dc404fe41e68da9150a869b14f46eb42484ae77c3aa400a9" + sourceSha256: "27fcdc8ffb6946e391dad039f695968231d1ae435c6a38dd9bb8c7fb00ed18b9", + x64Sha256: "ff83efc37c77c4ee920080d0eedfca497b504a561c876348af0fec25142a5f9c", + arm64Sha256: "25d37be0d294529b3466d73c0d879d18c7850c2d24450a7a9cc11d28b9a4cf1e" } } as const; export const AGY_SUBSCRIPTION_REALM = { diff --git a/src/runtime/native/artifacts/daimon-engine-broker-arm64 b/src/runtime/native/artifacts/daimon-engine-broker-arm64 index daeaf6e3c64540ced18ab7c60f8e232824e7ba86..712890f0a30f784d1246975a870e3a558743b823 100755 GIT binary patch delta 2458 zcmY*b32;;A5&r-88dDy4#C6k^NV-5X2I+0nVSFw<&^_3}#n?#d` z4TOBiZG%h23XyXm7qO8yc%Wy)0on`88^-7v|4SP)MfyGP8Ft}sa!xn>zW>KLIh4Kt zlg33F@E^^6Hx6N2NoFiP4?R^z`bRiiRa5-Fby+5|LyjSH$InJDol%sugqatEN>||r z@h66Y%lnCUStRWGQ`CE7UI$9mjvP`+Dne%|qtxTrv8cG^3R0omR@}EmV$LSy`8O(I zsM=1waIU(9o`hS~zHRzaQbCCvBq()wi>HKCs4%UFBA^&h2$dbrqAyTPD$XQX=fa|3 z4xwyEPS3fxHLj7TErMH*Y!s|Tkoss&s-cin2!iZ5Ez1rqdU-MW0(ktN<&`w0#8nH9XgJ`R{KbQO0n|V=D+`no!piQ|S3^mn4R|R~hrkGB_ zftt*i6C=TJSNU9vPDWWs*@0tR9_Kez0Tv1t@O;5nK+5(}gABJl?2I2|18i&%| zP|Df|OPLz=Sd{#5DT_6fu^7~&Q7*a4*f~!bi$Yz6@_ML@jabXsm4bENwHS4*y`i3c zZPa-0#~G|=v6J=eQl{j+A1hGL&Jhdyhc-FUdq1Gd!eDouuJ*I$Xm;ndXeMG7Y|X1( zaycV`a`t6Rl2<^RqS&mPnL%I2K4})%H(l{;UetQ$#Y`W}))|~e4f`rB-McWb-s``m zxH&(VCkI1fhA)SZF&iOl`v3UU72ZYIRD{?1s5pe{Q&xBnVUr-$>(=Qho4^IesUh5A z<0x3`Q&zS+hi~_Wfl10@slZH_WE09IkHF4_1h!xo*!-}_c=1tW$YIlOW|Q~=^+q{( zr(5m4^>mzf(G>5!oSoqPY9P^DiVYkR(|y@`@nbGwk@$#;Fbg4%P&^RfF2iUxp;sCd z#mW7=(6^!qgRAvA;YUl*ZYh_Hcz8m+*0}>%Z)mS|?nP*`Y%WXYte6&Qua2b^4{xPy ztNI}7Pp#@ZP~QsE7JEiJ#>iHYC-}>#SCMtmL{839k&+Doh_gOU$DzY&s%!p6arY74 ztvYiRPrBbaXB8GJ_e8i=4KO}SS-o*nO z;ZyOUM}nuI%+{DdM7Df4e!A0|zH}4@ZCY9fhirDb2%p+4YTZpNbeDDRXJD|KQbK6y zDVf!`L^DAz2Py8#XRzH~Mf>59-6R$B#LLRVVJr*}vTqbF*wfPkBICg=cr!ozp}R^| z7`tyCmhBe9SwhBTLMqA$(OpEHPgenJ3*#&2yWxiIji!79CF6P(f1-4sp(Tz9Dmm!H zc(RBS4_EO~g~bH_1JjM!)D9OKjr0^;Lp>FwCM}g9uW2*A2pvt9j4uW42ZMrk6dT3d zgfV&rf(=BjC92__rj4q6N}LdCdZ_kZ!*LYEd-tT8zrL|{&-Ak597rP9)YyDChAF$# zt@5rXBMz`OXF1nD5d5bq;_MW)S7LFY77MjK5b)0r1l%I7t-CFxcT-I8U?Fg_IhUS= zkD5)J-p2OjvuE`{Jdq<4HuPMej&s8z9w^+C!QGJ4Vv+h*SW&ggz5qQfFVW9|wsuPI zUsv2=F%>skhX))713ceaLqCG)Rtuekf3;@PcR=0t7`+bPZp)UA<10G;ESaO&hWw;; z`G3-e+xsd8xfdXev;(~DCV!~CIxo?=Wou)bqb{$vr0)m2{VBUI(E;L9l=C?0G{3>! zz%dw@Rg_#DWcs%hr5bT7;)*2P%AmjX?h!S8(;ZgPlDM@$SMH1`TB+9bM(!Y*Qyo#%B>WE?jHAE+ delta 2388 zcmYjT4NP0t6~6a9Lu?=kFUI^6|B*UzC}4g9DL*#0`OCVNF@%=35w_??S5henkSvNc zo(=0VQl;FTv`&+>A?v7SYni&4t?@QiXiF5W-9*{Qx@w#3E^XSS-9RNj49MMiCX77k zyzidx+;`4-_kH);^7AVBdDZZ=X80(HU&HLLZ*G}8xazE9I7H9pJQx*ORMduSoCioE zDKio>t@uWRiWMN|K`!niZ}h;SydnAwtmF+-fB59q9FdwJWbDHG7E?EUGW=bWiPA%G z!F-3x;WPQaNurm+?L|3iIsu0&%=8_2v7)N*w~iH=$gT1Q5;^~N;__RZXQT`rpW&Sa z?04`S+(P!aSWvJa#M9L|VX9!viJ`yb?h zDB3>cJP#t5w9dJevITH!_f|nJfb@q<>E4~BToB~eNm*{yVpM6yV}(Z^@_%h$+ntK2 zm}|ubS|z1G+|RLk(cg%^oUg>Qtv>WMN<8joS(T;|l(<=mXIbmer%D`*Rh2?%89ZK9 zNPRF=l|%jTT2&(*f`3=-#5c~a&Ov@xwI2EQ>YYMqFAP-g?N1_PPJ_0mu9)rc6*D!C zH=wCw#q7Gl#u9OyfOgqqW1}NBro!=H)W$@_6tqr9346|0!jf^^U01{YW!W5D&smE( zYuTsz)ZqW~UZ`c)Cu-Rz8B!2)?_Nz#>}=FyXV*4w3C0V^b|zp#ispMI`C1aEdG1wS(<2e0R(1bH`EEJbZ9m0}_Z z5fx!}TnmaCk98VP=n1`1$9dZi?YUHjJbS6M19w2Q#}OIEexgSlk(1!Hm)bHFR?LWN zq+@wCqUhm?dt*gAd+&|CIBth2yDPf|bLds%Q3Li^t0J2diM%COMT+t+fywa*Jp%_F zmYRqE%LBbckyx6wMk#vC5pf82b{&p$>DEex>qid#avbwSV)GEvrnF;&S%;qPfSZmk z8i0LuV7s`80^Nwm!~>KW)$&XbeK`evtH6w@^313bSNuX%+`j-uXNEMn!~=$8<#0*I zR~(!fhi0eSFd?v&rR0kpj?Cqeo7hAbi4Zk5+NEs7?2rpCI$iWG%scIx?0DP}ISspA z7WxTzU0NE0K34@j1{YnHx-zAW69Giq>3iqwcKGNoQU z#+5A`UE&o=t4UQNYK9P z6SOB#yVhoWrbi&C7Bc^Z8h&5DRozTUn=%zXuisuNh--7xt^ZkApS$rcZ|mPg<~69W zg#pSIyH&v~GT6FS16mq#Tk`Hp{)~#Wb%@#bP_4fGP||3Z4y>}G>K@w* z{>GCu3Q|*tbmdDPh~f4|mEx9_XRd*-sfy0Rc$1xu!b(#P{T*aAe}&$LZ#L_tSMfug zf0{%nDp1+{UEvp+wN!cyldL6(A>9s2X~c7gX>W3#iAQW1VzEd{$m=-x1`=r#I1zDdlJhadY5Z+mMqHTU{0UsWi9cW= z#P7iPA-jewa=sngH9$g-Mfc+{7DGLgx(!=X5em@epsq)&OTpKvRw8&A(Av<-;M+ZB z-IEx>bnz<<@6VwffD1ht!c${V+gqqHAn`Y}pTc~PndXA7*Fp!OrniD#fxg}%S^%TH z$LInW`!eV zlK%3+?A1U0`S1VOMCy{idS*cV(xCcV07q)5BmDl64oauNb~Hox!90FQtf^dXv?lyb udMCW^=nWr4X==D>>9mHP55!coC~5s~Q6M{k?$VH+_)~0~!BP@P*B3}>U~mQlwH&pA)>4kG2hW&k zNyGY9f-M)N3Mys7@C-*Kh@+DrhZEy|`!iBc7nxHAZ_;g#j1B;ND5_A9X_VRM>KdfeRRZTvV)pO%4AEpe&l%r z`Gf{Wcacr>u{-`ieo4Q$BS!MWjd!j~AW@n-ekC~_J~)0NAvM&HxrDfAapn+mSGYRU zVI<>e)1=#pjh>#=LY@l0bdS+MQt6h-ZVwpO7e$NXuHOiFS=#7eUw-B+AXaKsu~xE zC`NAik}~@nb>hJ%P)sSb!k!fo1e2hg`jo?dF?c_4#e9x2m)wQ~Rj7~H_w|LkN;3MEe{6@-YB zXZV4GQmD#q86gNBxlNMW{1Sqq(CdsW_ zF;!U)HFfN^Ta@2nAl1yVTcqG|aZNr%TdR_|njm2+sgcn<{Lgqud?DP$?T!j9H;B8P zhAIF<0W{$we32}syOD(5c|;1d5zZs@Rp;o`Pf#T(RA$fMa~G7q{7cKkMtaeiKCm!B z&%joA;0Y?A>8_!E2VfW6-y(l2f3F-rqw$95s7Xav=%^8>^7IA9avf2}A;)nfG}^4X zI^c%)yv@Q|Qf`uK-jz6F+=VktCvL5ZsA zQZsn{2L%Cn9RrJdpoQi0BisVK0XRI3>Seg(R%)A?;y(cHGS2nd$H=WPW2!+_5)Zl) zyWNROVz*eG0QZh-@;l0h5JRZd*No~avqX5>m6(op0*&ytS%*z)f530l%*=Ve3&EJ#MM>tW|ec`v>Jsx8@MWqm87&^)iQ}Vcd2Vm zc@YLM<1Dr4Qq3-VZ7$53&TD8eT0T&!sj}B*0F?q+*gI9+?GkA5WqUbE5Naj)yc9S< zS`C6U@j&@yj*}6zeA>?kUxgSiG}1q{}}ESTpzQ+N)OR6udVH0>L!eC9z;1N+0+-?x~StdIfjj3g!Mg@U`KI0l5<| zm6>>%V39EZyz-?Y zvA89XGupv;g`Rb%J=*pxLJJxH3@Og&#QWdzjs`QOm`}5KABPpdy8(-%tL*%g;aE3! zU8|{wb@i_yw*&89cc@XOTxNIUv>e7s5-(xS0oaT)Y*L^AdIN5&@Frb@qCsO&smE`?y9c=kj&&47gm} z<_&?RZf>6PT3xu1%kAf}ueNUj#%u4|BVY#2Q;|GwuM|@rvc#`teAzm!);zlvkXIV{ zJmrp?rdWhczK`4UseK@-ZL@>rT~ZCg;EYk3Fw&b7RX*^8+~o+GBR|`ebC-H^i1YC$ z)7EDzcRl%?){T_v57d`z#gCVsO8k#TCcuc^cbRfOqNrJsZO>L4rZP(%VubVv$~1k{ zHB$4Nt*R+wdW6(wF^u}95GkbtJT`M4HYhKEmoLrD^d8SI+~4BLA)g7JT5eRt4mIeE zAj80bfzSks=_ZkWa=%MnWoRYGi7BrWApEG$uTIbx&EzI7-VzNOi$WM zZF$9Mh*`8I6p2fXPXZCGc@_uTUqlOhw+~vN ziP(BY=F^yOsiCVqoH=U>A=|>`vu7p{YdALl#p&Zq?%{bWGvnTM9430sv$-hF4lBQ! z<6a)PU}`hEAgFGEdGwi=?m~lDL4~$(zkI)Hd1SJ8Y!l8qZK3=Hda7KAIt~ry*}83z}jg`%b>yBLJD_`hfi}g zI0}~zS0-;yq;-d<+}87FLC>E9*BVu6LD@yG93E@!+^P)+YecB0jw9(fEoL2QCKhTv znvO|0jxHqw!VO2$`(e=X?zlt?_l3t2{^s2TLO!PC#JI#BOPzG$iR-lVWCkrfX-Vw) alVlTp^5ostZKpKFZHG1L3%8tniTnpMpn$Oe delta 4470 zcmb7I3vg3a8onoOXv%7CVq4nEqXDkkLRVUWwJ4MXg1vzNL19*QKyko|Qn9S9e?K(SbO8I1V4s;iYuZUGEYvHQ%j}H^Oyy ze`_Ee)R)tR>^wW{Wio`G9o9kyh8u?0CXog@Y2+*9V7P7M7()I^4WpkX`LtwofAFhD z=NrjT+VsE>VxeC=5F?Aj?>uNU5EETD?htX&vcko%`&Hpx-b333gCIC&xs>aUnNyVm zvmTKv&t*G#j60-g0;j8__7p)-$|Tbd@NXy$>rsm5BbAG0VS_&(w0=ZlHv-)lSCTp|w)yP(g^;wG5|a zmm-e&ZEoe8{HR*w{WoJ;K>FMS?RXrkwYB>tVU!>=F-)8ala*E(3$>~BP}Am4DMOu) zi5#3MWypaJad|OBTm7kAO^~q!TO@~veIB96y#5bqc3(7z+w2B^G+Xrl0YDQz!r4K^ z#rKL?ehiUL}cGy;)j-%eqhhZ_#UN_LuFE46IO@Sf2XISgr2u&-I13RPr-X| z#WR#evm67x1h6Cc@@>jb%CG8?(>m*19}P5V;LQ}Q9@B9M&a zi_YYW$=AfH4kOQ#@~L_ZvWU1U27t#9;YCoJW!m|bOWXcsGvSg4^7DQHh-bHRAwV_8jek*{^PQqJp{Ey+iq$dOdz5Ht;}+8 zi!=`hQZ;<5qk(Ao9dgjGfwHdvr)4?!Fds^C$S=ik{CaF8S@~A>?IEoOK^~L5r~DEZ zIXsj;IdN#_8EEh_^B@*t6F7XIGP$xP+ez0<9PIt+2X0S~s7Xu0HAK(Daw&V%9T<0e z6SM4SaW~!E@Pa_P_p^*P;oGf;vAg`z)WCP5zl<^F@?B3915+`=iuj>%D2YPTfk+3u1;mEdg2DX(=_9|Um zIqhpOA~wjWTRHE)_jGaf|GC}d;2g=GBZB5mrk@Y^BwLv!FII00y@7N)mWszzeO z9H->roU>e`g^xOsl|AkdaP8FQN4d;AIkZfwk7sgiE%xem0YPH{j;GNlCmDov^sCFxzE?*02#eNub zwER*8y_8=`YU;(CDf*49^g4T8+ke*2vHHnLZ7^2-2m?9f)tkrvXr&BRwC2d8c>RKI zDT1;KYGwBfDG=1>MG39jN)Qd)S{0YrlUVve8A0~3_XIq>%ROgM-c>OSO_!uiQ@gPkn=y6veiPE&m zb4ZZ>dGb24j%G|5K&t5IDTDhL>u(nRJAdWL*dhIlwL~iU3fy$blytI|DpLko0=OXg zJ-+Wcq=Lpm`tg*3{U1fayTe5vyNsxD_wf7bF?wN28u^48-GyWyb-EuS57VG~G#N`{ z?g56ZWAwDUVAeYQe)Z4B%b<+Q%H&40dB^m-+NpuY*g`}j2z{}?#;sj&(;a+Vfn@xx zB^|4z8?iTMKju(c zps@VvPg>DNA)7VRTmu@B$F8uBS=7rNm<2K8b^3;0nda(y5D@pYJeRtr7n5z(VA2&K4;tj zM6?!J^4acU`nqRGN>CTE-4s!1;*94EIfujLGsY95hPOUEEs4wsC;n;m;W}zCVtvLEK@#5?|XWDD5grK(V>*%qy_Y;}kzV6k`*aLb-mwbd9PiZ;Z z$gfOy==U4wrgb&M8!mirS28V4vo|)(C3Ml|mDIlV&hU`N#R=AqO2(GqQ&hp&MlWN} z(=}U9i-@`BL)UCe39s1p8u40c7?X^G;CYKNKWKX$V>Z;(@-}0mKs}%`sBI-54&3Ui( zahMByNk!()Dw41wk!Jlpv+o>;d!T+MNI%^1n6U+U^iu22w{q66V5|^K{-C=H^KfF0 z_Q2)=D+4Cb!#n5al+-eYv}+HSJ_pW1V2DtAa``s|7SOqo5ow8U-YV_ns&q{xCG9qF zx~rT4uml_v-52Ta{T!8jf?L7EJ*)z!09AfmLv*YGm>(E!d+p(JEx8$8VL-b4feYB9|(frtWQbCu*QpT;q?>^n49{5=7fVy6o2Uu+{Yyq%p8jGc5atrC) zF&@6mW#Fvm(%~o4)+W=p_m3aa{X4(=cm0(HOgp#2j98FslzHu_Ggrv$;<^+2J3LBoVTjl4B#1yDinzF~_bZ i-}rZgjyZlm%{!h)>yKNk^`~^j`u#e6G#oqr4*3t5$!GEa diff --git a/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json b/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json index f2bcf17..ff8244c 100644 --- a/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json +++ b/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json @@ -1 +1 @@ -{"version":"daimon.engine-broker-native-build.v1","architecture":"x64","target":"linux/amd64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:36f60689f0a8af0e3108f5f53d78ed52b7d4b6f934c75b6184606dfa82bc741e","binary_sha256":"sha256:36dc76b134eb59cf5a6720b6f94228eb279108e20ea3343fa6efd9ffcb60a4d3","install_path":"/opt/daimon/bin/daimon-engine-broker"} +{"version":"daimon.engine-broker-native-build.v1","architecture":"x64","target":"linux/amd64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:27fcdc8ffb6946e391dad039f695968231d1ae435c6a38dd9bb8c7fb00ed18b9","binary_sha256":"sha256:ff83efc37c77c4ee920080d0eedfca497b504a561c876348af0fec25142a5f9c","install_path":"/opt/daimon/bin/daimon-engine-broker"} From 8d24c50692e4bc0f40adfb486c44023f598312b0 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:40:41 +0200 Subject: [PATCH 04/69] fix: write tool-output spills group-readable so the agent's own Grok worker can read them --- src/runtime/toolResultSpill.test.ts | 18 +++++++++++++++++- src/runtime/toolResultSpill.ts | 18 ++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/runtime/toolResultSpill.test.ts b/src/runtime/toolResultSpill.test.ts index b83a587..a5f6c82 100644 --- a/src/runtime/toolResultSpill.test.ts +++ b/src/runtime/toolResultSpill.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { chmod, chown, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import test from "node:test"; @@ -150,3 +150,19 @@ test("the bound and the exemption list come from the environment, and a nonsense assert.deepEqual([...resolveExemptToolNames({})], []); assert.deepEqual([...resolveExemptToolNames({ [TOOL_RESULT_EXEMPT_ENV]: " mcp_a_b , mcp_c_d ," })], ["mcp_a_b", "mcp_c_d"]); }); + +test("spilled files are readable by the directory's (worker) group and never by other users", async () => { + await withDirectory(async (directory) => { + // What a deployment provisions for a brokered worker: setgid tool-output in the worker's group. + const groups = (process.getgroups?.() ?? []).filter((gid) => gid !== process.getgid?.()); + const workerGroup = groups[0]; + if (workerGroup !== undefined) await chown(directory, process.getuid?.() ?? -1, workerGroup).catch(() => undefined); + await chmod(directory, 0o2750); + const previous = process.umask(0o077); + let capped; + try { capped = await cap({ content: [{ type: "text", text: "x".repeat(200_000) }] }, { spillDirectory: directory }); } finally { process.umask(previous); } + const file = await stat(capped.spillPath!); + assert.equal(file.mode & 0o777, 0o640, "group-readable even under a restrictive umask, never other-readable"); + assert.equal(file.gid, (await stat(directory)).gid, "the file carries the directory's group"); + }); +}); diff --git a/src/runtime/toolResultSpill.ts b/src/runtime/toolResultSpill.ts index 6e993a2..b0a8d55 100644 --- a/src/runtime/toolResultSpill.ts +++ b/src/runtime/toolResultSpill.ts @@ -181,6 +181,19 @@ const notice = (input: Readonly<{ + ` or \`grep -n "" ${input.spillPath}\` — and do NOT repeat this tool call to see it: an identical call returns this same truncation.]`; }; +/** + * Spilled files are group-readable and never other-readable. + * + * A brokered Grok worker runs as its own uid and reads a spill with + * `read_file`, so a 0600 file owned by the runtime (uid 2000) was unreadable to + * the very agent the notice sends there. The group grant reaches exactly that + * agent's worker only when the deployment provisions `tool-output` as + * `: 2750` (setgid, so each file inherits the worker + * group; `GROK_ENGINE_BROKER.worker.home.spillDirectory`). A directory Daimon + * creates itself stays 0700, so for every other engine nothing new is exposed. + */ +export const SPILL_FILE_MODE = 0o640; + /** * Write the full payload where the agent can read it, atomically. * @@ -192,8 +205,9 @@ const writeSpill = async (directory: string, name: string, text: string): Promis await mkdir(directory, { recursive: true, mode: 0o700 }); const file = path.join(directory, name); const temporary = `${file}.${process.pid}.${Date.now()}.tmp`; - const handle = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); - try { await handle.writeFile(text, "utf8"); await handle.sync(); } finally { await handle.close(); } + const handle = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, SPILL_FILE_MODE); + // Explicit, so a restrictive umask cannot strip the group read an agent's sandboxed worker needs. + try { await handle.chmod(SPILL_FILE_MODE); await handle.writeFile(text, "utf8"); await handle.sync(); } finally { await handle.close(); } try { await rename(temporary, file); } catch (error) { await unlink(temporary).catch(() => undefined); throw error; } return file; }; From 45d9e411425930e04e9672b47da57d824d730d14 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:42:47 +0200 Subject: [PATCH 05/69] docs: document the Grok worker temp and spill provisioning contract --- src/runtime/AGENTS.md | 19 +++++++++++++++++++ src/runtime/native/AGENTS.md | 3 +++ 2 files changed, 22 insertions(+) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 1628dc5..a0d4517 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -160,6 +160,25 @@ every profile inside bubblewrap, where a non-empty `deny` list is enforced; `grokWorkerSandboxProfile.ts` renders those profile bytes. A worker-uid process can neither write, rename, nor unlink any of the root-owned files. +Temp and spill isolation (`grokWorkerTmpAttestation.ts`, checked before every +turn; `GROK_ENGINE_BROKER.worker.home.{privateTmp,sharedTmp,spillDirectory}`). +Grok 1.0.34's strict profile grants shared `/tmp` and `/var/tmp` read-write +and refuses to start if either, or any path equal to or above a base grant, is +in `deny` (verified: `/tmp`, `/var/tmp`, `/run`, `/etc`, `sessions` all fail; +`/tmp/sub` works), so the profile cannot hide evaluator temp files. Instead: +- the launcher exports `TMPDIR=/tmp` (strict adds TMPDIR to its + read-write grants; Python, Node and `mktemp` use it); provision it + `: 0700`; +- `/tmp` and `/var/tmp` must be `root: 1774`: + Grok needs to open the directory, but without search or write a worker can + only list names — `cat`/`read_file` get EACCES and it cannot create files. + `1770`/`1771` make Grok refuse the profile; `1775`/`1777` leak. Any non-root + process outside that group that needs temp space must get its own `TMPDIR`; +- spills (`toolResultSpill.ts`) are written `0640`; provision + `/tool-output` as `2000: 2750` (setgid) under a + runtime home the worker can traverse, so each spill carries that agent's + worker group and no other worker can read it. + `agySubscriptionRealm.ts` owns the one host-level private D-Bus/Secret Service realm, durable keyring lease, bounded unlock stdin, and cleanup. `agySubscriptionBootstrap.ts` owns only the interactive first-enrollment AGY diff --git a/src/runtime/native/AGENTS.md b/src/runtime/native/AGENTS.md index 3e7f8ae..8fd8f77 100644 --- a/src/runtime/native/AGENTS.md +++ b/src/runtime/native/AGENTS.md @@ -25,6 +25,9 @@ environment: `DAIMON_MCP_CAPABILITY` and `DAIMON_PROVIDER_CAPABILITY` (Grok config reads the proxy capability through `env_key`). `--auth-provider` mode remains for callers of the older contract. +It also exports `TMPDIR=/tmp`, the worker's private temp +directory, derived only from the root-owned registration. + Received descriptors carry `MSG_CMSG_CLOEXEC` and can already occupy fds 3-5, so `launch()` lifts prompt, capability, output, executable and status fds above 16 before `dup2`-ing them into place; a `dup2` onto itself keeps close-on-exec From 7439edbbaa6a8e7063d6e6bfe2fec90184d0f288 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 07:56:47 +0200 Subject: [PATCH 06/69] fix: attest every registered Grok worker's private temp and close the temp attestation test gaps --- src/runtime/grokEngineBroker.ts | 4 +- src/runtime/grokWorkerAttestation.ts | 12 +- .../grokWorkerAttestationChecks.test.ts | 26 ++++- src/runtime/grokWorkerTmpAttestation.test.ts | 106 ++++++++++++------ src/runtime/grokWorkerTmpAttestation.ts | 42 +++++-- 5 files changed, 136 insertions(+), 54 deletions(-) diff --git a/src/runtime/grokEngineBroker.ts b/src/runtime/grokEngineBroker.ts index aceb394..35fd7cc 100644 --- a/src/runtime/grokEngineBroker.ts +++ b/src/runtime/grokEngineBroker.ts @@ -9,7 +9,7 @@ import { acquireGrokBrokerRealmLease } from "./grokBrokerRealmLease.js"; import { grokBrokerWorkerConfigSha256 } from "./grokBrokerWorkerConfig.js"; import { parseGrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; import { runGrokEngineBrokerTurn, type GrokEngineBrokerTurnResult } from "./grokEngineBrokerTurn.js"; -import { createGrokWorkerIsolationGuard,prepareGrokWorkerAttestation } from "./grokWorkerAttestation.js"; +import { createGrokWorkerIsolationGuard,grokBrokerAttestationInput,prepareGrokWorkerAttestation } from "./grokWorkerAttestation.js"; import { createLedgeredGrokInferenceGrants, GrokInferenceGrantRefused, type GrokInferenceGrantRequest } from "./grokInferenceGrants.js"; export { EngineBrokerTurnFailure, type GrokEngineBrokerTurnResult } from "./grokEngineBrokerTurn.js"; @@ -30,7 +30,7 @@ export type GrokEngineBroker = Awaited> */ export async function startGrokEngineBroker(options: Readonly<{ grokCommand: string; nativeClient: string; credentialHome: string; turnStore: string; registrations: readonly GrokEngineBrokerRegistration[]; inferenceLedgerPath?: string }>) { const registrations = new Map(options.registrations.map((entry) => [entry.agentId, { ...entry, model: parseGrokBrokerModelPolicy(entry.model) }])); if (registrations.size !== options.registrations.length) throw new Error("engine broker registration conflict"); - const attestationFor = (registration: GrokEngineBrokerRegistration) => ({ ...registration, brokerGid: 2100, configSha256: grokBrokerWorkerConfigSha256(registration.model) }); + const attestationFor = (registration: GrokEngineBrokerRegistration) => grokBrokerAttestationInput(registration, [...registrations.values()], grokBrokerWorkerConfigSha256(registration.model)); const inferenceLedgerPath=options.inferenceLedgerPath;const grants=inferenceLedgerPath===undefined?undefined:createLedgeredGrokInferenceGrants(inferenceLedgerPath); const lease=await acquireGrokBrokerRealmLease(options.credentialHome);const authority = new DurableGrokBrokerCredentialAuthority(options.grokCommand, options.credentialHome);try{await authority.initialize();}catch(error){await lease.close();throw error;} let proxy:Awaited>;try{proxy=await startGrokBrokerProxy(authority,undefined,undefined,undefined,grants);}catch(error){await lease.close();throw error;}let mcp:Awaited>|undefined;try{mcp=await startEngineBrokerMcpFacade();for(const registration of registrations.values())await prepareGrokWorkerAttestation(attestationFor(registration));}catch(error){if(mcp)await mcp.close().catch(()=>undefined);await proxy.close();await lease.close();throw error;}if(!mcp)throw new Error("engine broker unavailable");const turns = new EngineBrokerTurnRegistry(options.turnStore); const active = new Map }>(); let closed = false; const facade = mcp; diff --git a/src/runtime/grokWorkerAttestation.ts b/src/runtime/grokWorkerAttestation.ts index d10d01f..60bc8fc 100644 --- a/src/runtime/grokWorkerAttestation.ts +++ b/src/runtime/grokWorkerAttestation.ts @@ -4,7 +4,7 @@ import { lstat,open } from "node:fs/promises"; import path from "node:path"; import { verifyGrokWorkerHome } from "./grokWorkerHomeAttestation.js"; -import { verifyGrokWorkerTmp } from "./grokWorkerTmpAttestation.js"; +import { grokWorkerHomeForProfile, verifyGrokWorkerTmp, type GrokWorkerTmpOptions, type GrokWorkerTmpWorker } from "./grokWorkerTmpAttestation.js"; import { grokWorkerEventsPathFor } from "./grokWorkerSandboxProfile.js"; /** @@ -62,15 +62,19 @@ export function parseGrokWorkerSandboxProfile(bytes:Uint8Array,profileSha256:str * 1.0.34), and a root-owned read-only worker home whose `config.toml` hashes to * the declared renderer output (`grokWorkerHomeAttestation.ts`). */ -export async function prepareGrokWorkerAttestation(input:Readonly<{profilePath:string;eventsPath:string;profileSha256:string;workerUid:number;brokerGid:number;configSha256:string}>,profileOwner:Readonly<{uid:number;gid:number;sharedTmpRoots?:readonly string[]}>={uid:0,gid:0}):Promise{ +export async function prepareGrokWorkerAttestation(input:Readonly<{profilePath:string;eventsPath:string;profileSha256:string;workerUid:number;brokerGid:number;configSha256:string;registeredWorkers?:readonly Readonly<{profilePath:string;workerUid:number}>[]}>,profileOwner:Readonly<{uid:number;gid:number;tmp?:GrokWorkerTmpOptions}>={uid:0,gid:0}):Promise{ if(input.eventsPath!==grokWorkerEventsPathFor(input.profilePath))throw new Error("Grok worker sandbox events must be read from $GROK_HOME/sessions/sandbox-events.jsonl"); const profile=await secureOpen(input.profilePath,profileOwner.uid,profileOwner.gid,0o444,65_536);let bytes:Buffer|undefined;let denyPaths:readonly string[]=[];try{bytes=await profile.readFile();denyPaths=parseGrokWorkerSandboxProfile(bytes,input.profileSha256);}catch{throw new Error("Grok worker isolation attestation unavailable");}finally{bytes?.fill(0);await profile.close();} // The launcher exports TMPDIR=/tmp; the profile lives at /.grok/sandbox.toml. - if(path.basename(path.dirname(input.profilePath))!==".grok")throw new Error("Grok worker isolation attestation unavailable"); - await verifyGrokWorkerTmp(path.dirname(path.dirname(input.profilePath)),input.workerUid,profileOwner.sharedTmpRoots); + // Every registered worker's private temp is attested, not only this one's (a sibling's open temp is a shared channel). + const workers=[{profilePath:input.profilePath,workerUid:input.workerUid},...(input.registeredWorkers??[])].map((worker)=>({home:grokWorkerHomeForProfile(worker.profilePath),uid:worker.workerUid})); + if(workers.some((worker)=>worker.home===undefined))throw new Error("Grok worker isolation attestation unavailable"); + await verifyGrokWorkerTmp(workers as readonly GrokWorkerTmpWorker[],profileOwner.tmp); await verifyGrokWorkerHome(path.dirname(input.profilePath),input.configSha256); const events=await secureOpen(input.eventsPath,input.workerUid,input.brokerGid,0o640,16*1024*1024);try{const stat=await events.stat();return{dev:Number(stat.dev),ino:Number(stat.ino),size:Number(stat.size),denyPaths};}finally{await events.close();} } +/** The per-turn attestation input for one registration, carrying every registered worker so sibling temp is attested too. */ +export const grokBrokerAttestationInput=>(registration:T,registrations:readonly Readonly<{profilePath:string;workerUid:number}>[],configSha256:string)=>({...registration,brokerGid:2100,configSha256,registeredWorkers:registrations.map((entry)=>({profilePath:entry.profilePath,workerUid:entry.workerUid}))}); /** * The accepted `ProfileApplied` line of one turn, as an absolute byte range of * the events file plus its digest. diff --git a/src/runtime/grokWorkerAttestationChecks.test.ts b/src/runtime/grokWorkerAttestationChecks.test.ts index b5568a6..8bee20e 100644 --- a/src/runtime/grokWorkerAttestationChecks.test.ts +++ b/src/runtime/grokWorkerAttestationChecks.test.ts @@ -7,6 +7,7 @@ import test, { mock } from "node:test"; import { GrokWorkerAttestationFailure, + grokBrokerAttestationInput, prepareGrokWorkerAttestation, verifyGrokWorkerAttestation, type GrokWorkerAttestationSnapshot @@ -75,7 +76,7 @@ test("prepare refuses a worker home that fails attestation even when profile, te const events = path.join(home, "sessions", "sandbox-events.jsonl"); await writeFile(events, ""); await chmod(events, 0o640); const input = { profilePath: profile, eventsPath: events, profileSha256: createHash("sha256").update(text).digest("hex"), workerUid: self.uid, brokerGid: self.gid, configSha256: "0".repeat(64) }; - await assert.rejects(prepareGrokWorkerAttestation(input, { uid: self.uid, gid: Number((await stat(profile)).gid), sharedTmpRoots: [] }), (error: Error) => error.message === "Grok worker isolation attestation unavailable"); + await assert.rejects(prepareGrokWorkerAttestation(input, { uid: self.uid, gid: Number((await stat(profile)).gid), tmp: { sharedRoots: [root], sharedOwnerUid: self.uid, firstWorkerUid: self.uid } }), (error: Error) => error.message === "Grok worker isolation attestation unavailable"); }); test("prepare refuses a worker without a private temp directory before the home check", async (t) => { @@ -98,3 +99,26 @@ test("prepare refuses a worker without a private temp directory before the home await writeFile(stray.profilePath, text); await chmod(stray.profilePath, 0o444); await writeFile(stray.eventsPath, ""); await chmod(stray.eventsPath, 0o640); await assert.rejects(prepareGrokWorkerAttestation(stray, owner), (error: Error) => /attestation unavailable/u.test(error.message) && !/temp/u.test(error.message)); }); + +test("prepare refuses the current turn when a sibling registered worker's private temp is 0777", async (t) => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-guard-sibling-")); + t.after(() => rm(root, { recursive: true, force: true })); + const make = async (name: string) => { + const home = path.join(root, name), grok = path.join(home, ".grok"); + await mkdir(path.join(grok, "sessions"), { recursive: true }); await mkdir(path.join(home, "tmp"), { mode: 0o700 }); + return { home, profile: path.join(grok, "sandbox.toml"), events: path.join(grok, "sessions", "sandbox-events.jsonl") }; + }; + const own = await make("own"), sibling = await make("sibling"); + const text = '[profiles.daimon-strict]\nextends = "strict"\nrestrict_network = true\ndeny = []\n'; + await writeFile(own.profile, text); await chmod(own.profile, 0o444); + await writeFile(own.events, ""); await chmod(own.events, 0o640); + const registration = { profilePath: own.profile, eventsPath: own.events, profileSha256: createHash("sha256").update(text).digest("hex"), workerUid: self.uid, workspace: "/w" }; + const input = grokBrokerAttestationInput(registration, [registration, { profilePath: sibling.profile, workerUid: self.uid }], "0".repeat(64)); + assert.deepEqual(input.registeredWorkers.map((entry) => entry.profilePath), [own.profile, sibling.profile]); + const seams = { uid: self.uid, gid: Number((await stat(own.profile)).gid), tmp: { sharedRoots: [root], sharedOwnerUid: self.uid, firstWorkerUid: self.uid } }; + await chmod(root, 0o700); + // Sibling well provisioned: temp passes and the (root-only) home leg is what refuses. + await assert.rejects(prepareGrokWorkerAttestation({ ...input, brokerGid: self.gid }, seams), (error: Error) => error.message === "Grok worker isolation attestation unavailable"); + await chmod(path.join(sibling.home, "tmp"), 0o777); + await assert.rejects(prepareGrokWorkerAttestation({ ...input, brokerGid: self.gid }, seams), /temp isolation attestation unavailable/u); +}); diff --git a/src/runtime/grokWorkerTmpAttestation.test.ts b/src/runtime/grokWorkerTmpAttestation.test.ts index e05016a..a5a728c 100644 --- a/src/runtime/grokWorkerTmpAttestation.test.ts +++ b/src/runtime/grokWorkerTmpAttestation.test.ts @@ -1,53 +1,87 @@ import assert from "node:assert/strict"; -import { chmod, mkdir, mkdtemp, rm, symlink } from "node:fs/promises"; +import { chmod, mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import test from "node:test"; -import { assertGrokWorkerTmpEntries, verifyGrokWorkerTmp } from "./grokWorkerTmpAttestation.js"; +import { assertGrokWorkerTmpEntries, verifyGrokWorkerTmp, type GrokWorkerTmpOptions } from "./grokWorkerTmpAttestation.js"; -const worker = 2200; -const dir = (mode: number, uid: number, gid: number, kind: "dir" | "link" | "file" = "dir") => ({ - uid, gid, mode: (kind === "dir" ? 0o040000 : kind === "link" ? 0o120000 : 0o100000) | mode, - isDirectory: () => kind === "dir", isSymbolicLink: () => kind === "link" +const worker = 2200, sibling = 2201; +type Entry = Readonly<{ uid: number; gid: number; mode: number; isDirectory(): boolean }>; +const dir = (mode: number, uid: number, gid: number): Entry => ({ uid, gid, mode: 0o040000 | mode, isDirectory: () => true }); +const file = (mode: number, uid: number, gid: number): Entry => ({ uid, gid, mode: 0o100000 | mode, isDirectory: () => false }); +type Entries = { privateTmps: { uid: number; entry: Entry | undefined }[]; shared: (Entry | undefined)[] }; +const good = (): Entries => ({ privateTmps: [{ uid: worker, entry: dir(0o700, worker, worker) }, { uid: sibling, entry: dir(0o700, sibling, sibling) }], shared: [dir(0o1774, 0, 2000), dir(0o1774, 0, 2000)] }); +const withShared = (shared: Entry | undefined): Entries => ({ ...good(), shared: [shared, dir(0o1774, 0, 2000)] }); +const withOwn = (entry: Entry | undefined, uid = worker): Entries => ({ ...good(), privateTmps: [{ uid, entry }, good().privateTmps[1]!] }); +const withSibling = (entry: Entry | undefined): Entries => ({ ...good(), privateTmps: [good().privateTmps[0]!, { uid: sibling, entry }] }); +const refused = /temp isolation attestation unavailable/u; + +test("accepts private worker temps and shared temp roots the workers cannot open or write", () => { + assert.doesNotThrow(() => assertGrokWorkerTmpEntries(good())); + // Per contract the shared group only has to be below the worker range: the broker group 2100 is fine. + assert.doesNotThrow(() => assertGrokWorkerTmpEntries(withShared(dir(0o1774, 0, 2100)))); + assert.doesNotThrow(() => assertGrokWorkerTmpEntries(withShared(dir(0o1770, 0, 2000)))); }); -type Entry = ReturnType; -const good = (): { privateTmp: Entry | undefined; shared: (Entry | undefined)[] } => ({ privateTmp: dir(0o700, worker, worker), shared: [dir(0o1774, 0, 2000), dir(0o1774, 0, 2000)] }); -test("accepts private worker temp and shared temp roots the worker cannot open or write", () => { - assert.doesNotThrow(() => assertGrokWorkerTmpEntries(good(), worker)); - assert.doesNotThrow(() => assertGrokWorkerTmpEntries({ ...good(), shared: [dir(0o1770, 0, 2000), dir(0o700, 0, 0)] }, worker)); +test("refuses shared temp roots that are missing, not directories, not root-owned, worker-grouped, or open to others", () => { + const cases: Record = { + "missing": withShared(undefined), + "regular file": withShared(file(0o1774, 0, 2000)), + "owned by the org user": withShared(dir(0o1774, 2000, 2000)), + "group 2200 (a worker group)": withShared(dir(0o1774, 0, 2200)), + "other search (1775)": withShared(dir(0o1775, 0, 2000)), + "other write (1776)": withShared(dir(0o1776, 0, 2000)), + "default /tmp (1777)": withShared(dir(0o1777, 0, 0)), + "no shared roots at all": { ...good(), shared: [] } + }; + for (const [label, entries] of Object.entries(cases)) assert.throws(() => assertGrokWorkerTmpEntries(entries), refused, label); }); -test("refuses shared /tmp or /var/tmp a worker could traverse, write, or own through its group", () => { - const refusals: Record = { - "shared 1777 (default /tmp)": { ...good(), shared: [dir(0o1777, 0, 0), dir(0o1774, 0, 2000)] }, - "shared other search": { ...good(), shared: [dir(0o1774, 0, 2000), dir(0o1775, 0, 2000)] }, - "shared other write": { ...good(), shared: [dir(0o1776, 0, 2000), dir(0o1774, 0, 2000)] }, - "shared owned by a worker group": { ...good(), shared: [dir(0o1774, 0, worker), dir(0o1774, 0, 2000)] }, - "shared owned by the org user": { ...good(), shared: [dir(0o1774, 2000, 2000), dir(0o1774, 0, 2000)] }, - "shared missing": { ...good(), shared: [undefined, dir(0o1774, 0, 2000)] }, - "shared symlink": { ...good(), shared: [dir(0o777, 0, 0, "link"), dir(0o1774, 0, 2000)] }, - "private missing": { ...good(), privateTmp: undefined }, - "private owned by another worker": { ...good(), privateTmp: dir(0o700, worker + 1, worker + 1) }, - "private group readable": { ...good(), privateTmp: dir(0o750, worker, worker) }, - "private symlink": { ...good(), privateTmp: dir(0o700, worker, worker, "link") }, - "private is a file": { ...good(), privateTmp: dir(0o600, worker, worker, "file") } +test("refuses a private temp that is missing, not a directory, owned by someone else, or has any group or other bit", () => { + const cases: Record = { + "missing": withOwn(undefined), + "regular file": withOwn(file(0o600, worker, worker)), + "owned by another worker": withOwn(dir(0o700, sibling, sibling)), + "group read (0740)": withOwn(dir(0o740, worker, worker)), + "other execute (0701)": withOwn(dir(0o701, worker, worker)), + "other write (0702)": withOwn(dir(0o702, worker, worker)), + "other read (0704)": withOwn(dir(0o704, worker, worker)), + "worker uid below the worker range": withOwn(dir(0o700, 2100, 2100), 2100), + "no workers at all": { ...good(), privateTmps: [] } }; - for (const [label, entries] of Object.entries(refusals)) assert.throws(() => assertGrokWorkerTmpEntries(entries, worker), /temp isolation attestation unavailable/u, label); + for (const [label, entries] of Object.entries(cases)) assert.throws(() => assertGrokWorkerTmpEntries(entries), refused, label); +}); + +test("a misprovisioned sibling worker's temp refuses the current worker's turn", () => { + assert.throws(() => assertGrokWorkerTmpEntries(withSibling(dir(0o777, sibling, sibling))), refused); + assert.throws(() => assertGrokWorkerTmpEntries(withSibling(dir(0o770, sibling, sibling))), refused); + assert.throws(() => assertGrokWorkerTmpEntries(withSibling(undefined)), refused); }); -test("checks the real private temp directory under the worker home and the given shared roots", async (t) => { +test("on a real filesystem: sibling 0777 temp, symlinked temps and symlinked shared roots are refused", async (t) => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-grok-tmp-")); t.after(() => rm(root, { recursive: true, force: true })); const uid = process.getuid?.() ?? 0; - const home = path.join(root, "home"), shared = path.join(root, "shared"); - await mkdir(path.join(home, "tmp"), { recursive: true }); await mkdir(shared); - await chmod(path.join(home, "tmp"), 0o700); - // A shared root owned by the test user is refused (not root-owned) — as is a symlinked private temp. - await assert.rejects(verifyGrokWorkerTmp(home, uid, [shared]), /temp isolation attestation unavailable/u); - await rm(path.join(home, "tmp"), { recursive: true }); await symlink(shared, path.join(home, "tmp")); - await assert.rejects(verifyGrokWorkerTmp(home, uid, []), /temp isolation attestation unavailable/u); - await rm(path.join(home, "tmp")); await mkdir(path.join(home, "tmp"), { mode: 0o700 }); - await verifyGrokWorkerTmp(home, uid, []); + const own = path.join(root, "own"), other = path.join(root, "other"), shared = path.join(root, "shared"), elsewhere = path.join(root, "elsewhere"); + for (const directory of [path.join(own, "tmp"), path.join(other, "tmp"), shared, elsewhere]) { await mkdir(directory, { recursive: true }); await chmod(directory, 0o700); } + // Seams: this test runs unprivileged, so the owner and worker-range floor are the test user. + const options: GrokWorkerTmpOptions = { sharedRoots: [shared], sharedOwnerUid: uid, firstWorkerUid: uid }; + const workers = [{ home: own, uid }, { home: other, uid }]; + await verifyGrokWorkerTmp(workers, options); + + await chmod(path.join(other, "tmp"), 0o777); + await assert.rejects(verifyGrokWorkerTmp(workers, options), refused, "sibling 0777"); + await chmod(path.join(other, "tmp"), 0o700); + + await rm(path.join(own, "tmp"), { recursive: true }); await symlink(elsewhere, path.join(own, "tmp")); + await assert.rejects(verifyGrokWorkerTmp(workers, options), refused, "private temp symlink to a valid directory"); + await rm(path.join(own, "tmp")); await mkdir(path.join(own, "tmp"), { mode: 0o700 }); + + const linkedShared = path.join(root, "linked-shared"); await symlink(elsewhere, linkedShared); + await assert.rejects(verifyGrokWorkerTmp(workers, { ...options, sharedRoots: [linkedShared] }), refused, "shared root symlink to a valid directory"); + const regular = path.join(root, "regular"); await writeFile(regular, ""); await chmod(regular, 0o600); + await assert.rejects(verifyGrokWorkerTmp(workers, { ...options, sharedRoots: [regular] }), refused, "shared root regular file"); + await assert.rejects(verifyGrokWorkerTmp(workers, { ...options, sharedRoots: [] }), refused, "empty shared roots"); + await verifyGrokWorkerTmp(workers, options); }); diff --git a/src/runtime/grokWorkerTmpAttestation.ts b/src/runtime/grokWorkerTmpAttestation.ts index 4d2cf7b..863a9ab 100644 --- a/src/runtime/grokWorkerTmpAttestation.ts +++ b/src/runtime/grokWorkerTmpAttestation.ts @@ -4,11 +4,19 @@ import path from "node:path"; import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; -type Entry = Pick & Readonly<{ isDirectory(): boolean; isSymbolicLink(): boolean }>; +type Entry = Pick & Readonly<{ isDirectory(): boolean }>; const HOME = GROK_ENGINE_BROKER.worker.home; +const FIRST_WORKER_UID = GROK_ENGINE_BROKER.identities.firstWorkerUid; + +export type GrokWorkerTmpWorker = Readonly<{ home: string; uid: number }>; +/** Test seams only; production uses the manifest defaults. */ +export type GrokWorkerTmpOptions = Readonly<{ sharedRoots?: readonly string[]; sharedOwnerUid?: number; firstWorkerUid?: number }>; /** - * Temp-directory isolation for one worker, checked before every turn. + * Temp-directory isolation, checked before every turn for *every* registered + * worker, not only the one about to run: a misprovisioned sibling temp + * directory (group- or world-writable) would be a place this worker could + * write into and that sibling would read from. * * Grok 1.0.34's strict profile grants shared `/tmp` and `/var/tmp` * read-write, and it refuses to start when either (or any ancestor of a @@ -24,25 +32,37 @@ const HOME = GROK_ENGINE_BROKER.worker.home; * deployment that lets workers traverse or write them is refused; * - the worker's own `/tmp` (the launcher's compiled `TMPDIR`, which * strict grants read-write) is a real directory owned by the worker with no - * group or other access. + * group or other access, and every registered worker's is checked. + * + * + * Entries come from `lstat`, so a symlink is never a directory here: a + * symlinked temp root or private temp is refused by the directory check. * * Pure so every refusal is testable without root. */ -export function assertGrokWorkerTmpEntries(entries: Readonly<{ privateTmp: Entry | undefined; shared: readonly (Entry | undefined)[] }>, workerUid: number): void { +export function assertGrokWorkerTmpEntries(entries: Readonly<{ privateTmps: readonly Readonly<{ uid: number; entry: Entry | undefined }>[]; shared: readonly (Entry | undefined)[] }>, options: GrokWorkerTmpOptions = {}): void { const shared = HOME.sharedTmp; + const firstWorkerUid = options.firstWorkerUid ?? FIRST_WORKER_UID; + if (entries.shared.length === 0 || entries.privateTmps.length === 0) throw unavailable(); for (const entry of entries.shared) { - if (entry === undefined || !entry.isDirectory() || entry.isSymbolicLink() || entry.uid !== shared.uid || entry.gid >= shared.maxGroupExclusive || (Number(entry.mode) & 0o007 & ~shared.otherMode) !== 0) throw unavailable(); + if (entry === undefined || !entry.isDirectory() || entry.uid !== (options.sharedOwnerUid ?? shared.uid) || entry.gid >= shared.maxGroupExclusive || (Number(entry.mode) & 0o007 & ~shared.otherMode) !== 0) throw unavailable(); + } + for (const { uid, entry } of entries.privateTmps) { + if (!Number.isSafeInteger(uid) || uid < firstWorkerUid || entry === undefined || !entry.isDirectory() || entry.uid !== uid || (Number(entry.mode) & 0o077) !== 0) throw unavailable(); } - const own = entries.privateTmp; - if (own === undefined || !own.isDirectory() || own.isSymbolicLink() || own.uid !== workerUid || (Number(own.mode) & 0o077) !== 0) throw unavailable(); } -export async function verifyGrokWorkerTmp(workerHome: string, workerUid: number, sharedRoots: readonly string[] = HOME.sharedTmp.paths): Promise { +/** `workers` must list every registered worker (the running one included). */ +export async function verifyGrokWorkerTmp(workers: readonly GrokWorkerTmpWorker[], options: GrokWorkerTmpOptions = {}): Promise { const inspect = async (file: string): Promise => { try { return await lstat(file); } catch { return undefined; } }; assertGrokWorkerTmpEntries({ - privateTmp: await inspect(path.join(workerHome, HOME.privateTmp.relativeToWorkerHome)), - shared: await Promise.all(sharedRoots.map(inspect)) - }, workerUid); + privateTmps: await Promise.all(workers.map(async (worker) => ({ uid: worker.uid, entry: await inspect(path.join(worker.home, HOME.privateTmp.relativeToWorkerHome)) }))), + shared: await Promise.all((options.sharedRoots ?? HOME.sharedTmp.paths).map(inspect)) + }, options); } +/** A registration's worker home is the parent of its `/.grok/sandbox.toml`. */ +export const grokWorkerHomeForProfile = (profilePath: string): string | undefined => + path.basename(path.dirname(profilePath)) === ".grok" ? path.dirname(path.dirname(profilePath)) : undefined; + const unavailable = (): Error => new Error("Grok worker temp isolation attestation unavailable"); From a2ba04bcdcccdcc6f1547b174182a5d3024487f2 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 08:00:14 +0200 Subject: [PATCH 07/69] fix: pin the tool-output spill directory and publish spills by rename into the verified directory --- src/runtime/toolResultSpill.test.ts | 89 ++++++++++++++++++++++++++--- src/runtime/toolResultSpill.ts | 64 +++++++++++++++++++-- 2 files changed, 140 insertions(+), 13 deletions(-) diff --git a/src/runtime/toolResultSpill.test.ts b/src/runtime/toolResultSpill.test.ts index a5f6c82..a4b94c6 100644 --- a/src/runtime/toolResultSpill.test.ts +++ b/src/runtime/toolResultSpill.test.ts @@ -1,11 +1,12 @@ import assert from "node:assert/strict"; -import { chmod, chown, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; +import { chmod, chown, lstat, mkdir, mkdtemp, open, readFile, readdir, rm, stat, symlink, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import test from "node:test"; +import test, { mock } from "node:test"; import { renderMcpToolResult, MCP_TOOL_RESULT_MAX_BYTES, type McpUpstreamResult } from "./mcpToolResult.js"; import { + assertSpillDirectoryStat, capToolResult, DEFAULT_TOOL_RESULT_MAX_BYTES, MIN_TOOL_RESULT_MAX_BYTES, @@ -153,16 +154,88 @@ test("the bound and the exemption list come from the environment, and a nonsense test("spilled files are readable by the directory's (worker) group and never by other users", async () => { await withDirectory(async (directory) => { - // What a deployment provisions for a brokered worker: setgid tool-output in the worker's group. - const groups = (process.getgroups?.() ?? []).filter((gid) => gid !== process.getgid?.()); - const workerGroup = groups[0]; - if (workerGroup !== undefined) await chown(directory, process.getuid?.() ?? -1, workerGroup).catch(() => undefined); - await chmod(directory, 0o2750); + // What a deployment provisions for a brokered worker: setgid tool-output in a group that is not the runtime's own. + const workerGroup = (process.getgroups?.() ?? []).find((gid) => gid !== process.getgid?.()); + if (workerGroup !== undefined) { await chown(directory, process.getuid?.() ?? -1, workerGroup); await chmod(directory, 0o2750); } else await chmod(directory, 0o700); const previous = process.umask(0o077); let capped; try { capped = await cap({ content: [{ type: "text", text: "x".repeat(200_000) }] }, { spillDirectory: directory }); } finally { process.umask(previous); } - const file = await stat(capped.spillPath!); + assert.ok(capped.spillPath, "the spill was written"); + const file = await stat(capped.spillPath); assert.equal(file.mode & 0o777, 0o640, "group-readable even under a restrictive umask, never other-readable"); assert.equal(file.gid, (await stat(directory)).gid, "the file carries the directory's group"); }); }); + +const big = { content: [{ type: "text" as const, text: `HEAD${"y".repeat(100_000)}TAIL` }] }; +const spillName = "daimon-abc123.mcp_desk_archive_dump.log"; + +test("a symlinked, world-open, or group-open-without-setgid spill directory is refused and nothing is written", async () => { + await withDirectory(async (root) => { + const target = path.join(root, "target"); await mkdir(target, { mode: 0o700 }); + const linked = path.join(root, "linked"); await symlink(target, linked); + const capped = await cap(big, { spillDirectory: linked }); + assert.equal(capped.spillPath, undefined); + assert.equal(capped.details.full_output_saved, false); + assert.deepEqual(await readdir(target), [], "nothing written through the symlink"); + const workerGroup = (process.getgroups?.() ?? []).find((gid) => gid !== process.getgid?.()); + if (workerGroup !== undefined) { + // A foreign group without setgid: files would not inherit it, so it is refused. + const noSetgid = path.join(root, "foreign-group-no-setgid"); await mkdir(noSetgid); await chown(noSetgid, process.getuid?.() ?? -1, workerGroup); await chmod(noSetgid, 0o750); + assert.equal((await cap(big, { spillDirectory: noSetgid })).spillPath, undefined, "0750 foreign group without setgid"); + } + // 2750 in the runtime's own group is not a worker grant; 0701/0704 exceed 2750. + for (const mode of [0o777, 0o755, 0o750, 0o2770, 0o2757, 0o2750, 0o701, 0o704]) { + const directory = path.join(root, `mode-${mode.toString(8)}`); await mkdir(directory); await chmod(directory, mode); + const refused = await cap(big, { spillDirectory: directory }); + assert.equal(refused.spillPath, undefined, mode.toString(8)); + assert.deepEqual((await readdir(directory)).filter((name) => !name.startsWith(".")), [], mode.toString(8)); + } + }); +}); + +test("a destination symlink or a pre-existing 0666 file is replaced by a 0640 regular file without touching the target", async () => { + await withDirectory(async (root) => { + const directory = path.join(root, "tool-output"); await mkdir(directory, { mode: 0o700 }); + const victim = path.join(root, "victim.txt"); await writeFile(victim, "VICTIM", { mode: 0o644 }); + await symlink(victim, path.join(directory, spillName)); + const first = await cap(big, { spillDirectory: directory }); + assert.equal(first.spillPath, path.join(directory, spillName)); + const replaced = await lstat(first.spillPath!); + assert.ok(replaced.isFile() && !replaced.isSymbolicLink()); + assert.equal(replaced.mode & 0o777, 0o640); + assert.equal(await readFile(victim, "utf8"), "VICTIM", "the symlink target is untouched"); + + await rm(first.spillPath!); await writeFile(first.spillPath!, "stale", { mode: 0o666 }); await chmod(first.spillPath!, 0o666); + const second = await cap(big, { spillDirectory: directory }); + const rewritten = await lstat(second.spillPath!); + assert.equal(rewritten.mode & 0o777, 0o640); + assert.equal(await readFile(second.spillPath!, "utf8"), big.content[0].text); + }); +}); + +test("the spill directory must be owned by the runtime itself", () => { + const runtime = { uid: 2000, gid: 2000 }; + const entry = (uid: number, gid: number, mode: number) => ({ uid, gid, mode: 0o040000 | mode, isDirectory: () => true }); + assert.doesNotThrow(() => assertSpillDirectoryStat(entry(2000, 2000, 0o700), runtime)); + assert.doesNotThrow(() => assertSpillDirectoryStat(entry(2000, 2200, 0o2750), runtime)); + for (const [label, candidate] of [["owned by a worker", entry(2200, 2200, 0o700)], ["owned by root", entry(0, 2200, 0o2750)], ["not a directory", { ...entry(2000, 2000, 0o700), isDirectory: () => false }]] as const) { + assert.throws(() => assertSpillDirectoryStat(candidate, runtime), /spill directory/u, label); + } +}); + +test("a spill directory whose opened inode differs from the named one is refused", async () => { + await withDirectory(async (directory) => { + const probe = await open(directory, "r"); + const prototype = Object.getPrototypeOf(probe) as { stat: (...args: unknown[]) => Promise<{ ino: number }> }; + await probe.close(); + const original = prototype.stat; let calls = 0; + const swapped = mock.method(prototype, "stat", async function (this: unknown, ...args: unknown[]) { + const real = await original.apply(this, args); + calls += 1; + return calls === 1 ? Object.assign(Object.create(Object.getPrototypeOf(real)), real, { ino: Number(real.ino) + 1 }) : real; + }); + try { assert.equal((await cap(big, { spillDirectory: directory })).spillPath, undefined); } finally { swapped.mock.restore(); } + assert.deepEqual(await readdir(directory), []); + }); +}); diff --git a/src/runtime/toolResultSpill.ts b/src/runtime/toolResultSpill.ts index b0a8d55..f557c38 100644 --- a/src/runtime/toolResultSpill.ts +++ b/src/runtime/toolResultSpill.ts @@ -193,6 +193,7 @@ const notice = (input: Readonly<{ * creates itself stays 0700, so for every other engine nothing new is exposed. */ export const SPILL_FILE_MODE = 0o640; +export const SPILL_DIRECTORY_MAX_MODE = 0o2750; /** * Write the full payload where the agent can read it, atomically. @@ -203,13 +204,66 @@ export const SPILL_FILE_MODE = 0o640; */ const writeSpill = async (directory: string, name: string, text: string): Promise => { await mkdir(directory, { recursive: true, mode: 0o700 }); + const pinned = await pinSpillDirectory(directory); const file = path.join(directory, name); const temporary = `${file}.${process.pid}.${Date.now()}.tmp`; - const handle = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, SPILL_FILE_MODE); - // Explicit, so a restrictive umask cannot strip the group read an agent's sandboxed worker needs. - try { await handle.chmod(SPILL_FILE_MODE); await handle.writeFile(text, "utf8"); await handle.sync(); } finally { await handle.close(); } - try { await rename(temporary, file); } catch (error) { await unlink(temporary).catch(() => undefined); throw error; } - return file; + try { + const handle = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, SPILL_FILE_MODE); + let written: Awaited>; + // Explicit, so a restrictive umask cannot strip the group read an agent's sandboxed worker needs. + try { await handle.chmod(SPILL_FILE_MODE); await handle.writeFile(text, "utf8"); await handle.sync(); written = await handle.stat(); } finally { await handle.close(); } + if (!written.isFile() || written.nlink !== 1 || written.uid !== process.getuid?.()) throw new Error("spill file is not a private regular file"); + // Node has no openat: re-check that the directory entry still names the pinned inode before publishing into it. + await assertSameDirectory(directory, pinned); + // rename replaces a pre-existing destination entry (a symlink included) without following it. + await rename(temporary, file); + const published = await lstat(file); + if (!published.isFile() || published.dev !== written.dev || published.ino !== written.ino) throw new Error("spill file was replaced"); + return file; + } catch (error) { + await unlink(temporary).catch(() => undefined); + throw error; + } finally { + await pinned.handle.close(); + } +}; + +type PinnedDirectory = Readonly<{ handle: Awaited>; dev: number; ino: number }>; + +/** + * The spill directory must be a real directory owned by this runtime and no + * wider than `2750`: either Daimon's own `0700`, or a deployment-provisioned + * setgid directory whose group is not the runtime's own (a worker group). A + * symlinked, foreign-owned, world-accessible, or group-open-without-setgid + * directory is refused and nothing is written. Daimon cannot know *which* + * worker gid belongs to this agent; that mapping is the deployment's + * provisioning contract (`GROK_ENGINE_BROKER.worker.home.spillDirectory`). + */ +const pinSpillDirectory = async (directory: string): Promise => { + const before = await lstat(directory); + if (before.isSymbolicLink() || !before.isDirectory()) throw new Error("spill directory is not a real directory"); + const handle = await open(directory, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); + try { + const opened = await handle.stat(); + if (opened.dev !== before.dev || opened.ino !== before.ino) throw new Error("spill directory was replaced"); + assertSpillDirectoryStat(opened, { uid: process.getuid?.() ?? -1, gid: process.getgid?.() ?? -1 }); + return { handle, dev: Number(opened.dev), ino: Number(opened.ino) }; + } catch (error) { await handle.close(); throw error; } +}; + +/** Pure so a foreign owner is testable without root. */ +export const assertSpillDirectoryStat = (entry: Readonly<{ uid: number; gid: number; mode: number; isDirectory(): boolean }>, runtime: Readonly<{ uid: number; gid: number }>): void => { + const mode = Number(entry.mode) & 0o7777; + const groupOpen = (mode & 0o070) !== 0; + if (!entry.isDirectory() || entry.uid !== runtime.uid || (mode & ~SPILL_DIRECTORY_MAX_MODE) !== 0 + || (groupOpen && ((mode & 0o2000) === 0 || entry.gid === runtime.gid))) { + throw new Error("spill directory is not a private or provisioned worker-group directory"); + } +}; + +const assertSameDirectory = async (directory: string, pinned: PinnedDirectory): Promise => { + const [now, held] = await Promise.all([lstat(directory), pinned.handle.stat()]); + if (now.isSymbolicLink() || Number(now.dev) !== pinned.dev || Number(now.ino) !== pinned.ino || Number(held.ino) !== pinned.ino) throw new Error("spill directory was replaced"); }; /** Newest-first retention, so a busy agent cannot fill its own runtime home. */ From 4aef65c72bc9d12e83299a4cf8a590a18479e9b4 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 08:03:12 +0200 Subject: [PATCH 08/69] fix: refuse non-canonical Grok worker registration paths and truncated worker environment paths --- src/runtime/engineBrokerServiceCli.test.ts | 13 ++ src/runtime/engineBrokerServiceConfig.ts | 7 +- .../native/engineBrokerLauncherCore.inc | 166 ++---------------- .../engineBrokerLauncherIntegrationMain.inc | 60 +++++++ .../native/engineBrokerLauncherServer.inc | 150 ++++++++++++++++ src/runtime/native/launcherArgv.test.ts | 9 +- 6 files changed, 252 insertions(+), 153 deletions(-) diff --git a/src/runtime/engineBrokerServiceCli.test.ts b/src/runtime/engineBrokerServiceCli.test.ts index 20078ae..5522f10 100644 --- a/src/runtime/engineBrokerServiceCli.test.ts +++ b/src/runtime/engineBrokerServiceCli.test.ts @@ -63,3 +63,16 @@ test("v2 may declare an evaluator inference ledger that is never a subject ledge } assert.throws(() => parseEngineBrokerServiceConfig({ ...config("v1", [reg("agent-a", 0)]), inferenceLedgerPath: "/run/paideia-inference/inference.jsonl" }), /invalid engine broker service config/u); }); + +test("registration paths must be canonical, matching the native launcher's registration check", () => { + const good = reg("agent-a", 0); + for (const [field, value] of [ + ["workspace", "/workspace/0/../1"], ["workspace", "/workspace//0"], ["workspace", "/workspace/0/"], ["workspace", "/workspace/./0"], + ["profilePath", "/workers/0/../1/.grok/sandbox.toml"], ["profilePath", "/workers//0/.grok/sandbox.toml"] + ] as const) { + const registration = { ...good, [field]: value, ...(field === "profilePath" ? { eventsPath: value.replace(/sandbox\.toml$/u, "sessions/sandbox-events.jsonl") } : {}) }; + assert.throws(() => parseEngineBrokerServiceConfig(config("v1", [registration])), /invalid engine broker service config/u, `${field}=${value}`); + } + assert.throws(() => parseEngineBrokerServiceConfig({ ...config("v1", [good]), turnStore: "/var/lib/turns/" })); + assert.doesNotThrow(() => parseEngineBrokerServiceConfig(config("v1", [good]))); +}); diff --git a/src/runtime/engineBrokerServiceConfig.ts b/src/runtime/engineBrokerServiceConfig.ts index 742e2a5..af2a159 100644 --- a/src/runtime/engineBrokerServiceConfig.ts +++ b/src/runtime/engineBrokerServiceConfig.ts @@ -31,7 +31,12 @@ const V2_REGISTRATION = [...V1_REGISTRATION, "usageLedgerPath", "limits", "model const invalid = (): TypeError => new TypeError("invalid engine broker service config"); const plain = (value: unknown): value is Record => value !== null && typeof value === "object" && !Array.isArray(value); const exact = (value: Record, fields: readonly string[]): void => { if (Object.keys(value).length !== fields.length || fields.some((field) => !Object.hasOwn(value, field))) throw invalid(); }; -const absolute = (item: unknown): item is string => typeof item === "string" && item.startsWith("/") && !item.includes("/../") && !item.endsWith("/..") && !item.includes("\0"); +/** + * Absolute and canonical: no `.`/`..`/empty components and no trailing slash. + * The native launcher derives HOME, GROK_HOME and TMPDIR from the registered + * home and refuses a non-canonical one, so the broker's view must match it. + */ +const absolute = (item: unknown): item is string => typeof item === "string" && item.length > 1 && item.startsWith("/") && !item.endsWith("/") && path.posix.normalize(item) === item && !item.split("/").slice(1).some((part) => part === "." || part === "..") && !item.includes("\0"); /** The per-request stream written beside a registration's usage ledger. */ export const engineBrokerRequestLedgerPathFor = (usageLedgerPath: string): string => path.posix.join(path.posix.dirname(usageLedgerPath), "requests.jsonl"); diff --git a/src/runtime/native/engineBrokerLauncherCore.inc b/src/runtime/native/engineBrokerLauncherCore.inc index 41cc947..332fb0a 100644 --- a/src/runtime/native/engineBrokerLauncherCore.inc +++ b/src/runtime/native/engineBrokerLauncherCore.inc @@ -25,7 +25,6 @@ #include static __attribute__((noreturn)) void die(void) { _exit(111); } -static int bounded(const char *s, size_t n) { return memchr(s, 0, n) != NULL; } static int safe_component(const char *s, size_t n) { size_t i, l = strnlen(s, n); if (!l || l == n) @@ -157,12 +156,27 @@ static int load_registration(uint32_t slot, struct dbl_registration *out) { close(fd); return -1; } +/* Absolute, NUL-terminated within n, no empty, "." or ".." component and no + trailing slash: the launcher derives HOME, GROK_HOME and TMPDIR from it. */ +static int canonical_path(const char *s, size_t n) { + size_t l = strnlen(s, n), i = 0; + if (l < 2 || l == n || s[0] != '/' || s[l - 1] == '/') + return 0; + while (i < l) { + size_t start = ++i; + while (i < l && s[i] != '/') + i++; + if (i == start || (i - start == 1 && s[start] == '.') || + (i - start == 2 && s[start] == '.' && s[start + 1] == '.')) + return 0; + } + return 1; +} static int valid_registration(const struct dbl_registration *r, const struct dbl_request *q) { return r->uid >= 2200 && r->gid >= 2200 && - bounded(r->workspace, sizeof(r->workspace)) && - bounded(r->home, sizeof(r->home)) && r->workspace[0] == '/' && - r->home[0] == '/' && + canonical_path(r->workspace, sizeof(r->workspace)) && + canonical_path(r->home, sizeof(r->home)) && safe_component(r->agent_id, sizeof(r->agent_id)) && strcmp(r->agent_id, q->agent_id) == 0; } @@ -248,147 +262,3 @@ static uint64_t start_ticks(pid_t pid) { } return 0; } - -static __attribute__((noreturn)) void launch_fail(int status_fd, - uint32_t code) { - (void)full_write(status_fd, &code, sizeof(code)); - _exit(111); -} -static pid_t launch(const struct dbl_registration *r, int executable, - int prompt, int capability, int output, uint32_t *failure, - uint64_t *observed_start_ticks) { - unsigned char provider[DBL_MAX_TOKEN + 1] = {0}, mcp[DBL_MAX_TOKEN + 1] = {0}; - int status_pipe[2]; - if (capability_bundle(capability, provider, mcp) || - pipe2(status_pipe, O_CLOEXEC)) { - erase(provider, sizeof(provider)); - erase(mcp, sizeof(mcp)); - return -1; - } - pid_t p = fork(); - if (p < 0) { - close(status_pipe[0]); - close(status_pipe[1]); - erase(provider, sizeof(provider)); - erase(mcp, sizeof(mcp)); - return -1; - } - if (p != 0) { - uint32_t code = 0; - close(status_pipe[1]); - if (full_read(status_pipe[0], observed_start_ticks, - sizeof(*observed_start_ticks)) || - !*observed_start_ticks) { - close(status_pipe[0]); - erase(provider, sizeof(provider)); - erase(mcp, sizeof(mcp)); - kill(p, SIGKILL); - waitpid(p, NULL, 0); - return -1; - } - ssize_t got = read(status_pipe[0], &code, sizeof(code)); - close(status_pipe[0]); - erase(provider, sizeof(provider)); - erase(mcp, sizeof(mcp)); - if (got == 0) - return p; - if (got == (ssize_t)sizeof(code)) - *failure = code; - kill(p, SIGKILL); - waitpid(p, NULL, 0); - return -1; - } - close(status_pipe[0]); - if (prctl(PR_SET_PDEATHSIG, SIGKILL) || getppid() == 1 || setpgid(0, 0)) - launch_fail(status_pipe[1], 1); - uint64_t identity = start_ticks(getpid()); - if (!identity || full_write(status_pipe[1], &identity, sizeof(identity))) - launch_fail(status_pipe[1], 1); - struct rlimit z = {0, 0}; - if (setrlimit(RLIMIT_CORE, &z) || setgroups(0, NULL) || - prctl(PR_SET_KEEPCAPS, 1, 0, 0, 0) || setresgid(r->gid, r->gid, r->gid) || - setresuid(r->uid, r->uid, r->uid)) - launch_fail(status_pipe[1], 2); - zero_caps(); - if (prctl(PR_SET_KEEPCAPS, 0, 0, 0, 0) || - prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) - launch_fail(status_pipe[1], 3); - /* Received fds carry MSG_CMSG_CLOEXEC and may already sit on 3-5: dup2 onto - itself keeps close-on-exec (the prompt vanished at exec) and an - overlapping order clobbers a source. Lift all of them above the targets - first so every dup2 below changes the fd number and clears CLOEXEC. */ - int status_fd = fcntl(status_pipe[1], F_DUPFD_CLOEXEC, 16); - if (status_fd < 0) - launch_fail(status_pipe[1], 4); - int null_input = open("/dev/null", O_RDONLY | O_CLOEXEC); - int high_prompt = fcntl(prompt, F_DUPFD_CLOEXEC, 16), - high_capability = fcntl(capability, F_DUPFD_CLOEXEC, 16), - high_output = fcntl(output, F_DUPFD_CLOEXEC, 16), - high_executable = fcntl(executable, F_DUPFD_CLOEXEC, 16); - if (null_input < 0 || high_prompt < 0 || high_capability < 0 || - high_output < 0 || high_executable < 0 || - dup2(null_input, STDIN_FILENO) < 0 || dup2(high_prompt, 3) < 0 || - dup2(high_capability, 4) < 0 || dup2(high_output, STDOUT_FILENO) < 0 || - dup2(high_output, STDERR_FILENO) < 0) - launch_fail(status_fd, 4); - /* The worker needs fd 5 only to be executed: close-on-exec keeps the Grok - image descriptor out of the worker and every tool child (execveat with - AT_EMPTY_PATH still works for an ELF; only a #! script would lose it). */ - if (dup2(high_executable, 5) < 0 || fcntl(5, F_SETFD, FD_CLOEXEC) < 0) - launch_fail(status_fd, 5); - close_other_fds(status_fd); - char *const argv[] = {"grok", - "--sandbox", - "daimon-strict", - "--always-approve", - "--no-subagents", - "--prompt-file", - "/proc/self/fd/3", - "--no-memory", - "--disable-web-search", - "--no-plan", - "--verbatim", - "--system-prompt-override", - DBL_GROK_SYSTEM_PROMPT, - "--tools", - DBL_GROK_TOOLS, - "--max-turns", - DBL_GROK_MAX_TURNS, - "--cwd", - (char *)r->workspace, - "--output-format", - "streaming-messages-json", - "--model", - "daimon-broker-grok", - NULL}; - char home[300], grok[300], tmp[300], mcp_env[DBL_MAX_TOKEN + 24], - provider_env[DBL_MAX_TOKEN + 32]; - snprintf(home, sizeof(home), "HOME=%s", r->home); - snprintf(grok, sizeof(grok), "GROK_HOME=%s/.grok", r->home); - /* Worker-private temp: strict grants TMPDIR read-write, and shared /tmp is - kept from the worker by the deployment's modes (attested by the broker). */ - snprintf(tmp, sizeof(tmp), "TMPDIR=%s/tmp", r->home); - snprintf(mcp_env, sizeof(mcp_env), "DAIMON_MCP_CAPABILITY=%s", mcp); - /* Grok 1.0.34 never runs `[auth_provider.*]` helpers for a custom model, so - the worker's `[model.daimon-broker-grok] env_key` reads the turn-scoped - proxy capability from here. It is as exposed as the MCP capability. */ - snprintf(provider_env, sizeof(provider_env), "DAIMON_PROVIDER_CAPABILITY=%s", - provider); - erase(provider, sizeof(provider)); - erase(mcp, sizeof(mcp)); - char *const envp[] = {home, - grok, - tmp, - mcp_env, - provider_env, - "DAIMON_CAPABILITY_FD=4", - "PATH=/usr/local/bin:/usr/bin:/bin", - "LANG=C.UTF-8", - "TZ=UTC", - NULL}; - syscall(SYS_execveat, 5, "", argv, envp, AT_EMPTY_PATH); - erase(provider_env, sizeof(provider_env)); - erase(mcp_env, sizeof(mcp_env)); - launch_fail(status_fd, 6); -} - diff --git a/src/runtime/native/engineBrokerLauncherIntegrationMain.inc b/src/runtime/native/engineBrokerLauncherIntegrationMain.inc index 1191e2d..f773c55 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationMain.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationMain.inc @@ -1,3 +1,61 @@ +/* Non-canonical or unterminated homes/workspaces are refused at registration: + HOME, GROK_HOME and TMPDIR are derived from them. */ +static void noncanonical_registration_cases(struct dbl_registration r) { + const char *homes[] = {"/tmp/worker-home/../other", "/tmp//worker-home", + "/tmp/worker-home/", "/tmp/./worker-home", + "/tmp/worker-home/..", "relative/home", NULL}; + struct dbl_registration bad[8]; + size_t count = 0; + for (; homes[count]; count++) { + bad[count] = r; + bad[count].slot = 100 + (uint32_t)count; + memset(bad[count].home, 0, sizeof(bad[count].home)); + strcpy(bad[count].home, homes[count]); + } + bad[count] = r; + bad[count].slot = 100 + (uint32_t)count; + memset(bad[count].home, 'a', sizeof(bad[count].home)); /* over-long: no NUL */ + bad[count].home[0] = '/'; + count++; + bad[count] = r; + bad[count].slot = 100 + (uint32_t)count; + strcpy(bad[count].workspace, "/tmp/workspace/../etc"); + count++; + int f = open(DBL_REGISTRY, O_TRUNC | O_WRONLY, 0600); + check(f >= 0 && write(f, &r, sizeof(r)) == sizeof(r) && + write(f, bad, sizeof(bad[0]) * count) == + (ssize_t)(sizeof(bad[0]) * count), + "noncanonical registry"); + close(f); + pid_t child = fork(); + if (!child) { + setgid(DBL_BROKER_UID); + setuid(DBL_BROKER_UID); + for (size_t i = 0; i < count; i++) { + int s = connect_socket(), p = sealed("prompt"), + c = sealed_bundle("provider.Path-1", "mcp.Path-2"); + struct dbl_request q = request(); + struct dbl_result result; + q.slot = 100 + (uint32_t)i; + send_request(s, &q, p, c); + if (!read_all(s, &result, sizeof(result)) || + result.status != DBL_STATUS_PRELAUNCH_FAILED || + result.stage != DBL_STAGE_REGISTRATION || result.worker_pid != 0) + _exit(10 + (int)i); + close(s); + close(p); + close(c); + } + _exit(0); + } + int status; + waitpid(child, &status, 0); + check(WIFEXITED(status) && WEXITSTATUS(status) == 0, + "noncanonical registration refused"); + f = open(DBL_REGISTRY, O_TRUNC | O_WRONLY, 0600); + check(f >= 0 && write(f, &r, sizeof(r)) == sizeof(r), "registry restore"); + close(f); +} /* A worker that fails before or at exec must not run anything: the launcher child exits instead. The registered "executable" is a #! script; fd 5 is close-on-exec, so execveat(AT_EMPTY_PATH) cannot run it (a script needs @@ -113,6 +171,8 @@ int main(void) { check(WIFEXITED(status) && WEXITSTATUS(status) == 0, "org cases"); exec_failure_case(r); puts("native-stage exec failure complete"); + noncanonical_registration_cases(r); + puts("native-stage noncanonical registration complete"); kill(broker, SIGKILL); waitpid(broker, 0, 0); check(WIFEXITED(status) && WEXITSTATUS(status) == 0, "org cases"); diff --git a/src/runtime/native/engineBrokerLauncherServer.inc b/src/runtime/native/engineBrokerLauncherServer.inc index a6cb347..75f452b 100644 --- a/src/runtime/native/engineBrokerLauncherServer.inc +++ b/src/runtime/native/engineBrokerLauncherServer.inc @@ -1,3 +1,153 @@ +static __attribute__((noreturn)) void launch_fail(int status_fd, + uint32_t code) { + (void)full_write(status_fd, &code, sizeof(code)); + _exit(111); +} +static pid_t launch(const struct dbl_registration *r, int executable, + int prompt, int capability, int output, uint32_t *failure, + uint64_t *observed_start_ticks) { + unsigned char provider[DBL_MAX_TOKEN + 1] = {0}, mcp[DBL_MAX_TOKEN + 1] = {0}; + int status_pipe[2]; + if (capability_bundle(capability, provider, mcp) || + pipe2(status_pipe, O_CLOEXEC)) { + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); + return -1; + } + pid_t p = fork(); + if (p < 0) { + close(status_pipe[0]); + close(status_pipe[1]); + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); + return -1; + } + if (p != 0) { + uint32_t code = 0; + close(status_pipe[1]); + if (full_read(status_pipe[0], observed_start_ticks, + sizeof(*observed_start_ticks)) || + !*observed_start_ticks) { + close(status_pipe[0]); + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); + kill(p, SIGKILL); + waitpid(p, NULL, 0); + return -1; + } + ssize_t got = read(status_pipe[0], &code, sizeof(code)); + close(status_pipe[0]); + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); + if (got == 0) + return p; + if (got == (ssize_t)sizeof(code)) + *failure = code; + kill(p, SIGKILL); + waitpid(p, NULL, 0); + return -1; + } + close(status_pipe[0]); + if (prctl(PR_SET_PDEATHSIG, SIGKILL) || getppid() == 1 || setpgid(0, 0)) + launch_fail(status_pipe[1], 1); + uint64_t identity = start_ticks(getpid()); + if (!identity || full_write(status_pipe[1], &identity, sizeof(identity))) + launch_fail(status_pipe[1], 1); + struct rlimit z = {0, 0}; + if (setrlimit(RLIMIT_CORE, &z) || setgroups(0, NULL) || + prctl(PR_SET_KEEPCAPS, 1, 0, 0, 0) || setresgid(r->gid, r->gid, r->gid) || + setresuid(r->uid, r->uid, r->uid)) + launch_fail(status_pipe[1], 2); + zero_caps(); + if (prctl(PR_SET_KEEPCAPS, 0, 0, 0, 0) || + prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) + launch_fail(status_pipe[1], 3); + /* Received fds carry MSG_CMSG_CLOEXEC and may already sit on 3-5: dup2 onto + itself keeps close-on-exec (the prompt vanished at exec) and an + overlapping order clobbers a source. Lift all of them above the targets + first so every dup2 below changes the fd number and clears CLOEXEC. */ + int status_fd = fcntl(status_pipe[1], F_DUPFD_CLOEXEC, 16); + if (status_fd < 0) + launch_fail(status_pipe[1], 4); + int null_input = open("/dev/null", O_RDONLY | O_CLOEXEC); + int high_prompt = fcntl(prompt, F_DUPFD_CLOEXEC, 16), + high_capability = fcntl(capability, F_DUPFD_CLOEXEC, 16), + high_output = fcntl(output, F_DUPFD_CLOEXEC, 16), + high_executable = fcntl(executable, F_DUPFD_CLOEXEC, 16); + if (null_input < 0 || high_prompt < 0 || high_capability < 0 || + high_output < 0 || high_executable < 0 || + dup2(null_input, STDIN_FILENO) < 0 || dup2(high_prompt, 3) < 0 || + dup2(high_capability, 4) < 0 || dup2(high_output, STDOUT_FILENO) < 0 || + dup2(high_output, STDERR_FILENO) < 0) + launch_fail(status_fd, 4); + /* The worker needs fd 5 only to be executed: close-on-exec keeps the Grok + image descriptor out of the worker and every tool child (execveat with + AT_EMPTY_PATH still works for an ELF; only a #! script would lose it). */ + if (dup2(high_executable, 5) < 0 || fcntl(5, F_SETFD, FD_CLOEXEC) < 0) + launch_fail(status_fd, 5); + close_other_fds(status_fd); + char *const argv[] = {"grok", + "--sandbox", + "daimon-strict", + "--always-approve", + "--no-subagents", + "--prompt-file", + "/proc/self/fd/3", + "--no-memory", + "--disable-web-search", + "--no-plan", + "--verbatim", + "--system-prompt-override", + DBL_GROK_SYSTEM_PROMPT, + "--tools", + DBL_GROK_TOOLS, + "--max-turns", + DBL_GROK_MAX_TURNS, + "--cwd", + (char *)r->workspace, + "--output-format", + "streaming-messages-json", + "--model", + "daimon-broker-grok", + NULL}; + char home[300], grok[300], tmp[300], mcp_env[DBL_MAX_TOKEN + 24], + provider_env[DBL_MAX_TOKEN + 32]; + /* Worker-private temp: strict grants TMPDIR read-write, and shared /tmp is + kept from the worker by the deployment's modes (attested by the broker). + A truncated path must never name a different directory: fail instead. */ + if ((size_t)snprintf(home, sizeof(home), "HOME=%s", r->home) >= sizeof(home) || + (size_t)snprintf(grok, sizeof(grok), "GROK_HOME=%s/.grok", r->home) >= + sizeof(grok) || + (size_t)snprintf(tmp, sizeof(tmp), "TMPDIR=%s/tmp", r->home) >= + sizeof(tmp)) { + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); + launch_fail(status_fd, 6); + } + snprintf(mcp_env, sizeof(mcp_env), "DAIMON_MCP_CAPABILITY=%s", mcp); + /* Grok 1.0.34 never runs `[auth_provider.*]` helpers for a custom model, so + the worker's `[model.daimon-broker-grok] env_key` reads the turn-scoped + proxy capability from here. It is as exposed as the MCP capability. */ + snprintf(provider_env, sizeof(provider_env), "DAIMON_PROVIDER_CAPABILITY=%s", + provider); + erase(provider, sizeof(provider)); + erase(mcp, sizeof(mcp)); + char *const envp[] = {home, + grok, + tmp, + mcp_env, + provider_env, + "DAIMON_CAPABILITY_FD=4", + "PATH=/usr/local/bin:/usr/bin:/bin", + "LANG=C.UTF-8", + "TZ=UTC", + NULL}; + syscall(SYS_execveat, 5, "", argv, envp, AT_EMPTY_PATH); + erase(provider_env, sizeof(provider_env)); + erase(mcp_env, sizeof(mcp_env)); + launch_fail(status_fd, 6); +} + static void supervise(int client, pid_t pid, int output, struct dbl_result *out) { unsigned char bytes[DBL_MAX_OUTPUT + 1] = {0}; diff --git a/src/runtime/native/launcherArgv.test.ts b/src/runtime/native/launcherArgv.test.ts index d371bab..ed73ea9 100644 --- a/src/runtime/native/launcherArgv.test.ts +++ b/src/runtime/native/launcherArgv.test.ts @@ -21,7 +21,7 @@ const defines = (): ReadonlyMap => { /** The compiled worker argv, token by token, exactly as `launch()` passes it to `execveat`. */ const compiledArgv = (): readonly string[] => { - const source = read("engineBrokerLauncherCore.inc"); + const source = `${read("engineBrokerLauncherCore.inc")}${read("engineBrokerLauncherServer.inc")}`; const block = source.match(/char \*const argv\[\] = \{([\s\S]*?)NULL\};/u); assert.ok(block, "launcher argv array not found"); const values = defines(); @@ -55,15 +55,16 @@ test("the compiled system prompt is byte-identical to the contract prompt pinned }); test("the launcher exports the turn provider capability under the env_key the worker config reads", () => { - const source = read("engineBrokerLauncherCore.inc"); + const source = `${read("engineBrokerLauncherCore.inc")}${read("engineBrokerLauncherServer.inc")}`; assert.match(source, new RegExp(`"${GROK_BROKER_PROVIDER_CAPABILITY_ENV}=%s"`, "u")); assert.match(source, /char \*const envp\[\] = \{home,\s*grok,\s*tmp,\s*mcp_env,\s*provider_env,/u); assert.match(renderGrokBrokerWorkerConfig(), new RegExp(`\\nenv_key = "${GROK_BROKER_PROVIDER_CAPABILITY_ENV}"\\n`, "u")); }); test("the launcher gives every worker a private TMPDIR under its registered home", () => { - const source = read("engineBrokerLauncherCore.inc"); - assert.match(source, /snprintf\(tmp, sizeof\(tmp\), "TMPDIR=%s\/tmp", r->home\);/u); + const source = `${read("engineBrokerLauncherCore.inc")}${read("engineBrokerLauncherServer.inc")}`; + assert.match(source, /snprintf\(tmp, sizeof\(tmp\), "TMPDIR=%s\/tmp", r->home\) >=\s*sizeof\(tmp\)/u); + assert.match(source, /canonical_path\(r->home, sizeof\(r->home\)\)/u); assert.match(source, /char \*const envp\[\] = \{home,\s*grok,\s*tmp,/u); assert.equal(GROK_ENGINE_BROKER.worker.home.privateTmp.relativeToWorkerHome, "tmp"); }); From 4c761c84025df6b78ca9cca73d68cd883fea5bf9 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 08:03:26 +0200 Subject: [PATCH 09/69] build: rebuild native engine broker artifacts with canonical registration paths --- src/contracts/runtimeContractManifest.ts | 6 +++--- .../artifacts/daimon-engine-broker-arm64 | Bin 67456 -> 67456 bytes ...daimon-engine-broker-arm64.provenance.json | 2 +- .../native/artifacts/daimon-engine-broker-x64 | Bin 55392 -> 55392 bytes .../daimon-engine-broker-x64.provenance.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index e56f6ae..46a3588 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -129,9 +129,9 @@ export const GROK_ENGINE_BROKER = { projectionVersion: "noopolis.daimon.grok-broker-projection.v1", slotPreflightVersion: "noopolis.daimon.grok-slot-preflight.v2", artifacts: { - sourceSha256: "27fcdc8ffb6946e391dad039f695968231d1ae435c6a38dd9bb8c7fb00ed18b9", - x64Sha256: "ff83efc37c77c4ee920080d0eedfca497b504a561c876348af0fec25142a5f9c", - arm64Sha256: "25d37be0d294529b3466d73c0d879d18c7850c2d24450a7a9cc11d28b9a4cf1e" + sourceSha256: "c0082d4b366ffdb860d8154ee7f402b8f8be1f09d4eda277eda6965198ab6a75", + x64Sha256: "69e2865c722606a71501bc38d8b9c4c2c397e748327a8077e51e040319d1280d", + arm64Sha256: "16a3f89d84b7139d556b070a626c75ece224d5e05c52d48e045b38086c0a0382" } } as const; export const AGY_SUBSCRIPTION_REALM = { diff --git a/src/runtime/native/artifacts/daimon-engine-broker-arm64 b/src/runtime/native/artifacts/daimon-engine-broker-arm64 index 712890f0a30f784d1246975a870e3a558743b823..6c393474c393788fcddb983d037e3d6986f75d10 100755 GIT binary patch delta 8198 zcmc&(e{@vE^`CifcazNz$dX?hLfG9TBxFM%A-_NZ*$p891;vmZTN{Vrzu~lNsPKS6h1&zf#jx44b{BCla()sI=+5tqgn^C4Z2LFnPTdiTUX!F#VyT+Hd|aqG!C-s%MwoMKjL6=c>g{5 zLJMOz3D*+a9BJsx(Q)Pp`&1(2GEK0S!i>`NoG${&Ap*w-D25OH&)95Ln+kqBGp;kHobU#p66grW{fQ|xP z8ih`xqU3a%mYk62i9#pStw5IpT@!`c=@Fn01688XY}yO-b)cP5XfAyP^bAm66q-j_ zDX?z}>>Gtnr6oWY0}Vu>g>*mAjX)1Zp+)o}&|N@-QRoc%9OzM?Ls4i6C8ol@sj#p1 zt$j;rF3>8VNl|Dy-3fFxP3azERKwk&i8HLu-S3u7I^+llzDJvcJO;52nOfn6!92*R(Fh%*x9d z+x@aMlWv(b#k(EYd7S4!%y4B;PVM;=e2^&(FM$pM56a0s=Vb0Yd`0V$5}0LBPU{KE zDbBxxS7cT$`fM_FG;s9k7k#A=#w(PDRPYD_kV$cJNohEMK0z+{WapD;!xAQcA{u4x zYmk^S03`z}(Qpm}EAnu~W0aLu;Eh$qB;3G}ugV-)o8TRuShY$F08HnBJAAQfrBG~O z*VzT0s5ImubpHVZ+ZbzW<*d<%A|_ylTs9^x#$-&H@QCOKnbn%sIG}0iLWTkRC3p)28B=#V61*TTX#6QeaFgYt;>s>bwAHF^jE&!nhY(=KG&P+d0}{E9B?tkS&w%rLSz6 z1z$kIeo8ABzA-3E@FU521ekL2-?0Fd*lqlebiF;3AD}z!r9430_I!RHy>HJfs*D=v z8jSPLF{}*PNjOb0+M+kGk1US#DNl=WgIak4DBPee@K;UW?vYl5pRkLU+l~G#TH?rb z{16ffgVlcK>bJRUZLk0&Q=Cb_87%V)ddT76X8MgIi~oz>cgz_-2Q9j5Ue+p)uP{?` z_7vD@cD9W>X<2q|($2tKvDre2cDpin2R)LV$)6^Fc4k%=CT_sJIBV?)4rU_#8sjv za5r?_PR88F%Hq}RS-k%liS@sRwg6*&x&y+~KN`iNfggh|CMg3KZjt;KGua3qqYrZ@ zUULP_9r_ZIdk9@GZZ@jjVg6C@92Ug>_dJaHU>JW0_2w}C0P0BEoh^YGR61pL$sXt% z#FCF8Y6y=@75z}hbOr=((fL%JcY)tQPffWoqub234I)N4PSlUs8S6hCp6nh<%=7a1 z>5;sq)J4#|A0-GmcCvoNg+&I4=fA?IQ-A&q{46C-t>$}Z$CEqN#;@dN5KldoUV5krmB~5r1Q>Qj(gE&i(w@m-Fd9ecJO~SMff!5kH|Rf225$$v?hKR^-sHoZzf|= zb4H0O4(%`w-Z8k}AdWv4+R1z+s$JF0T*((oj*OuvQyj_#bkfgh$*(^pF_H{CenZ~SGNTYQ}FB1cIk-$8Rr z=JQ7CD0!3bC-=-uek9Pa$}0!`Z}q)>z$IF$=+=#{t(V0`AFUeLU7Ro5eDf=drRCn2?MV z$tSsHbxfgaXHWJDLkM+))odhqudCmY%lZb7b5-KXKpc*oFNc*Qr0Xd}^Tzu3UW5;n zvU03XWYPg)2N!sQt~!+b9H=v#t#x7wGg9kVYRaWnhSijgNaTtX}P_{xsX-EK83rrl~;O(yd$OtD|S--w( z@Uau)mlViMcD68s^LT-{T}^U|*=M1zN9Qx0&EPY@2eGTd1F_4-_-S^qJh2tYwULcy*V|%Q+dG?X^4{mdbw~^>1hztv5W+pOF&+*ZhtldUQ?KqQ zQ_ZNyq7)34sj-f76))mLcxxQ`(o?R!>nm4{sN=0{Xh*PI-QucHPZy5&7b4_2QCi(x zz1l6Qr*e|~1>mQm6m&>xY`0fZzs$7w@q*ZidtB$20ZDzA$?D&dC-|p;$ID{p4v(yE zQDjw|AiM(%)dWjb2P;!A*O}De=S(V`8H9;A(P4%D3N%LQ{rTf1~xB5?bGyEyS8oLZ_ zD<#%=u>~m?9fHVpHOv5InLQoo8y}Wg2(wUL@d)iG*MeLdFmH6oneS6ZJ`>?0~66p{wueFr-Pli$v zlSGIR=k~uZ;Yfh@;KE~G;q67{W&d|x8Pc%8p}=UH?HF|sqY3&>WTT->dhf{F|th!HTFvVhGWlB z28Nr={!2H;`A72O{U>b+{x7_Vew#2PbW?ENq-i1;zZ|yyn^MN<;8)^s{5v*?d)pnD zFxjh%Et^ph+i<@yAGT%#HCLB4hm#C!MiwqTIj$wV zj@n%L^G@O}vX+Ucnc_T!9>EQ+ewQ>{u};U*oF@e41qOZ)b?uv5Te|b}3v}2uS^DGx zeeL?;#Q9>pV$)ei5_0`>b^Q#fw3@M1-Q#JU`>s{{;*tKdnd(5+UW7qi@pwPU<2t`U zT#w?wJ0;bTqzt%4yWpoAbbhXAH-pC{MTkC+eDecjJUg&oUhz-|+XBx{b1nnsi=}I8 zT&rW$vCHu0q+y9hOfE%c|yvBYP+k=VZ@x@cK6pHdEtLC zL~QO<@M}2BXMVut%&otHjtw%~iuiK&E19m;%=G^LycovSzd#K4 z_WXX^^6WO@7{T9%>L;<$BdxXm%k2yNi?K&y-hy9*x}aa~H!`Z!fGC8@-BeUpEgd>fch%+bpV4D=4_kHx z=8D9*SyuGFAUbx_%=&6eUkH_P-4O?-<5k*J|8u^LEDM_~8_$XRkN!UmVxlr|k~$X7 z;YVoi!fL*Z#ujGswa8S{3Ln(6ykhS2XQ;&@Fp44R6t2_B<-H zklF2TXqpFf3+P(V+<>ON4(i&cX$DyBws$n`HqiI>V<6DL0Zls!>iJmH){862A2n?a zt|_ihumI3b(8ocC4r|&O(DR`BH-$f8Vp!mA{5|H184rPBO}hy|(g+lgxjkoI0{$_> z-#LuAV+As#ylf+E}g60K^_p1vPVn4JEk;P+vaPl1W9 z%snpYDd}l0hIk(3;7c@b?esW1sz*^$dg)*7+0vF?N?V&Jo#{nmg;d-}>()+}w)N4k z*WNdA`EOx7h(eS7DZ|rpE~>dO>TSQJtM185dlk*MiDo>tC>}J$zDDcrsgpdf(;FaV zd&#u!2d3dSAs9=m)@9vX)74{n+;6FUVyETF7cBFC^=r%a9TvHr$Nl-}kRN859#jp*G~Tw)zOc> z-8<3c46rl6a3|Iu+-ddCZ6`jWf2_~Tng-`udFb0NqE&2pt#Cpr`El~Z<)}T5oR=$_ z@Q6JTu4?f2{@}-NDb10uCXtUM3ApKq2ggW%#1|UzM0oK-xSxWxPHtwPKFIV&T$UJJ}_)a%E!?Hf|eIq2sJaosX&9YPX(a`rd(XfV|7`=`s z(JAy>Pv$YN*B&14JEC6p9Ss~F*NfBsU0ML+f2R|eBD~`Nv%pH)qgCR=Uhm(vuQ{*k z8syw74Vk)DNPN2GJ-l*?Cqj33bxR!nI4%T7=(eswz~&?bY4Id=WyrkJye7=9=~@Q% z8+VPkt*5J7MyJS`i0wKer2ccdo-T)h&zPWOhEzvr-_ouXz|G+URJOY7NdtxqvrAWZ zeFiqEJ=7`ksXV32VZ`jMYeR5^zS`9c*c|ypy0q&FBUD8`fO@))2=<$A@Ld@uOfzyK Q*N=sCJ@`tqiQD-90=M`?2><{9 delta 7830 zcmc&(dw5jUwcq=k$>a?)BxLfQnFR8fKpv1u$c2!T1V}(ph^b=Ll9;wq16E!ZL7fTW zEnZM_y3j!d2|g+lrG+L4sc;?CQratusaO!bOn_FXkDFWs5=c1rx6jPL+W+t*{)8xFT<68F5vBgB&VmN;5It-N z;5*37E#{`6R#Q-`i7I%LZCOBt{S%D~SYmP1!*8|!UmffVcCe4~rQ7&T^r)0;u7yD* zXmkqbY}z9gL|39V4rLCVmx}Ud1m%-KXQC9LB%qX_M55SHrlU-wYB?dg9yReyrF-Nw z^Q-{FN@O9rZAMH!9V| z5gAh@#!Q&74aK8AhX9FS^MQ$nGnthH<$~|h_A#m2KB7;GVYBkdktC$V(8GWz8dMOO*@tRD$PoCM1&!x*KUkAkut_u=w_hFVQ3P~1^Nikyf8GG9tHX)&?#YPD(wXN zIneqrG@ZTx8kZ0kzaR|Fr1%62O#`|p47E`u(8WMkg`rur1n3r^>%-7-v=!(FKsSe> zIdlZ*Fwh-gXdaCKosbCshM@&CF%kX+dN>T7NcRBU2=sUuT0~C)-4FC!7+OLHfPM{h zFbthc-vCV?3;$|2{OhEQvG6a@`0Ftn1s z02-GB|AwJe6rTkD0$mh_)=(wT#Xwhup)+U+&@DjMhoQ4*E6@*sZVp51=?KtapgY3Q zIW&@F$@+)PHjS(~xWgwe{4}&F`q#YU?Pu($`Q2GVCW#6yme@n6WtFKm$@*!RVo?mF z%EjbPyLt4$HO=FWW=(((ByCO4cNtVMh!Ws9$4c#(tl;fU z26cky0GQ4Lw|5xSTw&P2ZnX(K-rbOcr2cz!Y-Ma|D`$-zC}IFZz_KuC(I;c>F>Rt9 zbQWt`W3Q%Jgib$nK1u{*=k5j@lovtCcSUpd(w!M;DQCodptd}UZ4&epc$jKvrl&IQ zaJGW;4Qft15KOLt^NwZ0Mm%hY3HmQ<ZhBWtFtPJ;i|n04DVf=>3}{@`eV^1h z9aaKrwX&Jx7aBNtD zGjd=V4cHDZqX>OBHD{HlJO&Lv8!YRPFdP#Gel0NYO|&~}p?L+Q7vX?DIR&mvcdR0N zwv{iU>g@9PABvX1t6}R7dLsMT(nvLH3h&t_v7X;T=3}f|4?slvJHwbX@YArxBzNz{ zyCm=BG&VFaOt*|nz3E$s+w>_Ubc!Kgemp|m9^@Yf&*4F=f2SMuz99Z6>dirXHR{=n zwYT^RsB2tB$#&T5!<0`WYlw)85baRMaC!yLb$+bQE8w4{jPZA*%#LQ8`jDgSXX}P+ zjP-mK9PD;_YP^g0P<+n9u@$hqAH@$ncCKzn!6bK4Th3mdNfo)b@jiMgw~BA4Bd8xI zdtMiRhK}Xs@+W8{@8{CcReB)*UOtaL&Mz3-b`@KS5AGF#eHs(7x49b_nH^SyVu|Bc z^G3);n#*DxB11K-C)H8FtLcE_b^CpgStD6VlO8V||;gl(&HIqyD-)QILFL@+ATRJtme^A8NVA*OqQCeqShZd(K*6N2h ze8XDVkEz6&hC$4Amh;Kz+4i1s_+6}g+^+S#TFZq0h~2TacVrFoj>t^Ca2xl2cP3H2&{D8Lgz!Z z#L1-R(LveRIfNa5MV1{yvcVyC{Z(Y3EVj0{?>^Qema!5;7rdy8n63ctI*p204Zob~ zxL+iKG9-xixXO7)Lo*cIPUh4uV`E@-#A)$H8Eb4yLgHAi^az8YgW@fro5q|rP?3gg zX{=Fn#8B=6){)Ab98`Wpp$PA~UfiVnYnqB&>cy7v8+eqvR|MQebR?+g+z4gSsV{l3 z-05uN>+Uqm&@it_HnFm1({Pv9i&6@KU_Vorl@I&k>eQD@j&_lqGEGUvOaMAQe4g?oV6 z1Qw%XE+ja~F$17`B6I#3TOr(#?nD?`0J#FWFdQ!T;k=s8YVvxqHy)qc);&OUQ$I5IRec zXvJD;EzX`NArtW`!bhxWK3uH33s^G_L@p~s2MuhG>#`Q@alDQl)8g1b?rDip`8(Eu z$=^V>%2d0>ks}PDgV5n}3A_bYsgX5`7dYKjIWcbqDdiksdaqp&i-S%J^68 zXxj@90~1SRWht$1BKO21perJoxk^rL#oKJ7Z8V#2jbcmBuDx@$g8Q9tZ2@dthCS%^ z$Dp)U7prsHiq&Y;qZo7b=l2z>QT7rQe>L>u^|=4brV{l~M~NDNI$l5fU-Or!>y%0A zR|TWJH;h!ObE>DS=d(@TprQW!w&`lr_UY=EX=d+rTfXV)Ay%dSEn$rJy1k|NVT(K_2ew0I`oQB#WqC_X8kShNhyqXX zZG^7BQBK8y$J%QcQ+I|7aC+-_6H{N;@q|t;uhsLFj(0G1yN)Mza)ZE=1Nt_uzDO0* zXWq`$C@!h^OQ0QB^Y$-0B=wMAQqR^(>Y2x7efAM-Z9|Q1vVQZhWxIQaTBE&}?uhXY z=Ei!@S>wEacg1^)v3$Zd-XV^OLLW>|&k_}pF!valitJqRj+Fcr?)6x2i{l_-Wc4uJ zUNy;_jb#@b;NLJ#u_@Nm52_}Yju9vn+n{)-xLH6_jr#4uy1;J>)|*kEhdTKbC2bDO zh&K+gHq#Akv=Lh%-N1@6a0sVW&*huwXVtmWXI;@eE3iw&?XX;QyhiC!q`|U>bS%j+ zNnkEu;EPb#uHPaoj)F_{VRfdIeTlxU{%NXu5&ab#2ihdo*!Rk;9){Fj#n}D0i#|~E zfU7MY$59qj?KnkV$AQe^abM2iI{ylIcN}39LIp)kdkO|2U_@g=wEDs_8jhYoxns3+iQcCy>d2IRh^tCW4=23S|8{i`k4qd=WmA^&NKEFv>Yf^8S;i#nNbYDd(w}Qn zM-9Tn8kudtiViG^p#QF&)lyP^WhHv9|QG2iA`T8sgg~9sci5= z&&FA|to%Gf`zl3Bd|Y-n*swL)9;7k#Rpu1|RK}64 zqpRrN`ltElG*G|L{Qd=T=;;3?AV0ui)I57C|CqMU7Ps~L z{bWuyH@}QV;)*94^kJVjbUlAl9sk{2gO#76h{Xl`+pdzue=)km|I#+NmUU{{VdPiS z?~nwLdwt!SHcrU)XxbFed7w9gZUk)t^@FYhWp8QPW>6F8E+Gd!2Ws1^X>k(XI6$!o zY{EWGTMX(0-2!TON7IsJOoEG2KC`$i`xb}`v*j8*ueIx^D)G6BU8tXN=p1g4-21Dgga9vey{%D8S?wj!&r5pKia^d}L& z8&R5oiKeW2lxeH<5*J;(in5@aX00rw{uNQumTtPTB1?+fLzydcq!oK0D3kW>p~qGh zO6hOWZ&t2OJ^Qw%+0Y7BY+DU4$>UHR2SyvGo*5`j5Vh~8#~zv` zojX8>K=%JTjrzrp?sUViXEAM7Gs?Yz*KFNoe(AU78v2cS(@t~kvu~Q8d&4}#(`nx9 zHP<}9!~DYQ=Gqs3ZQk;lS-KQ+^79j){q^{di-QEQe_FKCo%PaBI<@Jp3L9R(g~&SLb9H5Fh(KzR^VJh*eAn?xW__38|&19Uj2e z5F$Ln)%C!-JG-7<{jR&qzdDaMhyKJ5{b#>1SJ!3O7WyMT7>Fmtiw_Lqi3<>>SOS&! zAYa-VsEDsG;;Ecp+D9GYHSrP9k!Yei;3eAYc8I4 z+CF?MFQx&s+f3#Wm#Zn*-}gja?0f1s*slw_<@@vi`u|=BFi}Ls|Jwt!TPw%+rmlZz zmpE5EeVn_bs7TMf5-&9Sak&>yi0<`lmw1tR^QZtEqR)By07n;(4#35p6q%Ptw+GpF z&z)eehixc)S9!L}Xru&JO7WZ!TK_!k2R(KJFB+qNtQ6M8M@u}Vw|edc9v$3BrLCS9 z4Cpe*&TI922G+DJpd9*i8|Se{VDzTt0XRh4JKSCY4vo)N|jp&!BUAp6@)*W}WxlYp=cc z+H0@9XNH!#ke0fTns`IYz!eFb!|kz>c!l*G)R8Bk4?sqz~lp!q>AEJnQLo!{n6z&s0IcQ~`yDEb@%f zn@BduI|J|C19C8A50rrcoDDewWq1Iyp?)YmQ5-0lD9I>8Q3j!;q1aLQv53Etlm*!f zg?*GGN!=v_UPhvXp$tG_2MwdB(5Gp6XsRa?h@TXc5ELr|JaH&}PaWV2f&bkH;w(^_VvgKIkm@0^F-BUPFv^k`4rLy}(B4ivX?>I8^43y-IcRz$`{eH;ko4y|1JJTi`& zBEzCS{XoKrm!9-|4`9bryFdc6jNLe^$)e{HtDr(G{qDrx6?e+<2^d!glpQ) z>Giz7vgjgoawy3hM~UVxT`gzzy3P;=ng})ztUZV|(qgcsV4Xp17<~wKJ=pvpHiEta zdj@Pl5Zi^iM#WK=s4iWLg4jsP0h;umLN8fs=+P?+ZM$3qU~U}ft3ROWS?2+ zBG@0nnuFNhloSpBM#H~BY+srPHV>>li0wyC`m_{*8fugIF8ofXxP57Q{MeKG;IAH9_oPdK>IpU~7Zep>zoBL9lf}>~OjP_BvQ! z5Icg>W8vRe_&12nuAdeAP6QRjX0NnFd&X5tl2ZOj*lrFvZRpqA@we{<{52&z@y2*A zSNU5sJbh0w=AGtm61bT$=PSQY-~)^~C;ICI-o=Fk-)tf^AX`M5IBJ`pMd^+fx9tIVC)n)f^kpA zc7Y9y6B%0szP1fGiLv=6`q%pY!bC47B%$yF<7CEdH`oJTGqx~p5%?726vj;gH#6ol z&+ilX0AoG}{dEHGV$8YEUn_6}V?Gc4H8;?|)^{Tle9rsJgu(}m`3(0L3H&bORK^7Y zzrmOfSbx62D;e`K?RN_N5@Ws}`0WC}z?jp&-y-mHjQMc(n>~V9$ONCteo5fR81v=B z-*%k`_%LI>H27Ntet3{l4YZ_%-Qg-l;RL5he-aVB* z(=yJGdEW2UFf`u5n@K&4c`4p=F7F<-9AuvNiuY1=euz2;D_-dJ-Icy)7pf({ zYg&0`XmN|$ijgQ)Gs2YWq34(5)q75jM{5kY;s`8*J{q9q0DVxW4+iKTV3oD^OH|r7 z$un4t>ls^d7A%cLojwjsh260O-q2KD-%6jS)jt+UuZMCEDEo(iarhcJm+1bh*UeXs zT%Db?xU zz6}3WZ?ZP=EMUTTP1Ck;Si`A`_orXq)0d_Zw!rYxEP@6v%{gA0`vSxH5G^pLCiWhz zXi!Hdm9K#oOT=WtbcwrSv=w%p;n? zF54r!vo4mLc3gpXlCIJDeu?rsbhBSi`EE)`?J0jx!%|b^F*GeT$;DqNoK|IANUa0LMxVyeZae8q zOy_-a4}8d61-;fUt4gh5SUz>4l{O4akY~}sfqhIlsA}XRnwQZ2FS(##!lvFbCUZp(eVSZJ--xhaKs&eK4sNopAIve|Gukj(elr~acNdwcf1M^EA@{v55&JO7#e@2F(^W-H|IJBoc zpOy{%N?uz(XV{Cfe1v`)Zk0cv=*(>SZOYGFefMW7)-BYcuiH(L>r53X2rb^k7GBKv zf6(`gsWO7;Dz`P?u^;0^ol>1It64N8YoYudt<9PnwHVe~dnkyIc$^z*q)L_%PsrP; zW<-B^5jBkHZ7%q4j$TYh!JN|7DfHur^_{KXVfMyGdS~P;LznN#kgfFp1yt6a;_e8; zH`Fl~^z2@<&uvZP_~#D9PE<$GqU_=F!}Ry;vGU*Pmuy$}`@hk&D{ku~xg_DX+H4w^ zlPbSS3v&kHL*|;C!}9A?JZgcV_tSJ?R7#}xgzG++_wW|}Nbb6qqDN=T7RnzT@42c@ z4eav6IJlH*?Bhm0fz&%NB;Nf02U;BkfXoWK>=dx6(vQn0I?q(yU;iQfe*=QsCF4X; zk7KAJX+!~uWcD5yu9mpE-+2$Gh}hyHYSqk5%}zrF@Q1(+d?#;>6$iCxbFQqE?~}Q1g^~W@ z>^^WA+QL#ccWxI(778Q9R(k{7CaG&L>a$M@qf5^22`8Y>&iELaQ+VgQ7M)m2$*x$> z+(5M#)h^iW;=Sg*sdfwKEDEq;g57pbZ}|{d^_3qrt>YlN;_&SdeJ2F0&4$Jfz9t4V zME-ZF}o+g{)VN5)v!iLN&RPly$R&-i9=MJ4m(7x9H?yp688j`K9>98W072ytR}@Y zc^Bqg-X8M2RRol)dhCtzcEeL$$7KDBQ|$PwY6I-J%J<4+vTl}~$C)yGK6=~#tLb@T z?J7P`H(3iX_&JJc0rrj2f!an+A9>!Zb4-OhnQsE0HJ=W+d#24q6QIG+;+ht`D|pBL zJHnKya2L}m=&|!DYD{$BRK3C7fd=iU8LeyX?&xyVe3~*Qx<~t8v01- zfOh9Z>CBi!PbnH= zocq`qPAFllNauDkXAW?)b#51P7I13)8BNO>h_|Qj=Uh>PESTp#FMP#&HHL!t{)Tw94e0Pk!^m|c}iHlS>r3%9vgKo+*B~#@xG*y$(4Ud{_i$H6fMFaC9pO9+5Nk zYwT%#2;;n;sHLar>$@MBz&SBh@qXDDkA3a%Vcx#SFq+y^oJEl?6|Y~gyTLYEprclD zRbZe&PQWKF-`K%PdzGr`XjN@|0Sz3@rta`NhtWw;OUaxY(`{czSe8J8>r-i1Zcf7Q zbM=+JAGbfO{wKVuW$c(Ts?N=jQ)ydnpAi4Z8)Q=)(Hn4k#^^HDqAzuCtB+EKGRRX1 zKCt^QshuzBbz;Y=zx{+NY*evGKLW6~bVr@-Kq8Ygbs*ugT><1D3e)&ZQ~?W;^ruD+ zOvU@Vy7440k5b+)KlQBQ$zi(q$+D5QlESjm5Y1NnUk*5HvaDq7Tr@4nTZ;EUYfLaM z-vymri#nBMN83t%R6-++g(`iX!bTe zh^wX>N1IT&1eJwnMY?MJwybodA*19oM|10`+a1ylWwD{=aT+x)+4Sh+Je3M-9>%V2 zJVAxy?sT8R(d#`|dHNZnYC9o(Wv|o#dIKmsf42ncqV>8VqOKEwt6URTMSR>jPRGZM zGWZ^&?&BXcJbr=}jepN@?|JGmp{F79JPn&*8PZ)pt11$1azt&j{*DdXxCHK1uj{8@ z1%Jna+dW@Yy$CDyo>qEkLQ>SwR*}~$_G3+O?;bUeKAw;`avCD2V; zQO98h=Lw$$&+DOM#z#l>HSl~8K1N;hlKSMKW{g8?*o0vqILs5Oi*uyB{@2vC$IuP* zUVfA&=H+;tfJ?SnmmymIiZj==^z?N>EWbq@0KpTALI|lY@yXCs@lgkfAXL)=B!9)h z>8A#5#nd_Dc=WTZg1>xFs*0^)OpZXWJk|x7f;F~2iWZU-fmD@q@R8dy$8@DH_#EHI zvvsk3d`RA>KKKRApEx>p06IbjiuY&2J$c@raEPe)(awo+hEYf9)WpPDW(*H4D=uLj zID14Pu{Br#a4}Q+9MuQW46G+5O?{h&SYH2E_$11!phm*J|05ti(Pakf0B(n} zMA{^$;n3&gn!La;v57WJo+Ni&nQt>3Izo>8vR+$fOVYmF!bFVD6xDf-=?UW&Fs7(n zvqRfGhhE9=9rLEHV!5sI`fM^zSs?eQFP`F*4JQxRZ=O1_vtdeQz43v!#}D}jenZ3O zyhwZt&H=2^2IOo&f&!P|4)EGRb`q6)r!#CE|7^7ZTqG{PES3R2cAWEP5)gOH$ zd)CA15E!nVt7+|cw-;)f1##b2;0sDLEfahZWFBM_c7_tr0 z{w0Uti^_34(WVR)6PH^tkASH z48XDy9YQ)+Y1(?orq>ZA2qp6ynl>FWAF>G2@)kOPv_qR94LxeRH?CB6n9Us@oWAd3*O<_Js}(wd5wI=o~;`XKWln;;7y+aQY} z&25^tj8eArlWS<)7Mrnl4h+)bEq&xLTD>JM*0M;`_^Wiu9cj*kC_+i3Lt7@xX_VB^ zPcEi04FlzqU~(hUP&XUPsV<^j4H>?c!-Z0MZR>bc zoCk#;t+q|JO~*RLVKmZRk>*znxsmZNhm4Q3REORbnY!HQjrBa8d!+ML$;YCnE!VV<@GiL`&E*EeiipUtvHWYnbQBJfgQ$<84cnYaxuKEECgw%f zOq>SkiY%Lm6ab|rC@o(}k=w@_6FElc{_PXe3-MM)nl=gw{9waKW#HBaaRuOB1$UA@ z-##;=9dGWxXd1#uJlvuS#}+uA8b9nI3%KmR(2T|bo_xHXdF?-&%m#N79DiTO51V-g z+y!tZ!RWs(F9+8Gj(;?ja>GNyLe9y%1N!Sww+SIVE1=Idwt+Jvyl39#JW>i|-VV-4WVE_xWBk zZo=y-`q?+K(`CGcU7$fbM|k)8xIllqVi1{fyRaf#t_s18mBn$#N4tc_>-Vrxk}DHArXh zYkl{6^I@M{fAYv&1HR96HG814v)L9E#+w39hs-7Us4XhLtIl@FQgR)&g*OG*CVKYh z@Nj3qZ%*2Bboi}dbPUHy@-Yvx$C6{V@VY*_wL02zY&g{&GaJs9li`buh>n@puB8!Q zEX7xIJHN0++41`VKOM4-)MfjR70!h`#*-bVq+b7@t0Kcy5ruX7gLjl( z#gq=&&v@@<$PCEtD9HxSfgFL7X<&Aggwg|L5K0z`6J;pMV3ZUTHwr(!bYU6}nSjDE z8fh|hGnw&{jS`A75Ty&sFp3O%RQqCWX-# z_#01=!Q-TtXtE`S<^>nh`>;`hOQpXF#vkI4w$Q4OH0dGQ6Ou;uU>CX}4y}K|zW1X40&z_B_gqRK)+>5JT#<~kO9W!vf zz!uXv1J?>1WctRy+XU`p`mcc(#Lf=mK;8!Cr(@yrbc!d%L?f&zRAvqBUS}-V4&Asq z-DC%?v&KaFf2FOUe_CUx#Tpvf{3}fpeF2T#7TUe#SDKC6bdL(TZ81_C-D{iEy<@?! zrtN~R=dCuJj@V)%I;NU=J6#3i_*}OMfiTl;TMQ-JJ4Yl1uogtPr})Si$_Kk5fbBvPz%B;6I)II$ zC15v!-59`jqs?H?f!z_nI_Nyu&Rsf3ydS{EQbd;+$_9HnfQ_RZu*G1N0JbMR1@?8Y z?E$Ql-T-?TY^V`!j%05-0`@1cjsVt0x4|ZNMSKI;ew5M`@dcY1!1kwUU{`{b1K5F7 z19m6asR3*X?E-rlY*7H4MpwYbMIpWctefJa5MQt>0@%Sc0qkP1s{_~}v;^!Xup0x| zVYC_SIj}nd*ep5^wsSP%8^C5$M0AXlQ}2%cTlfpku3lfI$s||o58cO$P8;&sF8obA zhrhOjT)Z(ZVwKvW;YmEsn75kREbxbn`506E0`Fza$D~>(a3fJHCNzej5(L9=>k8;m@g5kQ{ZP9^W{Lbdj;_%6P&ZeuCIA~g?SEUahR67o!@ z2EJFhkGtVnv7-&&zAFEsXOto6Lq%p#NsDp`J&~&)3zchztXqj!-vvd19Xecj6rMqo zK_4^d!#cgvpc@VP0~2K@#(O)7ewC(`WWiG~bb3E96b{F3L_pVbBU9s1WN7e z@b*Ow9*+G5PI}TDoA%nRVh1I=Ng}E|`yL8somA^aGS@ zN?ZlI`HWUd>HEI!QNhBdbV$jGF>b&615BSk&-dMKK7y2Q(VHrNNrSo$BjdSl00F;U zAk?$qlpQpqU#}h;gq^MGF%H=QDKG2Fg0t=$h|c+&w60&A^fWc~>mhZb&+zvhy3wzf z6itq#c=Hc8X;@O4bdY8x^_(-S9lo)67j^YiWV?b&zC=SOz^Shz45DRpB@Ufv0_na{ zaa0qttk^{3-Yb{32F&Ech&0Ej3)h_h3o(;HK=8VR+LC$?YJ?lC;qdasu1pSXi50%b zuy0MV{=kvi=^Bl7)WD}}eofBBYRc=MhWm6${|sqAy^HrJ=uH1a>0{FRr*vIgjkxo& zv-PiZ<>E!@Nuvhzy6>m&^`5*4m1hTUpnRX&$SJ~iLD(yw35x{_?`k<=bZB?#+Nsr+ zLG+ISW4rzXUAr5kGZ@Zn3K}@gejn^EovB{z3dQs(6Z+8nfw59AS~;+f?e=v|YvQ!c zQ`mN$ng))V_)5LWbbV&3Euj*;p8w^VoCdIRjb$r~toD*XZ6}`TMKnOU;!A(}$_OrChp_>WH3o zRm>ZP+49IV&!hYDa6sKpv1!xo`=Dv$&sXjnyd}1gFD*+7r+sN}_Y8u{Q}MG_f@(h~ zH`*B87N}KG+Gu(Dcxf>mOP^!*e@7$TPnoZ_(q{KNQN>Hp>HD#2svmIW%f8PsdSo@G8?PQJm9Ni#?p{4X35 zZ)Wt7s%dY=Y{^5(LwZOlBo8?+JzL**=$|C%FZ92|T+(xNVpz5`jqI80?x|KVZ$VCd z-hLKmovpGLmKJYfCr{?0Z}mN6s|shjI?q+${s8@iPOd4ClrXxU`HZxkW@QycPJp*d z-DRv02l9sU`8U*(^|Z8v3WoPbe4iWM+wS}$uU-sC#+Xu+NZK@fQ^;5U!RReZY5IuA z&F8+Ry(8rQd%xDStOF(8ungBImap{Fz4lO^E1B0nHz0PR5>A7&he?BIPWD)74sFf$ zbnE`5rrpSMO_UbI-Zjghu$&|q49XcKIcP@C$M_VRF>LOs%f~e zqVY^6=;E)A|9TKyuaWC*fD>_OYyuL8S&8BHHCBJq;^iYEI%6KTeJ-!_-@h-{RJ!cA z>fGx)B=bK+((PA9e!)k&Cb|Zwp%J4$Rp<%VNyp`&Q3J^HAw?hLiVo!!R1-MQWmg^s zkg*2sa@BF$^Alituu_iVdYE*MH{Xwx|Fpmg1E@?9j!Vt9?YZVk;CSFh6w{?>k*ICk z$4PRbL zCr*UMJN6iUGWZ76!Y29!ZwJ2)JX%V+tMK>^g7@issA#j%?nU!R=TOXNMZPQ9cMzUK zu|e%&@8^ZJ<%KC>ZMGF5=n<|mV1*kM5 zdvI{k&uwCe6+nT=Y!+OHz_ z)zV}+hTa>KFk4fugXc>lw^fRz;6Y}W%lA)(&pcl{AMu`Pp2s}X_qqMt4Y5G=#2JpB z$+%vL{lno=LXDo?>)Wp!#$Yf}=4e$0z^MmUoH1G7FZkk`ZK);l70loV$eC}<;akEc zS0Q3IQ?|{;X<%6gu89xDd|&HKTk#&|gPWmvcMc7|r$_Q3T;YKJ1~XQ*;C%<)ajc9g z24ySW%d`S|`y492r>nPCuTXDPXoqGC-uaH{$AC~{Uq{;;!1GnakGL?`%)On8`uAcE zR)Lg#IC;=F_$Y(zz2*B?ao26d`&pAIdi@2Ie=r<1o3O0dNgQ_qCVa_(z8HGcI>3~y zQ;!=}TF?yF`0q=4jRL6Mei$F*p0yd;bI^i2#2k{hsL`x*dziBu-2RuatYNs9Nhe69 z$60-fYjITN`_S3wPH5RUdqYL!u?4^Xj1S;1jEVRGR&7s;&3_IGK0cA}&ba;HzXspr zz*(T|F;ouOrwY|qtm^4x-3Hbf3wdVX6-q3zAI8STACRjbbvY25x45C>wr<$^JO`&i zZJ_jVQC$u)Yyo@;7OZ#DZ^z}t_L!{?e?D%D80mf)N_1-Z5);>&woD8%3G%?e|+D(TS&aV3sq-JEsE{5 zXgs}2126-~dwz5px@(^bU53zo4dAKJ#KlZgPJc?<$B#6xokKUqKVp95G~IvSdh>#d zbo0I*=BXE{TfQ@6sGhPbV}Is3+3ETbE1{{J2Nh>%_*)L#rTJRu!=xxBm+8s;_{ay5 zXK>(FesG)DfTib?v^qa7D--eW7za+U>lhA>?IGy@kpbpe)m?rQwt{5bS^!0_E52^tx)%kO&DbA<6rEz(QVgDisy-w-XpF_vx9bK@k zH9T^kN>{05qLC9_sZMkOX~CiDr^(pIEB_7fMD8uuw8bdH$~5gYucrOJe%-&ml)U`A-e5RiTcl}ic+V}?GzYd- z8Dulo*4h$H^MhZzK-1E}*Fok%+81iti);tElkv0otOq#{G82>C2FYI*7ofQvkY$jE znTKqL%tco^2O-C_Ez`8A*a3+LOLMJAd?|$A#)l3Rnr~~L5^v|%RCSTYtay- zA96LMa~&K*)e<5WErFanfo3hf<0c=h=@SuexPY9SmMMx3^@<$j=#-z;wrrn zvhF+7gKUBP39=2+?hJ=xoV6}UJ7gwgI%Ga%0c0WdZ|EnTrpXOymW?wtZ7o$b^y%(= zQq$1Fod8Mx z**Gb>cCn`QVlR1CXA(p^CVCT1`}e?ZZP1Km4IUXK6H}YKgx>u3c&V1ue-D%%py-{G z9$b&f+lsN6MqBNFG3Qww&vzPQb=CxpwI}`YIZ_ zYpmrbyl$lOUH7HrW4S0VpeFYg4~L@=Tom2q3c;1|vZ9;2rl+39+ol&a4eLrgT;nph zjo^4j_(|qJFj)U0EpHm&Nyls6%fGCW2QC*}$M3%EW-7RRaQrg^KddVTC)3rY81D+a z#jVz~$AwPs>g(W2!A%d~>cH&>hYOB)*z^T1_n)wErd&Fm-+!?jIT4 zj@RO&wABBys7o@q=v(3k?rSoJ(j+BrqWH+wsXsPc+mV{haQ=PpMw`2 zNs#8!t4C&;@joX1Y3^3<_}DK=+v|@VecCLoCeLxNq|lz@X^~E>cYZqLM3PRVMb3@T z*$%msJSWn^+6)%%PYeqyGy+#h4JU@(=}1S1rjm5hi(! Date: Thu, 17 Sep 2026 08:03:38 +0200 Subject: [PATCH 10/69] docs: record sibling temp attestation, spill directory pinning and canonical registration paths --- src/runtime/AGENTS.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index a0d4517..3551638 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -168,7 +168,8 @@ in `deny` (verified: `/tmp`, `/var/tmp`, `/run`, `/etc`, `sessions` all fail; `/tmp/sub` works), so the profile cannot hide evaluator temp files. Instead: - the launcher exports `TMPDIR=/tmp` (strict adds TMPDIR to its read-write grants; Python, Node and `mktemp` use it); provision it - `: 0700`; + `: 0700`. Every registered worker's private temp is attested + before any turn, so one misprovisioned sibling refuses all turns; - `/tmp` and `/var/tmp` must be `root: 1774`: Grok needs to open the directory, but without search or write a worker can only list names — `cat`/`read_file` get EACCES and it cannot create files. @@ -177,7 +178,16 @@ in `deny` (verified: `/tmp`, `/var/tmp`, `/run`, `/etc`, `sessions` all fail; - spills (`toolResultSpill.ts`) are written `0640`; provision `/tool-output` as `2000: 2750` (setgid) under a runtime home the worker can traverse, so each spill carries that agent's - worker group and no other worker can read it. + worker group and no other worker can read it. The writer pins the directory + (`O_DIRECTORY|O_NOFOLLOW`, dev/ino re-checked before publishing) and refuses + one that is a symlink, not owned by the runtime, wider than `2750`, or + group-open without setgid or in the runtime's own group; it cannot tell + *which* worker gid belongs to the agent, so that mapping stays the + deployment's. A spill is published by rename, replacing any existing entry + (a planted symlink included) without following it. +- registered workspace and home paths must be canonical (no `.`, `..`, empty + components or trailing slash) in both `service.json` and `registrations.bin`; + the launcher refuses the slot otherwise. `agySubscriptionRealm.ts` owns the one host-level private D-Bus/Secret Service realm, durable keyring lease, bounded unlock stdin, and cleanup. From 92fb1ca5060fb9c11aaf235f1a245278e573111d Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 15:52:07 +0200 Subject: [PATCH 11/69] fix: refuse Grok worker deny paths bubblewrap cannot materialize as the worker uid --- src/pi/grokSandbox.ts | 6 + src/runtime/AGENTS.md | 28 ++++ src/runtime/grokBrokerProjection.test.ts | 18 +++ src/runtime/grokBrokerProjection.ts | 28 +++- src/runtime/grokWorkerAttestation.ts | 14 +- .../grokWorkerAttestationChecks.test.ts | 34 +++- src/runtime/grokWorkerDenyPlacement.test.ts | 118 ++++++++++++++ src/runtime/grokWorkerDenyPlacement.ts | 147 ++++++++++++++++++ src/runtime/grokWorkerSandboxProfile.ts | 10 +- src/runtime/index.ts | 2 + 10 files changed, 397 insertions(+), 8 deletions(-) create mode 100644 src/runtime/grokWorkerDenyPlacement.test.ts create mode 100644 src/runtime/grokWorkerDenyPlacement.ts diff --git a/src/pi/grokSandbox.ts b/src/pi/grokSandbox.ts index ddc9ee4..1c448ed 100644 --- a/src/pi/grokSandbox.ts +++ b/src/pi/grokSandbox.ts @@ -8,6 +8,7 @@ import { readChild } from "./cliChildOutput.js"; import { cliChildEnvironment } from "./cliEnvironment.js"; import { terminateChild, trackCliChild } from "./cliProcess.js"; import { renderGrokSandboxArgs } from "./cliEngineSpawn.js"; +import { assertGrokWorkerDenyPathsPlaceable } from "../runtime/grokWorkerDenyPlacement.js"; import { GROK_WORKER_SANDBOX_EVENTS_RELATIVE_PATH, GROK_WORKER_SANDBOX_PROFILE, renderGrokWorkerSandboxProfile } from "../runtime/grokWorkerSandboxProfile.js"; export const GROK_DAIMON_SANDBOX_PROFILE = GROK_WORKER_SANDBOX_PROFILE; @@ -36,6 +37,11 @@ export async function prepareAndVerifyGrokSandbox( if (denied.some((entry) => overlaps(entry, cwd) || overlaps(entry, engineHome))) { throw unavailable(); } + // Grok 1.0.34 materializes every deny target inside bubblewrap as the uid it + // runs Grok under — here, this process's own — and refuses the whole profile + // if one cannot be resolved. Refusing now names the path; letting it through + // would kill every turn with a bare `bwrap: Can't create file at …`. + await assertGrokWorkerDenyPathsPlaceable(denied, { uid: process.getuid?.() ?? 0, gid: process.getgid?.() ?? 0 }); await writeProfile(engineHome, denied); const beforeBytes = await eventFileSize(path.join(engineHome, SANDBOX_EVENTS)); const child = trackCliChild(spawn(authority.command, [ diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 3551638..d9544a7 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -160,6 +160,34 @@ every profile inside bubblewrap, where a non-empty `deny` list is enforced; `grokWorkerSandboxProfile.ts` renders those profile bytes. A worker-uid process can neither write, rename, nor unlink any of the root-owned files. +Deny-path placement (`grokWorkerDenyPlacement.ts`). Grok 1.0.34 materializes +every `deny` entry inside bubblewrap **as the worker uid**, bind-mounting +`$GROK_HOME/sandbox-blocked-{file,dir}` over the target, so an entry is +placeable only when every ancestor directory is searchable by that uid and the +target already exists and is not a symlink. One unplaceable entry makes Grok +refuse the *whole* profile (`bwrap: Can't create file at …: Permission +denied`), so every turn of that worker fails, not just that path. Matrix: +`.runtime/grok-deny-placement/EVIDENCE.md` in the ecosystem folder. The rule +therefore has two halves: +- shape, decidable without a filesystem and asserted by the renderer: canonical, + and strictly below every base-profile grant (`GROK_WORKER_BASE_PROFILE_GRANTS`); +- placement, asserted by whoever provisions the paths — root provisioning and + every slot recycle on the Spawnfile side, `prepareGrokWorkerAttestation` + before every brokered turn, and `prepareAndVerifyGrokSandbox` on the direct + path, which runs as the worker uid itself. The broker (uid 2100) cannot + descend into a `2000: 0710` runtime home, so an `EACCES` below an + ancestor the worker *can* search is left undecided there; root, which holds + `CAP_DAC_READ_SEARCH`, decides every entry. + +When a protected path is not placeable, the deny entry is **lifted** to the +nearest ancestor that is — never adding `o+x` to a private directory, because a +lift masks a superset and never widens the worker's reach. The durable +wake-acceptance store is exactly that case: it lives under the organization's +`state` directory, which the ownership guard secures `2000:2000 0700`, so the +mask goes on that directory (`acceptanceStoreDenyPath` in +`grokBrokerProjection.ts`, which refuses a mask that does not contain the +store). + Temp and spill isolation (`grokWorkerTmpAttestation.ts`, checked before every turn; `GROK_ENGINE_BROKER.worker.home.{privateTmp,sharedTmp,spillDirectory}`). Grok 1.0.34's strict profile grants shared `/tmp` and `/var/tmp` read-write diff --git a/src/runtime/grokBrokerProjection.test.ts b/src/runtime/grokBrokerProjection.test.ts index 10794bb..592ca7a 100644 --- a/src/runtime/grokBrokerProjection.test.ts +++ b/src/runtime/grokBrokerProjection.test.ts @@ -52,3 +52,21 @@ test("a provisioned registration must describe its projection exactly", () => { assert.throws(() => verifyGrokBrokerRegistrationMatchesProjection(parse({ ...registration, ...drift }), projection), /does not match/u, JSON.stringify(drift)); } }); + +test("the acceptance store mask may be a covering directory, and must actually cover the store", () => { + // Grok 1.0.34 cannot materialize a deny target under a directory the worker cannot search, so a + // deployment that secures `/state` to `2000:2000 0700` masks that directory instead. + const store = "/var/lib/spawnfile/instance/state/wake-acceptance"; + const lifted = resolveOrganizationGrokBrokerProjection(config, "foreman", + { ...options, acceptanceStorePath: store, acceptanceStoreDenyPath: "/var/lib/spawnfile/instance/state" }); + assert.equal(lifted.denyPaths.includes("/var/lib/spawnfile/instance/state"), true); + assert.equal(lifted.denyPaths.includes(store), false); + // Default: the store itself, exactly as before. + assert.equal(resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, acceptanceStorePath: store }).denyPaths.includes(store), true); + for (const acceptanceStoreDenyPath of ["/var/lib/spawnfile/instance/other", "/var/lib/spawnfile/instance/state/wake-acceptance/inner"]) { + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, acceptanceStorePath: store, acceptanceStoreDenyPath }), + /acceptance store deny path must contain the acceptance store/u, acceptanceStoreDenyPath); + } + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, acceptanceStorePath: store, acceptanceStoreDenyPath: "/var" }), + /canonical absolute acceptanceStoreDenyPath|base profile grant/u); +}); diff --git a/src/runtime/grokBrokerProjection.ts b/src/runtime/grokBrokerProjection.ts index 4052e41..34414d4 100644 --- a/src/runtime/grokBrokerProjection.ts +++ b/src/runtime/grokBrokerProjection.ts @@ -55,6 +55,19 @@ export type OrganizationGrokBrokerProjectionOptions = Readonly<{ limits: EngineBrokerTurnLimits; /** The wake-acceptance store, always denied like the Codex projection's. */ acceptanceStorePath: string; + /** + * The deny entry that covers the acceptance store, when the store itself + * cannot be one. + * + * Grok 1.0.34 materializes every deny target inside bubblewrap as the worker + * uid, so a target whose parent directory the worker cannot search is + * unplaceable and makes Grok refuse the whole profile. The durable store sits + * under the organization's private `state` directory (`2000:2000 0700`), so a + * deployment that secures it that way declares the mask on that directory + * instead — strictly stronger, since nothing else lives there. Must contain + * the store; defaults to the store itself. + */ + acceptanceStoreDenyPath?: string; /** Evaluator and host-bind paths the deployment must keep from the worker (R4). */ denyPaths?: readonly string[]; /** sha256 of the seccomp profile bytes the deployment runs the worker under. */ @@ -79,21 +92,26 @@ export type OrganizationGrokBrokerProjectionOptions = Readonly<{ * * The agent must be a Grok agent that *declares* its model and reasoning * effort; nothing is defaulted. The deny list is Daimon's own protected set for - * this agent (realm, bootstrap, peers, acceptance store) plus the caller's - * evaluator paths, sorted and deduplicated exactly as the profile renderer - * does. A supplied `profileSha256` that differs is refused. + * this agent (realm, bootstrap, peers, and the mask covering the acceptance + * store) plus the caller's evaluator paths, sorted and deduplicated exactly as + * the profile renderer does. A supplied `profileSha256` that differs is + * refused. */ export function resolveOrganizationGrokBrokerProjection(config: unknown, agentId: string, options: OrganizationGrokBrokerProjectionOptions): OrganizationGrokBrokerProjection { const parsed = parseOrganizationRuntimeConfig(config); const agent = parsed.agents.find((entry) => entry.id === agentId); if (agent === undefined || agent.engine.kind !== "grok") throw new Error("Grok broker projection requires a known Grok agent"); if (agent.engine.model === undefined || agent.engine.reasoningEffort === undefined) throw new Error("Grok broker projection requires a declared model and reasoning effort"); - for (const [label, value] of [["workerHomePath", options.workerHomePath], ["acceptanceStorePath", options.acceptanceStorePath]] as const) { + const acceptanceStoreDenyPath = options.acceptanceStoreDenyPath ?? options.acceptanceStorePath; + for (const [label, value] of [["workerHomePath", options.workerHomePath], ["acceptanceStorePath", options.acceptanceStorePath], ["acceptanceStoreDenyPath", acceptanceStoreDenyPath]] as const) { if (!path.posix.isAbsolute(value) || path.posix.normalize(value) !== value || value === "/" || value.endsWith("/")) throw new Error(`Grok broker projection requires a canonical absolute ${label}`); } + if (options.acceptanceStorePath !== acceptanceStoreDenyPath && !options.acceptanceStorePath.startsWith(`${acceptanceStoreDenyPath}/`)) { + throw new Error("Grok broker projection acceptance store deny path must contain the acceptance store"); + } if (!/^[a-f0-9]{64}$/u.test(options.seccompProfileSha256)) throw new Error("Grok broker projection requires a seccomp profile sha256"); const model = agent.engine.model as GrokBrokerModel, reasoningEffort = agent.engine.reasoningEffort as GrokBrokerReasoningEffort; - const denyPaths = [...new Set([...grokSandboxProtectedPaths(agent.id, parsed.agents, [options.acceptanceStorePath]), ...(options.denyPaths ?? [])])].sort(); + const denyPaths = [...new Set([...grokSandboxProtectedPaths(agent.id, parsed.agents, [acceptanceStoreDenyPath]), ...(options.denyPaths ?? [])])].sort(); renderGrokWorkerSandboxProfile(denyPaths); const profileSha256 = grokWorkerSandboxProfileSha256(denyPaths); if (options.profileSha256 !== undefined && options.profileSha256 !== profileSha256) throw new Error("Grok broker projection profile digest mismatch"); diff --git a/src/runtime/grokWorkerAttestation.ts b/src/runtime/grokWorkerAttestation.ts index 60bc8fc..c441411 100644 --- a/src/runtime/grokWorkerAttestation.ts +++ b/src/runtime/grokWorkerAttestation.ts @@ -3,6 +3,7 @@ import { constants } from "node:fs"; import { lstat,open } from "node:fs/promises"; import path from "node:path"; +import { assertGrokWorkerDenyPathsPlaceable } from "./grokWorkerDenyPlacement.js"; import { verifyGrokWorkerHome } from "./grokWorkerHomeAttestation.js"; import { grokWorkerHomeForProfile, verifyGrokWorkerTmp, type GrokWorkerTmpOptions, type GrokWorkerTmpWorker } from "./grokWorkerTmpAttestation.js"; import { grokWorkerEventsPathFor } from "./grokWorkerSandboxProfile.js"; @@ -61,12 +62,23 @@ export function parseGrokWorkerSandboxProfile(bytes:Uint8Array,profileSha256:str * `$GROK_HOME/sessions/` (the root `sandbox-events.jsonl` stays empty on * 1.0.34), and a root-owned read-only worker home whose `config.toml` hashes to * the declared renderer output (`grokWorkerHomeAttestation.ts`). + * + * It also refuses a deny list bubblewrap could not materialize + * (`grokWorkerDenyPlacement.ts`), naming the entry and the ancestor that stops + * it. Without this the worker dies with a bare `bwrap: Can't create file at + * …: Permission denied` on *every* turn, because one unplaceable entry makes + * Grok refuse the whole profile. The broker cannot descend into a directory + * opened to the worker's group alone (`/tool-state` under a + * `2000: 0710` home), so an `EACCES` there is left undecided; root + * provisioning, which holds `CAP_DAC_READ_SEARCH`, is the authority that + * decides every entry. */ -export async function prepareGrokWorkerAttestation(input:Readonly<{profilePath:string;eventsPath:string;profileSha256:string;workerUid:number;brokerGid:number;configSha256:string;registeredWorkers?:readonly Readonly<{profilePath:string;workerUid:number}>[]}>,profileOwner:Readonly<{uid:number;gid:number;tmp?:GrokWorkerTmpOptions}>={uid:0,gid:0}):Promise{ +export async function prepareGrokWorkerAttestation(input:Readonly<{profilePath:string;eventsPath:string;profileSha256:string;workerUid:number;workerGid?:number;brokerGid:number;configSha256:string;registeredWorkers?:readonly Readonly<{profilePath:string;workerUid:number}>[]}>,profileOwner:Readonly<{uid:number;gid:number;tmp?:GrokWorkerTmpOptions}>={uid:0,gid:0}):Promise{ if(input.eventsPath!==grokWorkerEventsPathFor(input.profilePath))throw new Error("Grok worker sandbox events must be read from $GROK_HOME/sessions/sandbox-events.jsonl"); const profile=await secureOpen(input.profilePath,profileOwner.uid,profileOwner.gid,0o444,65_536);let bytes:Buffer|undefined;let denyPaths:readonly string[]=[];try{bytes=await profile.readFile();denyPaths=parseGrokWorkerSandboxProfile(bytes,input.profileSha256);}catch{throw new Error("Grok worker isolation attestation unavailable");}finally{bytes?.fill(0);await profile.close();} // The launcher exports TMPDIR=/tmp; the profile lives at /.grok/sandbox.toml. // Every registered worker's private temp is attested, not only this one's (a sibling's open temp is a shared channel). + await assertGrokWorkerDenyPathsPlaceable(denyPaths,{uid:input.workerUid,gid:input.workerGid??input.workerUid}); const workers=[{profilePath:input.profilePath,workerUid:input.workerUid},...(input.registeredWorkers??[])].map((worker)=>({home:grokWorkerHomeForProfile(worker.profilePath),uid:worker.workerUid})); if(workers.some((worker)=>worker.home===undefined))throw new Error("Grok worker isolation attestation unavailable"); await verifyGrokWorkerTmp(workers as readonly GrokWorkerTmpWorker[],profileOwner.tmp); diff --git a/src/runtime/grokWorkerAttestationChecks.test.ts b/src/runtime/grokWorkerAttestationChecks.test.ts index 8bee20e..45480ed 100644 --- a/src/runtime/grokWorkerAttestationChecks.test.ts +++ b/src/runtime/grokWorkerAttestationChecks.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; -import { appendFile, chmod, link, mkdir, mkdtemp, open, rm, stat, writeFile } from "node:fs/promises"; +import { appendFile, chmod, link, mkdir, mkdtemp, open, realpath, rm, stat, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import test, { mock } from "node:test"; @@ -122,3 +122,35 @@ test("prepare refuses the current turn when a sibling registered worker's privat await chmod(path.join(sibling.home, "tmp"), 0o777); await assert.rejects(prepareGrokWorkerAttestation({ ...input, brokerGid: self.gid }, seams), /temp isolation attestation unavailable/u); }); + +test("prepare refuses a deny entry bubblewrap could not materialize, before any other leg", async (t) => { + // The production defect: the wake-acceptance store sits under a `0700` organization state directory, + // so bubblewrap — which materializes every deny target as the worker uid — could not create it and + // Grok refused the whole profile, failing every turn with `bwrap: Can't create file at …`. + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "daimon-guard-deny-"))); + t.after(async () => { await chmod(path.join(root, "state"), 0o700); await rm(root, { recursive: true, force: true }); }); + const grokHome = path.join(root, ".grok"); + await mkdir(path.join(grokHome, "sessions"), { recursive: true }); + await mkdir(path.join(root, "tmp"), { mode: 0o700 }); + await mkdir(path.join(root, "state", "wake-acceptance"), { recursive: true }); + const profile = path.join(grokHome, "sandbox.toml"); + const events = path.join(grokHome, "sessions", "sandbox-events.jsonl"); + await writeFile(events, ""); await chmod(events, 0o640); + const owner = { uid: self.uid, gid: Number((await stat(path.join(root, "tmp"))).gid) }; + const withDeny = async (denied: string) => { + const text = `[profiles.daimon-strict]\nextends = "strict"\nrestrict_network = true\ndeny = ["${denied}"]\n`; + await chmod(profile, 0o644).catch(() => undefined); + await writeFile(profile, text); await chmod(profile, 0o444); + return { profilePath: profile, eventsPath: events, profileSha256: createHash("sha256").update(text).digest("hex"), workerUid: self.uid, brokerGid: self.gid, configSha256: "0".repeat(64) }; + }; + await chmod(path.join(root, "state"), 0o600); + await assert.rejects( + prepareGrokWorkerAttestation(await withDeny(path.join(root, "state", "wake-acceptance")), owner), + (error: Error) => /is not placeable/u.test(error.message) && error.message.includes(`cannot search ${path.join(root, "state")}`) + ); + // The lift: the private directory itself is placeable, so this leg passes and a later one refuses. + await assert.rejects( + prepareGrokWorkerAttestation(await withDeny(path.join(root, "state")), owner), + (error: Error) => !/is not placeable/u.test(error.message) + ); +}); diff --git a/src/runtime/grokWorkerDenyPlacement.test.ts b/src/runtime/grokWorkerDenyPlacement.test.ts new file mode 100644 index 0000000..f5013a0 --- /dev/null +++ b/src/runtime/grokWorkerDenyPlacement.test.ts @@ -0,0 +1,118 @@ +import assert from "node:assert/strict"; +import { chmodSync, mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { + assertGrokWorkerDenyPathPlacement, + assertGrokWorkerDenyPathShape, + assertGrokWorkerDenyPathsPlaceable, + GROK_WORKER_BASE_PROFILE_GRANTS, + grokWorkerCanSearch, + grokWorkerDenyPathChain, + GrokWorkerDenyPlacementError, + type GrokWorkerDenyPathEntry, + type GrokWorkerDenyPathStep +} from "./grokWorkerDenyPlacement.js"; +import { renderGrokWorkerSandboxProfile } from "./grokWorkerSandboxProfile.js"; + +const entry = (uid: number, gid: number, mode: number, kind: "dir" | "file" | "link" = "dir"): GrokWorkerDenyPathEntry => + ({ uid, gid, mode, isDirectory: () => kind === "dir", isSymbolicLink: () => kind === "link" }); +const worker = { uid: 2200, gid: 2200 }; +const steps = (denyPath: string, entries: readonly (GrokWorkerDenyPathEntry | string)[]): GrokWorkerDenyPathStep[] => + grokWorkerDenyPathChain(denyPath).map((target, index) => { + const value = entries[index]; + return typeof value === "string" ? { path: target, code: value } : { path: target, entry: value }; + }); + +test("search permission follows owner, then group, then other — as the worker's cleared-group process does", () => { + assert.equal(grokWorkerCanSearch(entry(2200, 2200, 0o700), worker), true); + assert.equal(grokWorkerCanSearch(entry(2200, 2200, 0o677), worker), false, "owner bits win even when group and other would allow"); + assert.equal(grokWorkerCanSearch(entry(2000, 2200, 0o710), worker), true); + assert.equal(grokWorkerCanSearch(entry(2000, 2000, 0o700), worker), false); + assert.equal(grokWorkerCanSearch(entry(0, 0, 0o711), worker), true); + assert.equal(grokWorkerCanSearch(entry(0, 0, 0o755), worker), true); + assert.equal(grokWorkerCanSearch(entry(0, 2000, 0o750), worker), false); +}); + +test("refuses a deny entry at or above any Grok 1.0.34 base-profile grant", () => { + for (const grant of GROK_WORKER_BASE_PROFILE_GRANTS) { + assert.throws(() => assertGrokWorkerDenyPathShape(grant), GrokWorkerDenyPlacementError, grant); + } + assert.throws(() => assertGrokWorkerDenyPathShape("/var"), /base profile grant \/var/u); + assert.throws(() => renderGrokWorkerSandboxProfile(["/run"]), /base profile grant \/run/u); + assert.throws(() => renderGrokWorkerSandboxProfile(["/var/lib/spawnfile/daimon/usage", "/tmp"]), /base profile grant \/tmp/u); + // Strictly below every grant is exactly what Grok accepts. + assertGrokWorkerDenyPathShape("/tmp/sub"); + assertGrokWorkerDenyPathShape("/var/lib/spawnfile/daimon/usage"); +}); + +test("refuses the wake-acceptance shape: a deny entry under a parent the worker cannot search", () => { + // `/state` is `2000:2000 0700`; the store beneath it is what production used to deny. + const denyPath = "/var/lib/spawnfile/instances/daimon/org/state/wake-acceptance"; + assert.throws( + () => assertGrokWorkerDenyPathPlacement(denyPath, steps(denyPath, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o711), entry(0, 0, 0o711), + entry(0, 0, 0o711), entry(2000, 2000, 0o711), entry(2000, 2000, 0o700), entry(2000, 2000, 0o700) + ]), worker), + (error: Error) => error instanceof GrokWorkerDenyPlacementError + && /cannot search \/var\/lib\/spawnfile\/instances\/daimon\/org\/state \(700 2000:2000\); deny that directory itself instead/u.test(error.message) + ); + // The lift production now emits: the private directory itself, whose own parent is traversable. + const lifted = "/var/lib/spawnfile/instances/daimon/org/state"; + assertGrokWorkerDenyPathPlacement(lifted, steps(lifted, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o711), entry(0, 0, 0o711), + entry(0, 0, 0o711), entry(2000, 2000, 0o711), entry(2000, 2000, 0o700) + ]), worker); +}); + +test("refuses a missing target, a symlink and a non-directory ancestor; leaves an undecidable EACCES alone", () => { + const denyPath = "/run/training/slot/state"; + assert.throws(() => assertGrokWorkerDenyPathPlacement(denyPath, steps(denyPath, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), "ENOENT" + ]), worker), /does not exist; bubblewrap would have to create it as the worker uid/u); + assert.throws(() => assertGrokWorkerDenyPathPlacement(denyPath, steps(denyPath, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o777, "link") + ]), worker), /is a symlink; bubblewrap refuses to bind over one/u); + assert.throws(() => assertGrokWorkerDenyPathPlacement(denyPath, steps(denyPath, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o644, "file"), entry(0, 0, 0o755), entry(0, 0, 0o700) + ]), worker), /is not a directory/u); + assert.throws(() => assertGrokWorkerDenyPathPlacement(denyPath, steps(denyPath, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), "EPERM" + ]), worker), /could not be read \(EPERM\)/u); + // The broker (uid 2100) cannot descend into a `2000: 0710` runtime home the worker itself can + // search, so an EACCES *below a worker-searchable ancestor* is undecided here, never a refusal. + assertGrokWorkerDenyPathPlacement(denyPath, steps(denyPath, [ + entry(0, 0, 0o755), entry(0, 0, 0o755), entry(0, 0, 0o755), entry(2000, 2200, 0o710), "EACCES" + ]), worker); +}); + +test("walks the real filesystem and names the ancestor that stops a deny entry", async () => { + // realpath: macOS `/var` is a symlink, and a symlinked ancestor is refused on purpose — Daimon's + // registered paths are canonical, and bubblewrap must bind over the inode the deny entry names. + const root = realpathSync(mkdtempSync(path.join(tmpdir(), "grok-deny-"))); + const state = path.join(root, "state"); + // This process stands in for root provisioning: it can stat every component, and judges searchability + // for the worker from the modes it reads. Here the worker is this uid, so only `state` blocks it. + const self = { uid: process.getuid?.() ?? 0, gid: process.getgid?.() ?? 0 }; + try { + mkdirSync(path.join(state, "wake-acceptance"), { recursive: true }); + writeFileSync(path.join(state, "wake-acceptance", "store.jsonl"), "{}\n"); + chmodSync(state, 0o600); + await assert.rejects( + assertGrokWorkerDenyPathsPlaceable([path.join(state, "wake-acceptance")], self), + (error: Error) => error instanceof GrokWorkerDenyPlacementError + && error.message.includes(`cannot search ${state}`) && error.message.includes("deny that directory itself instead") + ); + // The lift: the unsearchable directory itself is placeable, and masks strictly more. + await assertGrokWorkerDenyPathsPlaceable([state], self); + await assert.rejects(assertGrokWorkerDenyPathsPlaceable([path.join(root, "absent")], self), /does not exist/u); + symlinkSync(state, path.join(root, "link")); + await assert.rejects(assertGrokWorkerDenyPathsPlaceable([path.join(root, "link")], self), /is a symlink/u); + await assert.rejects(assertGrokWorkerDenyPathsPlaceable(["relative/path"], self), /not an absolute path/u); + } finally { + chmodSync(state, 0o700); + rmSync(root, { force: true, recursive: true }); + } +}); diff --git a/src/runtime/grokWorkerDenyPlacement.ts b/src/runtime/grokWorkerDenyPlacement.ts new file mode 100644 index 0000000..f2995d2 --- /dev/null +++ b/src/runtime/grokWorkerDenyPlacement.ts @@ -0,0 +1,147 @@ +import { lstat } from "node:fs/promises"; +import path from "node:path"; + +/** + * Paths Grok 1.0.34's strict base profile grants read or read-write. A `deny` + * entry that equals or contains one of them makes Grok refuse the profile + * outright (verified for `/tmp`, `/var/tmp`, `/run`, `/etc`, `/var` and + * `sessions`; `/tmp/sub` is accepted), so every entry must sit strictly below + * each grant it touches. + */ +export const GROK_WORKER_BASE_PROFILE_GRANTS = Object.freeze([ + "/bin", "/dev", "/etc", "/lib", "/proc", "/run", "/sbin", "/sys", "/tmp", "/usr", "/var", "/var/tmp" +] as const); + +const within = (candidate: string, root: string): boolean => candidate === root || candidate.startsWith(`${root}/`); + +export class GrokWorkerDenyPlacementError extends Error { + constructor(readonly denyPath: string, readonly reason: string) { + super(`Grok worker sandbox deny path ${JSON.stringify(denyPath)} is not placeable: ${reason}`); + this.name = "GrokWorkerDenyPlacementError"; + } +} + +/** + * The shape half of the deny-placement policy: everything decidable without + * touching a filesystem, so the profile renderer can refuse a bad entry before + * its bytes are ever pinned or written. + * + * `renderGrokWorkerSandboxProfile` already refuses entries that are relative, + * non-canonical, `/`, trailing-slashed, or carrying a character TOML or Grok + * would reinterpret; this adds the base-profile grant rule. + */ +export function assertGrokWorkerDenyPathShape(denyPath: string): void { + const grant = GROK_WORKER_BASE_PROFILE_GRANTS.find((candidate) => within(candidate, denyPath)); + if (grant !== undefined) { + throw new GrokWorkerDenyPlacementError(denyPath, `it equals or contains the base profile grant ${grant}, which Grok refuses`); + } +} + +/** The subset of `lstat` the placement rules read; pure so every refusal is testable without root. */ +export type GrokWorkerDenyPathEntry = Readonly<{ + uid: number; + gid: number; + mode: number; + isDirectory: () => boolean; + isSymbolicLink: () => boolean; +}>; + +/** One resolved path component: the entry, or the errno that stopped the walk. */ +export type GrokWorkerDenyPathStep = Readonly<{ path: string; entry?: GrokWorkerDenyPathEntry; code?: string }>; + +export type GrokWorkerDenyPathWorker = Readonly<{ uid: number; gid: number }>; + +/** + * POSIX search permission: owner bits win, then group, then other. The worker + * runs with its supplementary groups cleared, so its primary gid is the only + * group that can apply. + */ +export const grokWorkerCanSearch = (entry: GrokWorkerDenyPathEntry, worker: GrokWorkerDenyPathWorker): boolean => + entry.uid === worker.uid ? (entry.mode & 0o100) !== 0 + : entry.gid === worker.gid ? (entry.mode & 0o010) !== 0 + : (entry.mode & 0o001) !== 0; + +/** The `/`-rooted ancestor chain of `denyPath`, deepest last, followed by the entry itself. */ +export const grokWorkerDenyPathChain = (denyPath: string): readonly string[] => { + const components = denyPath.split("/").slice(1); + return ["/", ...components.map((_, index) => `/${components.slice(0, index + 1).join("/")}`)]; +}; + +/** + * The placement half of the policy, as a pure function of an already-walked + * chain. + * + * Grok 1.0.34 materializes every `deny` entry inside bubblewrap **as the worker + * uid**, by bind-mounting `$GROK_HOME/sandbox-blocked-{file,dir}` over the + * target. So bwrap must be able to *resolve* the target as that uid: every + * ancestor directory needs the search bit for it, and the target must already + * exist — otherwise bwrap tries to create it and needs write on the parent, + * which a private parent never grants. A single unplaceable entry makes Grok + * refuse the whole profile, so every turn of that worker fails, not just that + * path. Verified matrix: `.runtime/grok-deny-placement/EVIDENCE.md`. + * + * The walk may legitimately stop early: a caller that is neither root nor the + * worker (the broker, uid 2100) cannot descend into a directory the worker's + * own group opens to it alone — `/tool-state` under a + * `2000: 0710` runtime home is exactly that. An `EACCES` below an + * ancestor the *worker* can search is therefore "not decidable from here", not + * a refusal; every decidable failure still refuses. + */ +export function assertGrokWorkerDenyPathPlacement( + denyPath: string, + steps: readonly GrokWorkerDenyPathStep[], + worker: GrokWorkerDenyPathWorker +): void { + assertGrokWorkerDenyPathShape(denyPath); + const chain = grokWorkerDenyPathChain(denyPath); + if (steps.length !== chain.length || steps.some((step, index) => step.path !== chain[index])) { + throw new GrokWorkerDenyPlacementError(denyPath, "its resolved path chain does not match the entry"); + } + for (const [index, step] of steps.entries()) { + const ancestor = index < steps.length - 1; + if (step.entry === undefined) { + if (step.code === "ENOENT") throw new GrokWorkerDenyPlacementError(denyPath, `${step.path} does not exist; bubblewrap would have to create it as the worker uid`); + if (step.code !== "EACCES") throw new GrokWorkerDenyPlacementError(denyPath, `${step.path} could not be read (${step.code ?? "unknown error"})`); + // Undecidable from here, and only after every shallower ancestor passed. + return; + } + if (step.entry.isSymbolicLink()) throw new GrokWorkerDenyPlacementError(denyPath, `${step.path} is a symlink; bubblewrap refuses to bind over one`); + if (!ancestor) return; + if (!step.entry.isDirectory()) throw new GrokWorkerDenyPlacementError(denyPath, `${step.path} is not a directory`); + if (!grokWorkerCanSearch(step.entry, worker)) { + throw new GrokWorkerDenyPlacementError(denyPath, `worker uid ${worker.uid} cannot search ${step.path} (${(step.entry.mode & 0o7777).toString(8)} ${step.entry.uid}:${step.entry.gid}); deny that directory itself instead`); + } + } +} + +/** Walks one deny path on the real filesystem, recording what stopped it rather than throwing. */ +export async function readGrokWorkerDenyPathChain(denyPath: string): Promise { + const steps: GrokWorkerDenyPathStep[] = []; + for (const target of grokWorkerDenyPathChain(denyPath)) { + try { + steps.push({ path: target, entry: await lstat(target) }); + } catch (error) { + steps.push({ path: target, code: (error as NodeJS.ErrnoException).code }); + break; + } + } + const chain = grokWorkerDenyPathChain(denyPath); + while (steps.length < chain.length) steps.push({ path: chain[steps.length]!, code: steps.at(-1)?.code ?? "EACCES" }); + return steps; +} + +/** + * Fails closed before a worker is ever launched with a profile Grok would + * refuse. Callers with the widest view run it: root provisioning at container + * start and on every slot recycle, and the direct (non-broker) path, which runs + * as the worker uid itself. + */ +export async function assertGrokWorkerDenyPathsPlaceable( + denyPaths: readonly string[], + worker: GrokWorkerDenyPathWorker +): Promise { + for (const denyPath of denyPaths) { + if (!path.posix.isAbsolute(denyPath)) throw new GrokWorkerDenyPlacementError(denyPath, "it is not an absolute path"); + assertGrokWorkerDenyPathPlacement(denyPath, await readGrokWorkerDenyPathChain(denyPath), worker); + } +} diff --git a/src/runtime/grokWorkerSandboxProfile.ts b/src/runtime/grokWorkerSandboxProfile.ts index b6ff126..881ffb1 100644 --- a/src/runtime/grokWorkerSandboxProfile.ts +++ b/src/runtime/grokWorkerSandboxProfile.ts @@ -1,6 +1,8 @@ import { createHash } from "node:crypto"; import path from "node:path"; +import { assertGrokWorkerDenyPathShape } from "./grokWorkerDenyPlacement.js"; + export const GROK_WORKER_SANDBOX_PROFILE = "daimon-strict" as const; export const GROK_WORKER_SANDBOX_EVENTS_RELATIVE_PATH = "sessions/sandbox-events.jsonl" as const; @@ -17,7 +19,12 @@ export const GROK_WORKER_SANDBOX_EVENTS_RELATIVE_PATH = "sessions/sandbox-events * * Entries are sorted and deduplicated so equal sets render equal bytes (and the * same `profileSha256`). A path that is not absolute and canonical, or that - * carries a character TOML or Grok would reinterpret, is refused. + * carries a character TOML or Grok would reinterpret, is refused — and so is + * one that equals or contains a base-profile grant, the first half of the + * deny-placement policy (`grokWorkerDenyPlacement.ts`). The other half — + * the entry exists and every ancestor is searchable by the worker uid — needs + * a filesystem, so it is asserted by whoever provisions the paths and, on the + * direct path, before every turn. */ export function renderGrokWorkerSandboxProfile(denyPaths: readonly string[] = []): string { const denied = [...new Set(denyPaths)].sort(); @@ -25,6 +32,7 @@ export function renderGrokWorkerSandboxProfile(denyPaths: readonly string[] = [] if (!path.posix.isAbsolute(entry) || path.posix.normalize(entry) !== entry || entry === "/" || entry.endsWith("/") || /["\\\u0000-\u001f\u007f*?[\]]/u.test(entry)) { throw new TypeError("invalid Grok worker sandbox deny path"); } + assertGrokWorkerDenyPathShape(entry); } return [ `[profiles.${GROK_WORKER_SANDBOX_PROFILE}]`, diff --git a/src/runtime/index.ts b/src/runtime/index.ts index 24195c5..b3f2ed1 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -14,6 +14,8 @@ export { GROK_INFERENCE_CLIENT_MODEL_ID, GROK_INFERENCE_GRANT_ENV, grokInference export { GROK_INFERENCE_PROXY_BASE_URL, ENGINE_BROKER_INFERENCE_FAILURE_CODES, type EngineBrokerInferenceFailureCode } from "./engineBrokerInferenceProtocol.js"; export { dedupeInferenceUsageRows, INFERENCE_USAGE_LEDGER_VERSION, GROK_INFERENCE_PURPOSES, type GrokInferencePurpose } from "./inferenceUsageLedger.js"; export { GROK_INFERENCE_AUTH_STALE_BODY } from "./grokInferenceProxy.js"; +export { assertGrokWorkerDenyPathPlacement, assertGrokWorkerDenyPathShape, assertGrokWorkerDenyPathsPlaceable, GROK_WORKER_BASE_PROFILE_GRANTS, grokWorkerCanSearch, grokWorkerDenyPathChain, GrokWorkerDenyPlacementError, readGrokWorkerDenyPathChain } from "./grokWorkerDenyPlacement.js"; +export type { GrokWorkerDenyPathEntry, GrokWorkerDenyPathStep, GrokWorkerDenyPathWorker } from "./grokWorkerDenyPlacement.js"; export { grokWorkerSandboxProfileSha256, renderGrokWorkerSandboxProfile } from "./grokWorkerSandboxProfile.js"; export { engineBrokerRequestLedgerPathFor, parseEngineBrokerServiceConfig, type EngineBrokerServiceConfig, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; export { DEFAULT_GROK_BROKER_TURN_LIMITS, ENGINE_BROKER_LIMIT_REASONS, type EngineBrokerLimitReason, type EngineBrokerTurnAccounting, From 8c7e309faf836837118b6d2619480e0676cde9ff Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 16:02:47 +0200 Subject: [PATCH 12/69] fix: let the Grok broker projection mask the wake-acceptance store through a covering directory --- src/runtime/grokBrokerProjection.test.ts | 25 +++++++++--------- src/runtime/grokBrokerProjection.ts | 33 +++++++++++------------- 2 files changed, 27 insertions(+), 31 deletions(-) diff --git a/src/runtime/grokBrokerProjection.test.ts b/src/runtime/grokBrokerProjection.test.ts index 592ca7a..960e917 100644 --- a/src/runtime/grokBrokerProjection.test.ts +++ b/src/runtime/grokBrokerProjection.test.ts @@ -53,20 +53,19 @@ test("a provisioned registration must describe its projection exactly", () => { } }); -test("the acceptance store mask may be a covering directory, and must actually cover the store", () => { +test("the acceptance store mask may be the directory that covers the store", () => { // Grok 1.0.34 cannot materialize a deny target under a directory the worker cannot search, so a - // deployment that secures `/state` to `2000:2000 0700` masks that directory instead. + // deployment that secures `/state` to `2000:2000 0700` declares that directory here. const store = "/var/lib/spawnfile/instance/state/wake-acceptance"; - const lifted = resolveOrganizationGrokBrokerProjection(config, "foreman", - { ...options, acceptanceStorePath: store, acceptanceStoreDenyPath: "/var/lib/spawnfile/instance/state" }); - assert.equal(lifted.denyPaths.includes("/var/lib/spawnfile/instance/state"), true); + const state = "/var/lib/spawnfile/instance/state"; + const lifted = resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, acceptanceStorePath: state }); + assert.equal(lifted.denyPaths.includes(state), true); assert.equal(lifted.denyPaths.includes(store), false); - // Default: the store itself, exactly as before. - assert.equal(resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, acceptanceStorePath: store }).denyPaths.includes(store), true); - for (const acceptanceStoreDenyPath of ["/var/lib/spawnfile/instance/other", "/var/lib/spawnfile/instance/state/wake-acceptance/inner"]) { - assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, acceptanceStorePath: store, acceptanceStoreDenyPath }), - /acceptance store deny path must contain the acceptance store/u, acceptanceStoreDenyPath); - } - assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, acceptanceStorePath: store, acceptanceStoreDenyPath: "/var" }), - /canonical absolute acceptanceStoreDenyPath|base profile grant/u); + // The store itself still works where its parent is traversable, and produces a different digest — + // a recomputation that disagrees with the deployment fails closed rather than certifying the slot. + const leaf = resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, acceptanceStorePath: store }); + assert.equal(leaf.denyPaths.includes(store), true); + assert.notEqual(grokBrokerProjectionSha256(leaf), grokBrokerProjectionSha256(lifted)); + assert.throws(() => resolveOrganizationGrokBrokerProjection(config, "foreman", { ...options, acceptanceStorePath: "/var" }), + /base profile grant/u); }); diff --git a/src/runtime/grokBrokerProjection.ts b/src/runtime/grokBrokerProjection.ts index 34414d4..298b433 100644 --- a/src/runtime/grokBrokerProjection.ts +++ b/src/runtime/grokBrokerProjection.ts @@ -53,21 +53,22 @@ export type OrganizationGrokBrokerProjectionOptions = Readonly<{ architecture: "arm64" | "x64"; usageLedgerPath: string; limits: EngineBrokerTurnLimits; - /** The wake-acceptance store, always denied like the Codex projection's. */ - acceptanceStorePath: string; /** - * The deny entry that covers the acceptance store, when the store itself - * cannot be one. + * The deny entry that protects the durable wake-acceptance store: the store + * itself, or a directory containing it. * - * Grok 1.0.34 materializes every deny target inside bubblewrap as the worker - * uid, so a target whose parent directory the worker cannot search is - * unplaceable and makes Grok refuse the whole profile. The durable store sits - * under the organization's private `state` directory (`2000:2000 0700`), so a - * deployment that secures it that way declares the mask on that directory - * instead — strictly stronger, since nothing else lives there. Must contain - * the store; defaults to the store itself. + * Grok 1.0.34 materializes every deny target inside bubblewrap **as the + * worker uid**, so a target whose parent directory the worker cannot search + * is unplaceable and makes Grok refuse the whole profile — every turn of that + * worker then fails, not just that path. Deployments that keep the store + * under a private `state` directory (`2000:2000 0700`) therefore declare that + * directory here: it covers the store, nothing else lives there, and lifting + * the mask adds the worker no reach, where opening the parent with `o+x` + * would. The caller is the one party that knows both the layout and the + * modes; whoever recomputes this projection must pass the same value or the + * digests will not match, which is the intended fail-closed outcome. */ - acceptanceStoreDenyPath?: string; + acceptanceStorePath: string; /** Evaluator and host-bind paths the deployment must keep from the worker (R4). */ denyPaths?: readonly string[]; /** sha256 of the seccomp profile bytes the deployment runs the worker under. */ @@ -102,16 +103,12 @@ export function resolveOrganizationGrokBrokerProjection(config: unknown, agentId const agent = parsed.agents.find((entry) => entry.id === agentId); if (agent === undefined || agent.engine.kind !== "grok") throw new Error("Grok broker projection requires a known Grok agent"); if (agent.engine.model === undefined || agent.engine.reasoningEffort === undefined) throw new Error("Grok broker projection requires a declared model and reasoning effort"); - const acceptanceStoreDenyPath = options.acceptanceStoreDenyPath ?? options.acceptanceStorePath; - for (const [label, value] of [["workerHomePath", options.workerHomePath], ["acceptanceStorePath", options.acceptanceStorePath], ["acceptanceStoreDenyPath", acceptanceStoreDenyPath]] as const) { + for (const [label, value] of [["workerHomePath", options.workerHomePath], ["acceptanceStorePath", options.acceptanceStorePath]] as const) { if (!path.posix.isAbsolute(value) || path.posix.normalize(value) !== value || value === "/" || value.endsWith("/")) throw new Error(`Grok broker projection requires a canonical absolute ${label}`); } - if (options.acceptanceStorePath !== acceptanceStoreDenyPath && !options.acceptanceStorePath.startsWith(`${acceptanceStoreDenyPath}/`)) { - throw new Error("Grok broker projection acceptance store deny path must contain the acceptance store"); - } if (!/^[a-f0-9]{64}$/u.test(options.seccompProfileSha256)) throw new Error("Grok broker projection requires a seccomp profile sha256"); const model = agent.engine.model as GrokBrokerModel, reasoningEffort = agent.engine.reasoningEffort as GrokBrokerReasoningEffort; - const denyPaths = [...new Set([...grokSandboxProtectedPaths(agent.id, parsed.agents, [acceptanceStoreDenyPath]), ...(options.denyPaths ?? [])])].sort(); + const denyPaths = [...new Set([...grokSandboxProtectedPaths(agent.id, parsed.agents, [options.acceptanceStorePath]), ...(options.denyPaths ?? [])])].sort(); renderGrokWorkerSandboxProfile(denyPaths); const profileSha256 = grokWorkerSandboxProfileSha256(denyPaths); if (options.profileSha256 !== undefined && options.profileSha256 !== profileSha256) throw new Error("Grok broker projection profile digest mismatch"); From 089c3f86b99cbe17a2025e9d6be5b77ea71cfadb Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 20:06:44 +0200 Subject: [PATCH 13/69] fix: accept the contracted traverse-only runtime home for brokered Grok agents --- src/contracts/runtimeContractManifest.ts | 4 ++ src/runtime/physicalReadiness.test.ts | 67 +++++++++++++++++++++++- src/runtime/physicalReadiness.ts | 60 +++++++++++++++++---- 3 files changed, 119 insertions(+), 12 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 46a3588..95a6673 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -71,6 +71,10 @@ export const GROK_ENGINE_BROKER = { // denied, so the deployment keeps them from every worker by mode: root-owned, a non-worker // group (< 2200), others read-only (Grok needs to open the directory) and no search/write. sharedTmp: { paths: ["/tmp", "/var/tmp"], uid: 0, maxGroupExclusive: 2_200, otherMode: 0o4, mode: 0o1774 }, + // The organization runtime home of a brokered Grok agent: traverse-only for the + // worker group so the worker can reach `tool-output/` and nothing else (no group + // read, no group write, no world bits; `physicalReadiness.ts` refuses anything else). + organizationRuntimeHome: { owner: "organization", group: "worker", mode: 0o710 }, // Spilled tool output the worker reads with read_file: setgid directory in the worker's group, // files written 0640 by the runtime, never other-readable. spillDirectory: { relativeToRuntimeHome: "tool-output", owner: "organization", group: "worker", mode: 0o2750, fileMode: 0o640 } diff --git a/src/runtime/physicalReadiness.test.ts b/src/runtime/physicalReadiness.test.ts index bdbb248..b645cce 100644 --- a/src/runtime/physicalReadiness.test.ts +++ b/src/runtime/physicalReadiness.test.ts @@ -4,7 +4,7 @@ import os from "node:os"; import path from "node:path"; import test from "node:test"; -import { prepareOrganizationRuntimePaths } from "./physicalReadiness.js"; +import { assertRuntimeDirectory, prepareOrganizationRuntimePaths } from "./physicalReadiness.js"; import type { OrganizationRuntimeAgentConfig } from "./organizationRuntime.js"; const agent = (workspacePath: string, runtimeHomePath: string): OrganizationRuntimeAgentConfig => ({ @@ -46,3 +46,68 @@ test("preflight requires safe workspace and private runtime roots, and proves ph await assert.rejects(prepareOrganizationRuntimePaths([agent(workspace, workspace)]), /overlap/); } finally { await rm(root, { recursive: true, force: true }); } }); + +const withRoots = async (body: (root: string) => Promise): Promise => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-physical-")); + try { await body(root); } finally { await rm(root, { force: true, recursive: true }); } +}; + +const runtime = { uid: 2000, gid: 2000, firstWorkerUid: 2200 }; +const entry = (mode: number, uid = 2000, gid = 2000, kind: "dir" | "link" = "dir") => ({ + uid, gid, mode: (kind === "dir" ? 0o040000 : 0o120000) | mode, + isDirectory: () => kind === "dir", isSymbolicLink: () => kind === "link" +}); + +test("a brokered Grok runtime home is accepted at exactly 2000: 0710 and nothing wider", () => { + assert.doesNotThrow(() => assertRuntimeDirectory(entry(0o710, 2000, 2200), "runtimeHomePath", "worker-traversable", runtime)); + assert.doesNotThrow(() => assertRuntimeDirectory(entry(0o710, 2000, 2201), "runtimeHomePath", "worker-traversable", runtime)); + const refusals: Record> = { + "0700 (no worker traversal, pre-P1b layout)": entry(0o700, 2000, 2200), + "0711 (world traverse)": entry(0o711, 2000, 2200), + "0712": entry(0o712, 2000, 2200), + "0714": entry(0o714, 2000, 2200), + "0730 (group write)": entry(0o730, 2000, 2200), + "0750 (group read)": entry(0o750, 2000, 2200), + "0770": entry(0o770, 2000, 2200), + "0777": entry(0o777, 2000, 2200), + "2710 (setgid)": entry(0o2710, 2000, 2200), + "owned by a worker": entry(0o710, 2200, 2200), + "owned by root": entry(0o710, 0, 2200), + "group is the runtime's own": entry(0o710, 2000, 2000), + "group below the worker range": entry(0o710, 2000, 2100), + "a symlink": entry(0o710, 2000, 2200, "link") + }; + for (const [label, candidate] of Object.entries(refusals)) { + assert.throws(() => assertRuntimeDirectory(candidate, "runtimeHomePath", "worker-traversable", runtime), /runtimeHomePath/u, label); + } +}); + +test("every other engine's runtime home stays exactly 0700, and a workspace stays group/other-write free", () => { + assert.doesNotThrow(() => assertRuntimeDirectory(entry(0o700), "runtimeHomePath", "private", runtime)); + for (const mode of [0o710, 0o701, 0o750, 0o770, 0o711, 0o755, 0o2700]) { + assert.throws(() => assertRuntimeDirectory(entry(mode, 2000, 2200), "runtimeHomePath", "private", runtime), /must have mode 0700/u, mode.toString(8)); + } + // The brokered Grok workspace contract (2000: 0750) passes the workspace shape. + assert.doesNotThrow(() => assertRuntimeDirectory(entry(0o750, 2000, 2200), "workspacePath", "safe", runtime)); + assert.doesNotThrow(() => assertRuntimeDirectory(entry(0o700), "workspacePath", "safe", runtime)); + for (const mode of [0o770, 0o720, 0o702, 0o777]) { + assert.throws(() => assertRuntimeDirectory(entry(mode, 2000, 2200), "workspacePath", "safe", runtime), /must not grant group or other write/u, mode.toString(8)); + } +}); + +test("the engine kind decides the runtime home shape on a real filesystem", async () => { + await withRoots(async (root) => { + const workspace = path.join(root, "workspace"), home = path.join(root, "home"); + await mkdir(workspace, { mode: 0o700 }); + await mkdir(home, { mode: 0o710 }); + const grok = { ...agent(workspace, home), engine: { kind: "grok" as const, model: "grok-4.6" as const, reasoningEffort: "low" as const } }; + // 0710 reaches the Grok branch: only the worker-group requirement is left to refuse it here. + await assert.rejects(prepareOrganizationRuntimePaths([grok]), /group-owned by the agent's Grok worker group/u); + // The same home refuses a Codex agent for being wider than 0700. + await assert.rejects(prepareOrganizationRuntimePaths([agent(workspace, home)]), /must have mode 0700/u); + await chmod(home, 0o700); + const authority = await prepareOrganizationRuntimePaths([agent(workspace, home)]); + await authority.close(); + await assert.rejects(prepareOrganizationRuntimePaths([grok]), /must have mode 0710 for a brokered Grok agent/u); + }); +}); diff --git a/src/runtime/physicalReadiness.ts b/src/runtime/physicalReadiness.ts index 075ea47..50ce64a 100644 --- a/src/runtime/physicalReadiness.ts +++ b/src/runtime/physicalReadiness.ts @@ -2,9 +2,26 @@ import { constants, type Stats } from "node:fs"; import { lstat, open, realpath } from "node:fs/promises"; import path from "node:path"; +import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; import type { OrganizationRuntimeAgentConfig } from "./organizationRuntime.js"; type Identity = Readonly<{ dev: number; ino: number; uid: number; mode: number }>; +/** + * `private` is every engine's runtime home: 0700, nothing but the runtime user. + * + * `worker-traversable` is the brokered Grok shape, and only that shape: the + * agent's own sandboxed worker runs as another uid and must be able to *walk + * into* this home to read the setgid `tool-output/` spill directory the + * truncation notice sends it to (`GROK_ENGINE_BROKER.worker.home.organizationRuntimeHome`). + * Traverse-only means `0710`: no group read (the worker cannot list the home or + * see the acceptance store, telemetry, memory or credential names) and no group + * write. Anything wider — `0711`, `0750`, `0770`, any world bit — is refused, + * as is a group that is not a worker group. Daimon cannot tell *which* worker + * gid belongs to this agent; the per-slot mapping is the deployment's + * provisioning contract, re-checked by the slot preflight receipt's worker-uid + * canaries. + */ +type DirectoryShape = "safe" | "private" | "worker-traversable"; type Directory = { readonly configured: string; readonly real: string; readonly fd: Awaited>; readonly identity: Identity; closed: boolean }; /** @@ -20,6 +37,11 @@ export type OrganizationRuntimePathAuthority = Readonly<{ close(): Promise; }>; +/** Only a brokered Grok agent's home is worker-traversable; every other engine keeps 0700. */ +function runtimeHomeShape(agent: OrganizationRuntimeAgentConfig): DirectoryShape { + return agent.engine.kind === "grok" ? "worker-traversable" : "private"; +} + export async function prepareOrganizationRuntimePaths( agents: readonly OrganizationRuntimeAgentConfig[] ): Promise { @@ -28,7 +50,7 @@ export async function prepareOrganizationRuntimePaths( try { for (const agent of agents) { workspaces.set(agent.id, await verifyDirectory(agent.workspacePath, "workspacePath", "safe")); - homes.set(agent.id, await verifyDirectory(agent.runtimeHomePath, "runtimeHomePath", "private")); + homes.set(agent.id, await verifyDirectory(agent.runtimeHomePath, "runtimeHomePath", runtimeHomeShape(agent))); } const roots = [...workspaces.values(), ...homes.values()]; for (let left = 0; left < roots.length; left += 1) for (let right = left + 1; right < roots.length; right += 1) { @@ -51,7 +73,7 @@ export async function prepareOrganizationRuntimePaths( if (workspace === undefined || home === undefined) throw new Error(`no runtime path authority for ${agent.id}`); await Promise.all([ verifyIdentity(workspace, "workspacePath", "safe"), - verifyIdentity(home, "runtimeHomePath", "private") + verifyIdentity(home, "runtimeHomePath", runtimeHomeShape(agent)) ]); }; return { @@ -70,10 +92,10 @@ export async function prepareOrganizationRuntimePaths( }; } -async function verifyDirectory(configured: string, label: string, mode: "safe" | "private"): Promise { +async function verifyDirectory(configured: string, label: string, shape: DirectoryShape): Promise { await assertNoSymlinkComponents(configured); const before = await lstat(configured); - assertDirectory(before, label, mode); + assertDirectory(before, label, shape); const fd = await open(configured, constants.O_RDONLY | directoryFlag() | noFollow()); try { const opened = await fd.stat(); @@ -89,7 +111,7 @@ async function verifyDirectory(configured: string, label: string, mode: "safe" | } } -async function verifyIdentity(directory: Directory, label: string, mode: "safe" | "private"): Promise { +async function verifyIdentity(directory: Directory, label: string, shape: DirectoryShape): Promise { if (directory.closed) throw new Error(`${label} authority is closed`); await assertNoSymlinkComponents(directory.configured); const entry = await lstat(directory.configured); @@ -97,7 +119,7 @@ async function verifyIdentity(directory: Directory, label: string, mode: "safe" if (!sameIdentity(identity(entry), directory.identity) || !sameIdentity(identity(opened), directory.identity)) { throw new Error(`${label} changed after readiness validation`); } - assertDirectory(entry, label, mode); + assertDirectory(entry, label, shape); if (await realpath(directory.configured) !== directory.real) throw new Error(`${label} changed after readiness validation`); } @@ -112,12 +134,28 @@ async function assertNoSymlinkComponents(target: string): Promise { } } -function assertDirectory(entry: Stats, label: string, mode: "safe" | "private"): void { +/** Identity of the process Daimon runs as; a seam so every refusal is testable unprivileged. */ +export type RuntimeIdentity = Readonly<{ uid: number; gid: number; firstWorkerUid?: number }>; +type DirectoryEntry = Readonly<{ uid: number; gid: number; mode: number; isDirectory(): boolean; isSymbolicLink(): boolean }>; + +/** Pure shape check for a caller-prepared runtime root. */ +export function assertRuntimeDirectory(entry: DirectoryEntry, label: string, shape: DirectoryShape, runtime: RuntimeIdentity): void { if (!entry.isDirectory() || entry.isSymbolicLink()) throw new Error(`${label} must be an existing real directory`); - if (entry.uid !== process.getuid?.()) throw new Error(`${label} must be owned by the runtime user`); - const permissions = entry.mode & 0o777; - if (mode === "private" && permissions !== 0o700) throw new Error(`${label} must have mode 0700`); - if (mode === "safe" && (permissions & 0o022) !== 0) throw new Error(`${label} must not grant group or other write access`); + if (entry.uid !== runtime.uid) throw new Error(`${label} must be owned by the runtime user`); + const permissions = Number(entry.mode) & 0o7777; + if (shape === "private" && permissions !== 0o700) throw new Error(`${label} must have mode 0700`); + if (shape === "worker-traversable") { + const home = GROK_ENGINE_BROKER.worker.home.organizationRuntimeHome; + if (permissions !== home.mode) throw new Error(`${label} must have mode 0710 for a brokered Grok agent`); + if (entry.gid < (runtime.firstWorkerUid ?? GROK_ENGINE_BROKER.identities.firstWorkerUid) || entry.gid === runtime.gid) { + throw new Error(`${label} must be group-owned by the agent's Grok worker group`); + } + } + if (shape === "safe" && (permissions & 0o022) !== 0) throw new Error(`${label} must not grant group or other write access`); +} + +function assertDirectory(entry: Stats, label: string, shape: DirectoryShape): void { + assertRuntimeDirectory(entry, label, shape, { uid: process.getuid?.() ?? -1, gid: process.getgid?.() ?? -1 }); } function identity(entry: Stats): Identity { return { dev: entry.dev, ino: entry.ino, uid: entry.uid, mode: entry.mode & 0o7777 }; } From 277ea2686fb94ab68f9bf2e17abd8a59b8ea339b Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 20:09:05 +0200 Subject: [PATCH 14/69] fix: create every runtime-home directory private so a traversable Grok home exposes only tool-output --- src/observability/causalEvents.ts | 7 ++- src/observability/orgObserver.ts | 3 +- src/pi/cliSession.ts | 3 +- src/pi/piHarness.ts | 5 +- src/pi/turnTrace.ts | 3 +- src/pi/worldTrajectory.ts | 3 +- src/runtime/runtimeHomeLayout.test.ts | 90 +++++++++++++++++++++++++++ src/runtime/runtimeHomeLayout.ts | 13 ++++ 8 files changed, 118 insertions(+), 9 deletions(-) create mode 100644 src/runtime/runtimeHomeLayout.test.ts create mode 100644 src/runtime/runtimeHomeLayout.ts diff --git a/src/observability/causalEvents.ts b/src/observability/causalEvents.ts index 044b4ca..9a29dd5 100644 --- a/src/observability/causalEvents.ts +++ b/src/observability/causalEvents.ts @@ -2,6 +2,7 @@ import { createHash, randomUUID } from "node:crypto"; import { constants } from "node:fs"; import { appendFile, mkdir, open, readFile, rename, stat, unlink } from "node:fs/promises"; import path from "node:path"; +import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "../runtime/runtimeHomeLayout.js"; /** * Daimon's own copy of the `noopolis.causal-event.v1` wire envelope. Field- @@ -107,7 +108,7 @@ const readSeqStore = async (runtimeHomePath: string): Promise => const writeSeqStore = async (runtimeHomePath: string, store: CausalSeqStore): Promise => { const directory = telemetryDir(runtimeHomePath); - await mkdir(directory, { recursive: true }); + await mkdir(directory, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); const file = seqFilePath(runtimeHomePath); const temporary = `${file}.${randomUUID()}.tmp`; const handle = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); @@ -182,7 +183,7 @@ export const nextCausalSeq = async (input: { const lockPath = path.resolve(telemetryDir(input.runtimeHomePath), "causal.seq.lock"); const previous = seqAllocationQueues.get(lockPath) ?? Promise.resolve(); const allocation = previous.catch(() => undefined).then(async () => { - await mkdir(telemetryDir(input.runtimeHomePath), { recursive: true }); + await mkdir(telemetryDir(input.runtimeHomePath), { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); await acquireSeqLock(lockPath); try { const store = await readSeqStore(input.runtimeHomePath); @@ -207,7 +208,7 @@ export const nextCausalSeq = async (input: { /** Appends one CausalEvent record as a line of `runtimeHome/telemetry/causal.jsonl`. */ export const appendCausalEvent = async (runtimeHomePath: string, event: CausalEvent): Promise => { - await mkdir(telemetryDir(runtimeHomePath), { recursive: true }); + await mkdir(telemetryDir(runtimeHomePath), { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); await appendFile(jsonlFilePath(runtimeHomePath), `${JSON.stringify(event)}\n`, "utf8"); }; diff --git a/src/observability/orgObserver.ts b/src/observability/orgObserver.ts index c6dbd7f..2d21ef8 100644 --- a/src/observability/orgObserver.ts +++ b/src/observability/orgObserver.ts @@ -2,6 +2,7 @@ import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; import type { MemoryRecallAudit } from "@noopolis/mneme"; +import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "../runtime/runtimeHomeLayout.js"; export interface WakeBenchRow { agent: string; @@ -207,7 +208,7 @@ export class OrgObserver { async write(runtimeRoot: string): Promise { const telemetryDir = path.join(runtimeRoot, "telemetry"); - await mkdir(telemetryDir, { recursive: true }); + await mkdir(telemetryDir, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); const summaryRecord = { assertions: this.assertions, behavior: this.behaviorSummary(), diff --git a/src/pi/cliSession.ts b/src/pi/cliSession.ts index 49f8af8..ec4de9e 100644 --- a/src/pi/cliSession.ts +++ b/src/pi/cliSession.ts @@ -35,6 +35,7 @@ import { decodeGrokHeadlessResult } from "./grokHeadlessResult.js"; import { terminateChild, trackCliChild } from "./cliProcess.js"; import type { PiSessionLike } from "./piAgentHandle.js"; import type { PiSessionFactoryInput } from "./piHarness.js"; +import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "../runtime/runtimeHomeLayout.js"; export type CliEngineKind = "agy" | "codex" | "grok"; @@ -145,7 +146,7 @@ export const prepareCliRuntimeHome = async (runtimeHomePath: string | undefined) `${runtimeHomePath}/.local/state`, `${runtimeHomePath}/.cache`, `${runtimeHomePath}/.tmp` - ].map((directory) => mkdir(directory, { recursive: true }))); + ].map((directory) => mkdir(directory, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }))); }; const childSecretValues = (redactedNames: readonly string[]): readonly string[] => diff --git a/src/pi/piHarness.ts b/src/pi/piHarness.ts index 19a1ace..69096d9 100644 --- a/src/pi/piHarness.ts +++ b/src/pi/piHarness.ts @@ -23,6 +23,7 @@ import { type PiWakeEnvironmentContextRef } from "./piAgentWakeSupport.js"; import { DAIMON_WAKE_ID_ENV } from "./cliEnvironment.js"; import { createPiWorldTools, piWorldToolNames, type PiWorldBinding } from "./worldTools.js"; import type { PiWorldToolContextRef } from "./worldNudge.js"; +import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "../runtime/runtimeHomeLayout.js"; import { bindPiRawTrainingCapture, validatePiRawTrainingCaptureOptions, @@ -105,10 +106,10 @@ export class PiHarnessAdapter implements AgentHarnessAdapter { `${input.runtimeHomePath}/.cache`, `${input.runtimeHomePath}/.tmp`, `${input.runtimeHomePath}/tool-state` - ].map((directory) => mkdir(directory, { recursive: true }))); + ].map((directory) => mkdir(directory, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }))); await mkdir(input.workspacePath, { recursive: true }); const memoryRuntimeHomePath = this.options.memory?.runtimeHomePath ?? input.runtimeHomePath; - await mkdir(memoryRuntimeHomePath, { recursive: true }); + await mkdir(memoryRuntimeHomePath, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); const modelSpec = this.options.model ?? { auth: { method: "codex" as const }, provider: "openai", diff --git a/src/pi/turnTrace.ts b/src/pi/turnTrace.ts index 0f0f45c..5ac198b 100644 --- a/src/pi/turnTrace.ts +++ b/src/pi/turnTrace.ts @@ -6,6 +6,7 @@ import type { MemoryPrepareTurnResult, MemoryWakeMode } from "@noopolis/mneme"; import type { HarnessModelSpec, WakeEvent } from "../core/types.js"; import { redactCredentialText } from "../core/credentialRedaction.js"; +import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "../runtime/runtimeHomeLayout.js"; export interface PiTurnTraceModel { authMethod: NonNullable["method"]; @@ -269,7 +270,7 @@ export const writeTurnTraceRecord = async ( ): Promise => { const telemetryPath = path.join(runtimeHomePath, "telemetry"); const turnsPath = path.join(telemetryPath, "turns"); - await mkdir(turnsPath, { recursive: true }); + await mkdir(turnsPath, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); const body = `${JSON.stringify(record, null, 2)}\n`; await writeFile(path.join(turnsPath, `${sanitizeTraceFileId(record.turn_id)}.json`), body, "utf8"); await appendFile(path.join(telemetryPath, "turns.ndjson"), `${JSON.stringify(record)}\n`, "utf8"); diff --git a/src/pi/worldTrajectory.ts b/src/pi/worldTrajectory.ts index 93d59df..6883ed0 100644 --- a/src/pi/worldTrajectory.ts +++ b/src/pi/worldTrajectory.ts @@ -5,6 +5,7 @@ import path from "node:path"; import type { PiTurnTraceModel } from "./turnTrace.js"; import { redactTraceText, sanitizeTraceFileId } from "./turnTrace.js"; import type { PiWorldTurnContext } from "./worldNudge.js"; +import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "../runtime/runtimeHomeLayout.js"; export const WORLD_TRAJECTORY_SCHEMA = "daimon.world_trajectory.v1" as const; @@ -189,7 +190,7 @@ export const persistPiWorldTrajectory = async ( }; const telemetryPath = path.join(input.runtimeHomePath, "telemetry"); const trajectoriesPath = path.join(telemetryPath, "world-trajectories"); - await mkdir(trajectoriesPath, { recursive: true }); + await mkdir(trajectoriesPath, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); const bytes = `${JSON.stringify(record, null, 2)}\n`; await writeFile( path.join(trajectoriesPath, `${sanitizeTraceFileId(input.turnId)}.json`), diff --git a/src/runtime/runtimeHomeLayout.test.ts b/src/runtime/runtimeHomeLayout.test.ts new file mode 100644 index 0000000..6c4363d --- /dev/null +++ b/src/runtime/runtimeHomeLayout.test.ts @@ -0,0 +1,90 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, readdir, rm, stat } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { appendCausalEvent, CAUSAL_EVENT_VERSION, nextCausalSeq } from "../observability/causalEvents.js"; +import { summarizePrompt, writeTurnTraceRecord } from "../pi/turnTrace.js"; +import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "./runtimeHomeLayout.js"; + +/** + * A brokered Grok runtime home is traversable by its worker uid (0710), so + * anything Daimon creates inside it must stay 0700 — a default `mkdir` would + * make telemetry (prompts, replies, trajectories) readable by the model's own + * sandboxed worker. + */ +const withTraversableHome = async (body: (home: string) => Promise): Promise => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-home-layout-")); + const home = path.join(root, "runtime-home"); + await mkdir(home, { mode: 0o710 }); + try { await body(home); } finally { await rm(root, { force: true, recursive: true }); } +}; +const mode = async (target: string): Promise => (await stat(target)).mode & 0o7777; + +test("the runtime-home subdirectory mode grants nobody but the runtime user", () => { + assert.equal(RUNTIME_HOME_SUBDIRECTORY_MODE, 0o700); +}); + +test("telemetry directories Daimon creates in a traversable runtime home are private", async () => { + await withTraversableHome(async (home) => { + await appendCausalEvent(home, { + version: CAUSAL_EVENT_VERSION, id: "daimon:t1:turn.output.completed", type: "turn.output.completed", + occurred_at: "2026-01-01T00:00:00.000Z", actor: { kind: "agent", id: "a" }, subject: { kind: "turn", id: "t1" }, + causes: [], run_id: "run", seq: 1, payload: {} + } as never); + assert.equal(await mode(path.join(home, "telemetry")), RUNTIME_HOME_SUBDIRECTORY_MODE); + await rm(path.join(home, "telemetry"), { recursive: true }); + + await nextCausalSeq({ runtimeHomePath: home, agentId: "a", turnId: "t1", count: 1 } as never); + assert.equal(await mode(path.join(home, "telemetry")), RUNTIME_HOME_SUBDIRECTORY_MODE); + + await writeTurnTraceRecord(home, { + agent_id: "mapper", completed_at: "2026-01-01T00:00:01.000Z", + engine: { auth_method: "none", kind: "pi", model: "llama3.2", provider: "local" }, + memory: { enabled: false }, prompt: summarizePrompt("hi"), reply: { output_chars: 2, reply_given: true }, + schema: "daimon.turn_trace.v1", session: { dispose_after_wake: false, mode: "awake", thread_id: "t" }, + started_at: "2026-01-01T00:00:00.000Z", status: "completed", timings_ms: { total: 1 }, tools: [], + turn_id: "turn-1", wake: { event_id: "w", kind: "message" } + }); + assert.equal(await mode(path.join(home, "telemetry", "turns")), RUNTIME_HOME_SUBDIRECTORY_MODE); + }); +}); + +// Creations that are not inside an agent's runtime home. +const OUTSIDE_RUNTIME_HOME = [ + "src/runtime/native/copyArtifact.ts", "src/observability/emitCausalFixture.ts", "src/pi/auth.ts", "src/runtime/cli.ts" +]; + +const mkdirCalls = (source: string): string[] => { + const calls: string[] = []; + for (let index = source.indexOf("mkdir("); index !== -1; index = source.indexOf("mkdir(", index + 1)) { + let depth = 0; + for (let cursor = index + "mkdir".length; cursor < source.length; cursor += 1) { + if (source[cursor] === "(") depth += 1; + else if (source[cursor] === ")") { depth -= 1; if (depth === 0) { calls.push(source.slice(index, cursor + 1)); break; } } + } + } + return calls; +}; + +test("no runtime-home directory is created without an explicit private mode", async () => { + // Source policy: a default `mkdir` under an agent's runtime home would be 0755, + // and the home of a brokered Grok agent is traversable by its worker uid. + const offenders: string[] = []; + const walk = async (directory: string): Promise => { + for (const entry of await readdir(directory, { withFileTypes: true })) { + const target = path.join(directory, entry.name); + if (entry.isDirectory()) { if (entry.name !== "fixtures" && entry.name !== "artifacts") await walk(target); continue; } + if (!entry.name.endsWith(".ts") || entry.name.endsWith(".test.ts") || OUTSIDE_RUNTIME_HOME.includes(target)) continue; + const source = await readFile(target, "utf8"); + for (const call of mkdirCalls(source)) { + // The workspace is a caller-prepared root with its own contract (group-readable for Grok). + if (call.includes("mode:") || call.includes("{ mode }") || call.includes("workspacePath")) continue; + offenders.push(`${target}: ${call.replace(/\s+/gu, " ").slice(0, 90)}`); + } + } + }; + await Promise.all(["src/pi", "src/observability", "src/runtime", "src/mcp", "src/core"].map(walk)); + assert.deepEqual(offenders, []); +}); diff --git a/src/runtime/runtimeHomeLayout.ts b/src/runtime/runtimeHomeLayout.ts new file mode 100644 index 0000000..f8a280d --- /dev/null +++ b/src/runtime/runtimeHomeLayout.ts @@ -0,0 +1,13 @@ +/** + * Mode for every directory Daimon creates inside an agent's runtime home. + * + * A brokered Grok agent's runtime home is `0710` + * (`GROK_ENGINE_BROKER.worker.home.organizationRuntimeHome`) so its sandboxed + * worker can traverse into the setgid `tool-output/` spill directory. Traverse + * is all it may have: anything Daimon creates in that home — telemetry traces + * (prompts, replies, world trajectories), tool state, receipts, the engine's + * XDG directories and the private `.tmp` — stays `0700`, so a default + * `mkdir` (0755 under the usual umask) never turns a traversable home into a + * readable one. + */ +export const RUNTIME_HOME_SUBDIRECTORY_MODE = 0o700; From bcd2f8ec6856226eb9f23b61b9dc3c140dafb818 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 20:09:54 +0200 Subject: [PATCH 15/69] docs: state the traverse-only Grok runtime home rule and the private subdirectory rule --- src/runtime/AGENTS.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index d9544a7..4d83329 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -203,6 +203,18 @@ in `deny` (verified: `/tmp`, `/var/tmp`, `/run`, `/etc`, `sessions` all fail; only list names — `cat`/`read_file` get EACCES and it cannot create files. `1770`/`1771` make Grok refuse the profile; `1775`/`1777` leak. Any non-root process outside that group that needs temp space must get its own `TMPDIR`; +- the organization runtime home of a brokered Grok agent is `2000: + 0710` — traverse-only, so the worker can reach `tool-output/` and nothing + else. `physicalReadiness.ts` accepts exactly that shape for a `grok` agent + (owner the runtime user, mode `0710`, group a worker group that is not the + runtime's own) and keeps the plain `0700` rule for every other engine; wider + (`0711`, `0730`, `0750`, `0770`, any world bit, setgid) is refused, and so is + a `0700` home for a Grok agent, because its worker could not read its own + spills. Everything Daimon creates inside a runtime home is `0700` + (`runtimeHomeLayout.ts`: telemetry, turn traces, world trajectories, + `tool-state`, the engine XDG directories, `.tmp`), so a traversable home + still exposes nothing but `tool-output/`. A deployment-provisioned memory + home under that runtime home must stay `0700` for the same reason; - spills (`toolResultSpill.ts`) are written `0640`; provision `/tool-output` as `2000: 2750` (setgid) under a runtime home the worker can traverse, so each spill carries that agent's From d485784ec7df45d9e595ea4e40938316956a3c8f Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 21:08:42 +0200 Subject: [PATCH 16/69] fix: name the mounted tools in the agent identity envelope --- src/runtime/engineDispatcher.ts | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/runtime/engineDispatcher.ts b/src/runtime/engineDispatcher.ts index 4666aea..a5864a5 100644 --- a/src/runtime/engineDispatcher.ts +++ b/src/runtime/engineDispatcher.ts @@ -56,7 +56,8 @@ export async function startOrganizationRuntimeEngine( readablePaths: codexSandboxReadablePaths(canonicalAgent) } : undefined; - const adapter = adapterFor(canonicalAgent, controlTokenEnv, readiness.verify, readiness.executablePath, readiness.engineHomePath, paths?.verify, agyBusAddress, [...await createProductionAgentTools(canonicalAgent, wakeContext), ...(agent.attention !== undefined && attention !== undefined ? attentionTools(agent.id, attention) : [])], wakeContext, grokSandbox,grokBroker,codexSandboxPaths); + const mountedTools = [...await createProductionAgentTools(canonicalAgent, wakeContext), ...(agent.attention !== undefined && attention !== undefined ? attentionTools(agent.id, attention) : [])]; + const adapter = adapterFor(canonicalAgent, controlTokenEnv, readiness.verify, readiness.executablePath, readiness.engineHomePath, paths?.verify, agyBusAddress, mountedTools, wakeContext, grokSandbox,grokBroker,codexSandboxPaths, mountedTools.map((tool) => tool.name)); const handle = await adapter.startAgent({ id: canonicalAgent.id, name: canonicalAgent.name, @@ -118,16 +119,16 @@ export function codexSandboxReadablePaths( return [path.join(currentAgent.runtimeHomePath, "tool-output")]; } -function adapterFor(agent: OrganizationRuntimeAgentConfig, controlTokenEnv: string, verifyExecutable: () => Promise, executablePath: string, engineHomePath: string, verifyRuntimePaths?: () => Promise, agyBusAddress?: string, productionTools: readonly import("@earendil-works/pi-coding-agent").ToolDefinition[] = [], wakeEnvironmentContext: import("../pi/piAgentWakeSupport.js").PiWakeEnvironmentContextRef = {}, verifyGrokSandbox?: () => Promise,grokBroker?:EngineBrokerTurnClient,codexSandboxPaths?: { readonly protectedPaths: readonly string[]; readonly readablePaths: readonly string[] }): PiHarnessAdapter { +function adapterFor(agent: OrganizationRuntimeAgentConfig, controlTokenEnv: string, verifyExecutable: () => Promise, executablePath: string, engineHomePath: string, verifyRuntimePaths?: () => Promise, agyBusAddress?: string, productionTools: readonly import("@earendil-works/pi-coding-agent").ToolDefinition[] = [], wakeEnvironmentContext: import("../pi/piAgentWakeSupport.js").PiWakeEnvironmentContextRef = {}, verifyGrokSandbox?: () => Promise,grokBroker?:EngineBrokerTurnClient,codexSandboxPaths?: { readonly protectedPaths: readonly string[]; readonly readablePaths: readonly string[] }, mountedToolNames: readonly string[] = []): PiHarnessAdapter { const engine = agent.engine.kind; const sessionFactory = createCliSessionFactory( engine === "agy" - ? { engine, maxToolTurns: AGY_MAX_TOOL_TURNS, timeoutMs: 180_000, dbusSessionBusAddress: agyBusAddress, redactedEnvironmentNames: [controlTokenEnv], identityPrompt: identityEnvelope(agent), command: executablePath, engineHomePath, verifyExecutable, verifyRuntimePaths, + ? { engine, maxToolTurns: AGY_MAX_TOOL_TURNS, timeoutMs: 180_000, dbusSessionBusAddress: agyBusAddress, redactedEnvironmentNames: [controlTokenEnv], identityPrompt: identityEnvelope(agent, mountedToolNames), command: executablePath, engineHomePath, verifyExecutable, verifyRuntimePaths, // AGY has no broker to meter it, so the session hands its decoded // terminal-frame usage straight to the same ledger the Grok broker // appends to. `recordTurnUsage` is advisory and never rejects. onTurnUsage: (usage, outcome) => recordTurnUsage(resolveTurnUsageLedgerPath(), { agent: agent.id, wake: wakeEnvironmentContext.current ?? "wake", engine: "agy", usage, outcome }) } - : { engine, redactedEnvironmentNames: [controlTokenEnv], identityPrompt: identityEnvelope(agent), command: executablePath, engineHomePath, verifyExecutable, verifyRuntimePaths, + : { engine, redactedEnvironmentNames: [controlTokenEnv], identityPrompt: identityEnvelope(agent, mountedToolNames), command: executablePath, engineHomePath, verifyExecutable, verifyRuntimePaths, ...(engine === "codex" ? { // Codex has no broker to meter it, so publish terminal-frame usage // to the shared advisory ledger — on the wake that published and on @@ -176,11 +177,24 @@ function grokBrokerTurnFor(agent: OrganizationRuntimeAgentConfig, grokBroker: En * CLI engines do not consume Pi's resource loader. Frame the same immutable * identity in JSON so arbitrary names/instructions cannot change its shape. */ -function identityEnvelope(agent: OrganizationRuntimeAgentConfig): string { +/** + * The caller-owned prompt preamble. + * + * It names the mounted tools explicitly. A CLI engine reaches Daimon's tools + * over MCP, and Grok exposes MCP tools only through a deferred `search_tool` + * catalog, so an agent whose instructions name another engine's tool spelling + * can finish a turn having called nothing. The declared names are the caller's + * own configuration, not engine-supplied text. + */ +function identityEnvelope(agent: OrganizationRuntimeAgentConfig, mountedToolNames: readonly string[] = []): string { return [ "", JSON.stringify({ id: agent.id, name: agent.name, instructions: agent.instructions }), "", + ...(mountedToolNames.length === 0 ? [] : [ + `Your mounted tools are exactly: ${mountedToolNames.join(", ")}. Call them by these names; ` + + "your instructions may spell them differently. No other tool reaches the newsroom." + ]), "Colleagues only hear you when you call moltnet_send; your terminal response is a private note to the runtime, not a message to anyone — keep it to one line or leave it empty. " + "Do not seek transport credentials or invoke a transport CLI unless the caller explicitly mounted an authenticated transport tool.", "The following is the current wake event." From 0c16fc5d2294eb9b8489e08bacdb117d7c8bc204 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 21:15:29 +0200 Subject: [PATCH 17/69] build: stage the packaged Linux engine broker on every packing host --- src/runtime/native/copyArtifact.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/runtime/native/copyArtifact.ts b/src/runtime/native/copyArtifact.ts index 1166aa6..be3c30d 100644 --- a/src/runtime/native/copyArtifact.ts +++ b/src/runtime/native/copyArtifact.ts @@ -4,8 +4,14 @@ import { fileURLToPath } from 'node:url'; await import('./verifyArtifacts.ts'); const root = path.dirname(fileURLToPath(import.meta.url)); -const architecture = process.arch; -if (!['x64', 'arm64'].includes(architecture) || process.platform !== 'linux') { +// The broker artifacts are prebuilt, provenance-verified Linux executables checked into +// this repository, so staging one is a packaging step and not a host capability: the +// published tarball must carry `dist/runtime/native/daimon-engine-broker` on every packing +// host, because the runtime image installs that tarball with no lifecycle script that could +// stage it later. `DAIMON_ENGINE_BROKER_ARCH` selects the packaged Linux target when it is +// not the host's own architecture. +const architecture = process.env.DAIMON_ENGINE_BROKER_ARCH?.trim() || process.arch; +if (!['x64', 'arm64'].includes(architecture)) { if (process.env.DAIMON_REQUIRE_ENGINE_BROKER === '1') throw new Error('native engine broker is Linux x64/arm64 only'); process.exit(0); } From 1af01fd4a58ac67a2cf4cbfedb340665f5bc5a45 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 21:51:03 +0200 Subject: [PATCH 18/69] fix: refuse broker proxy policy misses non-retryably with a named reason --- src/runtime/grokBrokerProxy.test.ts | 7 +++-- src/runtime/grokBrokerProxy.ts | 41 +++++++++++++++++++++++--- src/runtime/grokInferenceProxy.test.ts | 18 ++++++----- src/runtime/grokInferenceProxy.ts | 14 +++++++-- 4 files changed, 62 insertions(+), 18 deletions(-) diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index 0e6e6de..81714d7 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -46,7 +46,8 @@ test("proxy refuses a fail-open tool set or an undeclared effort without calling const token = proxy.capabilities.issue("agent", "turn"); arm(proxy, async () => undefined); const full = [...lean, ...["search_replace", "todo_write", "write", "monitor"].map((name) => ({ type: "function", function: { name } }))]; for (const payload of [leanBody({ tools: full }), leanBody({ tools: [{ type: "function", function: { name: "session_title" } }] }), leanBody({ reasoning_effort: "high" }), leanBody({ reasoning_effort: undefined })]) { - assert.equal(await post(proxy.port, token, payload), 503); + // A policy miss is non-retryable: 400, so Grok fails fast instead of retrying a 503. + assert.equal(await post(proxy.port, token, payload), 400); } assert.equal(calls, 0); assert.equal(await post(proxy.port, token, leanBody()), 200); assert.equal(calls, 1); assert.ok(accessed >= 1); @@ -68,7 +69,7 @@ test("the session-title sink is refused before capability, guard, credential, or try { const token = proxy.capabilities.issue("agent", "turn", 60_000, 1); arm(proxy, async () => { guarded++; }); const title = JSON.stringify({ model: "disabled", max_tokens: 100, temperature: 0, stream: true, messages: [{ role: "user", content: "prompt-derived" }], tool_choice: { type: "function", function: { name: "session_title" } }, tools: [{ type: "function", function: { name: "session_title" } }] }); - assert.equal(await post(proxy.port, GROK_SESSION_TITLE_SINK_KEY, title), 503); + assert.equal(await post(proxy.port, GROK_SESSION_TITLE_SINK_KEY, title), 400); assert.deepEqual({ calls, accessed, guarded }, { calls: 0, accessed: 0, guarded: 0 }); // The turn capability (budget 1 request) is untouched and still serves the real request. assert.equal(await post(proxy.port, token, leanBody()), 200); @@ -85,7 +86,7 @@ test("the isolation guard is awaited before the first upstream call, and a faili order.push("guard-start"); await new Promise((resolve) => setTimeout(resolve, 30)); order.push("guard-end"); if (fail) throw new Error("no enforcement evidence"); }); - assert.equal(await post(proxy.port, token, leanBody()), 503); + assert.equal(await post(proxy.port, token, leanBody()), 400); assert.equal(upstreamCalls, 0); assert.deepEqual(order, ["guard-start", "guard-end"]); fail = false; order.length = 0; diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index 4d1e6d0..294754e 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -34,14 +34,28 @@ export async function startGrokBrokerProxy(authority: GrokBrokerCredentialAuthor return { port: address.port, capabilities,registerIsolationGuard(turnId,guard){guards.set(turnId,guard);},revokeIsolationGuard(turnId){guards.delete(turnId);},registerTurn(turnId,turn){turns.set(turnId,{policy:parseGrokBrokerModelPolicy(turn.policy),meter:turn.meter});},revokeTurn(turnId){turns.delete(turnId);}, close: () => new Promise((resolve, reject) => server.close((error) => error === undefined ? resolve() : reject(error))) }; } +/** + * A refusal the caller must not retry. + * + * Every internal refusal used to collapse into one bare 503. Grok treats 503 as + * transient and blind-retries the same request (observed: 14 retries, ~141k + * estimated tokens, then `exit 1`), so a policy miss burned a turn's budget and + * reported itself as an engine crash. Policy refusals now answer 400 with a + * reason, and only genuinely transient faults keep 503. + */ +export class GrokBrokerProxyRefusal extends Error { + constructor(readonly reason: string) { super(`grok broker proxy refused: ${reason}`); } +} + async function serve(request: IncomingMessage, response: ServerResponse, authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream, capabilities: EngineBrokerCapabilities,guards:MapPromise>,turns:Map,fallback:GrokBrokerModelPolicy,grants?:GrokInferenceGrants): Promise { let settle:((usage:ReturnType)=>void)|undefined; try { const body = await readBody(request); const headers = Object.fromEntries(Object.entries(request.headers).map(([key, value]) => [key, Array.isArray(value) ? value[0] : value])); const match=headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u); - if(match&&match[1]!.startsWith(GROK_ENGINE_BROKER.inferenceGrants.tokenPrefix)){if(!grants)throw new Error();return await serveGrokInferenceGrant({method:request.method??"",pathname:new URL(request.url??"/","http://127.0.0.1").pathname,headers,body,token:match[1]!},response,grants,authority,upstream);} - const scope=match?capabilities.inspectToken(match[1]!):undefined;if(!scope)throw new Error();const guard=guards.get(scope.turnId),turn=turns.get(scope.turnId);if(!guard||!turn)throw new Error();await guard(); - let token = await authority.accessToken(false);const rejectedDigest=createHash("sha256").update(token).digest("hex"); let prepared = authorizeGrokBrokerProxyRequest({ method: request.method ?? "", pathname: new URL(request.url ?? "/", "http://127.0.0.1").pathname, headers, body }, capabilities, token, turn.policy ?? fallback); token = ""; + if(match&&match[1]!.startsWith(GROK_ENGINE_BROKER.inferenceGrants.tokenPrefix)){if(!grants)throw new GrokBrokerProxyRefusal("inference_grants_unavailable");return await serveGrokInferenceGrant({method:request.method??"",pathname:new URL(request.url??"/","http://127.0.0.1").pathname,headers,body,token:match[1]!},response,grants,authority,upstream);} + const scope=match?capabilities.inspectToken(match[1]!):undefined;if(!scope)throw new GrokBrokerProxyRefusal("unknown_capability");const guard=guards.get(scope.turnId),turn=turns.get(scope.turnId);if(!guard||!turn)throw new GrokBrokerProxyRefusal("no_active_turn"); + try { await guard(); } catch (error) { throw new GrokBrokerProxyRefusal("worker_isolation_unverified"); } + let token = await authority.accessToken(false);const rejectedDigest=createHash("sha256").update(token).digest("hex"); let prepared = authorizeRequestOrRefuse({ method: request.method ?? "", pathname: new URL(request.url ?? "/", "http://127.0.0.1").pathname, headers, body }, capabilities, token, turn.policy ?? fallback); token = ""; // The spend gate runs after the body is proven a real lean worker request // (a refused session-title body never counts) and before any upstream call. const admission=turn.meter.admit(); @@ -52,7 +66,26 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori if (result.status === 401) { token = authority.refreshAfterRejection?await authority.refreshAfterRejection(rejectedDigest):await authority.accessToken(true);const refreshedDigest=createHash("sha256").update(token).digest("hex"); prepared = { ...prepared, headers: { ...prepared.headers, authorization: `Bearer ${token}` } }; token = ""; result = await upstream(prepared,admission.signal);if(result.status===401)await authority.markRejected(refreshedDigest); } settle?.(parseGrokUpstreamUsage(result.body,result.headers["content-type"])); response.writeHead(result.status, { "content-type": result.headers["content-type"] ?? "application/json", "cache-control": "no-store" }); response.end(result.body); - } catch { settle?.(undefined); response.writeHead(503, { "content-type": "application/json", "cache-control": "no-store" }); response.end('{"error":"broker unavailable"}'); } + } catch (error) { + settle?.(undefined); + // Name the refusal on the broker's own stderr (reason code only, never a body + // or a token) so a failing turn is diagnosable without a stub harness. + const refusal = error instanceof GrokBrokerProxyRefusal ? error.reason : "broker_unavailable"; + process.stderr.write(`[grok-proxy] refused: ${refusal}\n`); + if (error instanceof GrokBrokerProxyRefusal) { + response.writeHead(400, { "content-type": "application/json", "cache-control": "no-store" }); + response.end(JSON.stringify({ error: "broker refused this request", reason: refusal })); + return; + } + response.writeHead(503, { "content-type": "application/json", "cache-control": "no-store" }); + response.end('{"error":"broker unavailable"}'); + } +} + +/** The body gate, refused non-retryably: a rejected body is a policy miss, never a transient fault. */ +function authorizeRequestOrRefuse(...args: Parameters): ReturnType { + try { return authorizeGrokBrokerProxyRequest(...args); } + catch { throw new GrokBrokerProxyRefusal("request_body_rejected"); } } async function readBody(request: IncomingMessage): Promise { const chunks: Buffer[] = []; let bytes = 0; for await (const chunk of request) { const value = Buffer.from(chunk); bytes += value.length; if (bytes > 2 * 1024 * 1024) throw new Error("too large"); chunks.push(value); } return Buffer.concat(chunks); } const defaultUpstream: GrokBrokerUpstream = async (request, signal) => { const result = await fetch(request.url, { method: "POST", headers: request.headers, body: Buffer.from(request.body), ...(signal === undefined ? {} : { signal }) }); return { status: result.status, headers: { "content-type": result.headers.get("content-type") ?? "application/json" }, body: new Uint8Array(await result.arrayBuffer()) }; }; diff --git a/src/runtime/grokInferenceProxy.test.ts b/src/runtime/grokInferenceProxy.test.ts index 8fcb20e..1bf6aee 100644 --- a/src/runtime/grokInferenceProxy.test.ts +++ b/src/runtime/grokInferenceProxy.test.ts @@ -60,9 +60,10 @@ test("a grant refuses any tools member, the session_title request, and undeclare judgeBody({ messages: [{ role: "tool", content: "x" }] }), judgeBody({ messages: [{ role: "assistant", content: null, tool_calls: [] }] }), judgeBody({ response_format: { type: "json_object" } }) ]; - for (const body of refused) assert.equal((await post(port, token, body)).status, 503, body.slice(0, 120)); - assert.equal((await post(port, token, judgeBody(), "1.0.30")).status, 503); - assert.equal((await post(port, GROK_SESSION_TITLE_SINK_KEY, titleBody)).status, 503); + // Policy misses are non-retryable: 400, so a judge fails fast instead of retrying a 503. + for (const body of refused) assert.equal((await post(port, token, body)).status, 400, body.slice(0, 120)); + assert.equal((await post(port, token, judgeBody(), "1.0.30")).status, 400); + assert.equal((await post(port, GROK_SESSION_TITLE_SINK_KEY, titleBody)).status, 400); assert.equal(bodies.length, 0); assert.equal(rows.length, 0); }); }); @@ -70,8 +71,8 @@ test("a grant refuses any tools member, the session_title request, and undeclare test("an expired, released or unknown grant is refused", async () => { await withProxy(async ({ port, grants, bodies }) => { const released = grants.issue({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }); grants.release(released.grantId); - assert.equal((await post(port, released.token, judgeBody())).status, 503); - assert.equal((await post(port, `inference_${"A".repeat(43)}`, judgeBody())).status, 503); + assert.equal((await post(port, released.token, judgeBody())).status, 400); + assert.equal((await post(port, `inference_${"A".repeat(43)}`, judgeBody())).status, 400); assert.equal(bodies.length, 0); }); let now = 5_000; @@ -81,7 +82,7 @@ test("an expired, released or unknown grant is refused", async () => { const { token } = grants.issue({ model: "grok-4.6", reasoningEffort: "low", purpose: "judge" }); assert.equal((await post(proxy.port, token, judgeBody())).status, 200); now += 600_000; - assert.equal((await post(proxy.port, token, judgeBody())).status, 503); + assert.equal((await post(proxy.port, token, judgeBody())).status, 400); } finally { grants.close(); await proxy.close(); } }); @@ -93,8 +94,9 @@ test("a grant token never authorizes a subject turn and a turn capability never proxy.registerTurn("turn-a", { policy: { model: "grok-4.6", reasoningEffort: "low" }, meter: new GrokBrokerTurnMeter({ maxRequests: 4, maxTokens: 10_000, timeoutMs: 60_000 }) }); const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); const leanBody = JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: lean }); - assert.equal((await post(port, grantToken, leanBody)).status, 503); - assert.equal((await post(port, turnToken, judgeBody())).status, 503); + // Cross-use is a policy miss on both paths: refused 400, never a retryable 503. + assert.equal((await post(port, grantToken, leanBody)).status, 400); + assert.equal((await post(port, turnToken, judgeBody())).status, 400); assert.equal(bodies.length, 0); assert.equal((await post(port, turnToken, leanBody)).status, 200); assert.equal((await post(port, grantToken, judgeBody())).status, 200); diff --git a/src/runtime/grokInferenceProxy.ts b/src/runtime/grokInferenceProxy.ts index 95169c1..34acc99 100644 --- a/src/runtime/grokInferenceProxy.ts +++ b/src/runtime/grokInferenceProxy.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto"; import type { ServerResponse } from "node:http"; +import { GrokBrokerProxyRefusal } from "./grokBrokerProxy.js"; import type { GrokBrokerCredentialAuthority, GrokBrokerUpstream } from "./grokBrokerProxy.js"; import { parseGrokUpstreamUsage } from "./grokBrokerTurnMeter.js"; import type { GrokInferenceGrants } from "./grokInferenceGrants.js"; @@ -30,8 +31,10 @@ export async function serveGrokInferenceGrant(input: GrokInferenceProxyInput, re let settle: ((usage: ReturnType) => void) | undefined; try { const grant = grants.authorize(input.token); - if (grant === undefined) throw new Error("inference grant unavailable"); - let prepared = authorizeGrokInferenceProxyRequest(input, "pending", grant.policy); + if (grant === undefined) throw new GrokBrokerProxyRefusal("unknown_or_expired_grant"); + let prepared: ReturnType; + try { prepared = authorizeGrokInferenceProxyRequest(input, "pending", grant.policy); } + catch { throw new GrokBrokerProxyRefusal("request_body_rejected"); } let token = await authority.accessToken(false); const rejectedDigest = createHash("sha256").update(token).digest("hex"); prepared = withBearer(prepared, token); token = ""; const admission = grant.meter.admit(); @@ -47,9 +50,14 @@ export async function serveGrokInferenceGrant(input: GrokInferenceProxyInput, re } settle?.(parseGrokUpstreamUsage(result.body, result.headers["content-type"])); json(response, result.status, result.body, result.headers["content-type"]); - } catch { + } catch (error) { settle?.(undefined); + // Same rule as the subject path: a policy miss is non-retryable (400), because a + // retryable 503 makes the client re-send a request the broker will never accept, + // charging estimated usage for every attempt. 503 stays for transient faults only. + process.stderr.write(`[grok-proxy] inference refused: ${error instanceof GrokBrokerProxyRefusal ? error.reason : "broker_unavailable"}\n`); if (authority.isStale?.() === true) json(response, 401, GROK_INFERENCE_AUTH_STALE_BODY); + else if (error instanceof GrokBrokerProxyRefusal) json(response, 400, JSON.stringify({ error: "broker refused this request", reason: error.reason })); else json(response, 503, '{"error":"broker unavailable"}'); } } From 7b9a92c9e182bdcc7bb08f7034b78398a3597f25 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 21:58:43 +0200 Subject: [PATCH 19/69] fix: keep the Grok session-title sink on its transient refusal shape --- src/runtime/grokBrokerProxy.test.ts | 3 ++- src/runtime/grokBrokerProxy.ts | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index 81714d7..79e8685 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -69,7 +69,8 @@ test("the session-title sink is refused before capability, guard, credential, or try { const token = proxy.capabilities.issue("agent", "turn", 60_000, 1); arm(proxy, async () => { guarded++; }); const title = JSON.stringify({ model: "disabled", max_tokens: 100, temperature: 0, stream: true, messages: [{ role: "user", content: "prompt-derived" }], tool_choice: { type: "function", function: { name: "session_title" } }, tools: [{ type: "function", function: { name: "session_title" } }] }); - assert.equal(await post(proxy.port, GROK_SESSION_TITLE_SINK_KEY, title), 400); + // The title sink keeps its transient 503 shape: a 4xx there ends Grok's session. + assert.equal(await post(proxy.port, GROK_SESSION_TITLE_SINK_KEY, title), 503); assert.deepEqual({ calls, accessed, guarded }, { calls: 0, accessed: 0, guarded: 0 }); // The turn capability (budget 1 request) is untouched and still serves the real request. assert.equal(await post(proxy.port, token, leanBody()), 200); diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index 294754e..1b89d58 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -4,6 +4,7 @@ import type { AddressInfo } from "node:net"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; import { DEFAULT_GROK_BROKER_MODEL_POLICY, parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; import { authorizeGrokBrokerProxyRequest } from "./grokBrokerProxyRequest.js"; +import { GROK_SESSION_TITLE_SINK_KEY } from "./grokBrokerWorkerConfig.js"; import { parseGrokUpstreamUsage, type GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; import { serveGrokInferenceGrant } from "./grokInferenceProxy.js"; @@ -49,8 +50,10 @@ export class GrokBrokerProxyRefusal extends Error { async function serve(request: IncomingMessage, response: ServerResponse, authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream, capabilities: EngineBrokerCapabilities,guards:MapPromise>,turns:Map,fallback:GrokBrokerModelPolicy,grants?:GrokInferenceGrants): Promise { let settle:((usage:ReturnType)=>void)|undefined; + let titleSink = false; try { const body = await readBody(request); const headers = Object.fromEntries(Object.entries(request.headers).map(([key, value]) => [key, Array.isArray(value) ? value[0] : value])); + titleSink = headers.authorization === `Bearer ${GROK_SESSION_TITLE_SINK_KEY}`; const match=headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u); if(match&&match[1]!.startsWith(GROK_ENGINE_BROKER.inferenceGrants.tokenPrefix)){if(!grants)throw new GrokBrokerProxyRefusal("inference_grants_unavailable");return await serveGrokInferenceGrant({method:request.method??"",pathname:new URL(request.url??"/","http://127.0.0.1").pathname,headers,body,token:match[1]!},response,grants,authority,upstream);} const scope=match?capabilities.inspectToken(match[1]!):undefined;if(!scope)throw new GrokBrokerProxyRefusal("unknown_capability");const guard=guards.get(scope.turnId),turn=turns.get(scope.turnId);if(!guard||!turn)throw new GrokBrokerProxyRefusal("no_active_turn"); @@ -72,6 +75,14 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori // or a token) so a failing turn is diagnosable without a stub harness. const refusal = error instanceof GrokBrokerProxyRefusal ? error.reason : "broker_unavailable"; process.stderr.write(`[grok-proxy] refused: ${refusal}\n`); + // Grok's own session-title call is refused by design. It must keep the transient + // 503 shape it has always had: a hard 4xx on that internal request ends Grok's + // session, which surfaces as the worker exiting 1 mid-turn. + if (titleSink) { + response.writeHead(503, { "content-type": "application/json", "cache-control": "no-store" }); + response.end('{"error":"broker unavailable"}'); + return; + } if (error instanceof GrokBrokerProxyRefusal) { response.writeHead(400, { "content-type": "application/json", "cache-control": "no-store" }); response.end(JSON.stringify({ error: "broker refused this request", reason: refusal })); From 1a6ee5d2d326ad0bdc516d8ac53bb48fdc5a4ac6 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 21:58:57 +0200 Subject: [PATCH 20/69] test: keep the grant-path title sink refusal transient --- src/runtime/grokInferenceProxy.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/runtime/grokInferenceProxy.test.ts b/src/runtime/grokInferenceProxy.test.ts index 1bf6aee..8e1f44c 100644 --- a/src/runtime/grokInferenceProxy.test.ts +++ b/src/runtime/grokInferenceProxy.test.ts @@ -63,7 +63,8 @@ test("a grant refuses any tools member, the session_title request, and undeclare // Policy misses are non-retryable: 400, so a judge fails fast instead of retrying a 503. for (const body of refused) assert.equal((await post(port, token, body)).status, 400, body.slice(0, 120)); assert.equal((await post(port, token, judgeBody(), "1.0.30")).status, 400); - assert.equal((await post(port, GROK_SESSION_TITLE_SINK_KEY, titleBody)).status, 400); + // The title sink keeps its transient 503 shape on the grant path too. + assert.equal((await post(port, GROK_SESSION_TITLE_SINK_KEY, titleBody)).status, 503); assert.equal(bodies.length, 0); assert.equal(rows.length, 0); }); }); From b54f66f9c9cf333dddafd30f2f797baff3ac4f31 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 22:55:08 +0200 Subject: [PATCH 21/69] fix: lead an inbox turn with each delivery's own text and keep the accounting after it --- src/runtime/attentionDispatcher.test.ts | 14 +++++++++ src/runtime/attentionDispatcher.ts | 41 ++++++++++++++++++++++--- 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/src/runtime/attentionDispatcher.test.ts b/src/runtime/attentionDispatcher.test.ts index 4774ae2..71dce93 100644 --- a/src/runtime/attentionDispatcher.test.ts +++ b/src/runtime/attentionDispatcher.test.ts @@ -200,3 +200,17 @@ test("claim-renewal failure revokes execution authority, stops cognition, and la const accepted = await f.control.accept(request("after-fence")); assert.equal(accepted.state, "stopped"); assert.equal(accepted.blocked!.reason, "ledger_unavailable"); } finally { WakeAcceptanceStore.prototype.renewClaim = original; await f.cleanup(); } }); + +test("an inbox turn leads with each delivery's own text and keeps the accounting after the work", async () => { + const f = await fixture(); + try { + await f.control.accept(request("d-1")); await until(() => f.core.wakes.length === 1); + const text = f.core.wakes[0]!.event.text; + // The task comes first: a delivery's text is the work, not a JSON payload to account for. + assert.match(text, /^Carry out this delivery\./u); + assert.match(text, //u); + const task = text.indexOf("Handle d-1"), accounting = text.indexOf("daimon_inbox_disposition"); + assert.ok(task >= 0 && accounting > task, "accounting must follow the delivery text"); + assert.ok(text.indexOf("Machine-readable payload:") > accounting, "payload stays a trailing appendix"); + } finally { await f.cleanup(); } +}); diff --git a/src/runtime/attentionDispatcher.ts b/src/runtime/attentionDispatcher.ts index b1a8f77..208bd5d 100644 --- a/src/runtime/attentionDispatcher.ts +++ b/src/runtime/attentionDispatcher.ts @@ -171,14 +171,47 @@ export function selectBatch(records: readonly StoredWakeAcceptanceRecord[], agen return selected.length ? selected : [first]; } +/** One claimed delivery, rendered as the task it is. */ +function deliveryBlock(message: unknown, index: number): string | undefined { + if (message === null || typeof message !== "object") return undefined; + const row = message as Record; + const text = typeof row.text === "string" ? row.text : undefined; + if (text === undefined) return undefined; + const from = typeof row.from === "string" ? row.from : undefined; + const kind = typeof row.kind === "string" ? row.kind : "delivery"; + const id = typeof row.delivery_id === "string" ? row.delivery_id : `#${index + 1}`; + return [``, text, ""].join("\n"); +} + +/** + * The inbox turn, task first. + * + * A delivery's own text *is* the work. Leading with bookkeeping and handing the + * model `JSON.stringify(messages)` buried the task: an agent read the JSON, did + * the accounting and deferred without doing the job (observed on Grok: nine + * model requests, no tool calls, nothing filed). The deliveries are therefore + * rendered as labelled blocks and the `daimon_inbox` accounting follows them as + * what to do *after* the work, with the machine-readable payload kept as a + * trailing appendix while it fits the same budget. + */ function inboxPrompt(messages: readonly unknown[], maxBytes = 12000): string { const body = JSON.stringify(messages); + const blocks = messages.map(deliveryBlock).filter((block): block is string => block !== undefined); + const accounting = "\nWhen the work above is done, record each delivery with daimon_inbox_disposition (complete), or defer the ones you could not finish; use daimon_inbox for deliveries and remaining allowances. Reading or ending this turn never completes a delivery, and deferred work waits for a later external wake.\n"; + const header = blocks.length === 1 ? "Carry out this delivery.\n" : `Carry out these ${blocks.length} deliveries.\n`; + const fits = (value: string): boolean => Buffer.byteLength(value) <= ORGANIZATION_RUNTIME_MAX_WAKE_TEXT_BYTES + && [...value].length <= ORGANIZATION_RUNTIME_MAX_STRING_CODEPOINTS; + if (blocks.length > 0 && Buffer.byteLength(blocks.join("\n\n")) <= maxBytes) { + const task = header + blocks.join("\n\n") + accounting; + const withPayload = `${task}\nMachine-readable payload: ${body}`; + // The inbox budget bounds selection; the v1 execution boundary independently + // bounds the complete prompt, including metadata, escaping, and instructions. + if (Buffer.byteLength(body) <= maxBytes && fits(withPayload)) return withPayload; + if (fits(task)) return task; + } const prefix = "Handle this inbox turn. Use daimon_inbox for deliveries and remaining allowances. Explicitly call daimon_inbox_disposition for each handled delivery (complete) or unfinished delivery (defer). Reading or ending this turn never completes a delivery. Deferred work waits for a later external wake.\n"; const prompt = prefix + body; - // The inbox budget bounds selection; the v1 execution boundary independently - // bounds the complete prompt, including metadata, escaping, and instructions. - if (Buffer.byteLength(body) > maxBytes || Buffer.byteLength(prompt) > ORGANIZATION_RUNTIME_MAX_WAKE_TEXT_BYTES - || [...prompt].length > ORGANIZATION_RUNTIME_MAX_STRING_CODEPOINTS) { + if (Buffer.byteLength(body) > maxBytes || !fits(prompt)) { return prefix + "The selected payload exceeds the prompt budget; read it with daimon_inbox."; } return prompt; From 8496d44c32a023b8608b420863b5800723cb80f1 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 23:30:42 +0200 Subject: [PATCH 22/69] fix: name a failed brokered worker's own reason instead of its exit code --- src/contracts/runtimeContractManifest.ts | 6 +-- src/runtime/AGENTS.md | 19 +++++++++ src/runtime/engineBrokerControlClient.ts | 2 +- src/runtime/engineBrokerNativeClient.test.ts | 26 +++++++++--- src/runtime/engineBrokerNativeClient.ts | 38 ++++++++++++++---- src/runtime/engineBrokerProtocol.test.ts | 15 +++++++ src/runtime/engineBrokerProtocol.ts | 15 ++++++- src/runtime/engineBrokerService.test.ts | 6 +++ src/runtime/native/AGENTS.md | 7 ++++ .../artifacts/daimon-engine-broker-arm64 | Bin 67456 -> 67456 bytes ...daimon-engine-broker-arm64.provenance.json | 2 +- .../native/artifacts/daimon-engine-broker-x64 | Bin 55392 -> 55392 bytes .../daimon-engine-broker-x64.provenance.json | 2 +- src/runtime/native/engineBrokerLauncher.h | 15 ++++--- .../engineBrokerLauncherIntegrationCore.inc | 8 ++-- ...ngineBrokerLauncherIntegrationLauncher.inc | 29 +++++++++++++ .../engineBrokerLauncherIntegrationMain.inc | 3 +- .../native/engineBrokerLauncherModes.inc | 10 +++-- .../native/engineBrokerLauncherServer.inc | 32 +++++++++++---- src/runtime/native/fixtureWorker.c | 2 +- 20 files changed, 195 insertions(+), 42 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 95a6673..41baa05 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -133,9 +133,9 @@ export const GROK_ENGINE_BROKER = { projectionVersion: "noopolis.daimon.grok-broker-projection.v1", slotPreflightVersion: "noopolis.daimon.grok-slot-preflight.v2", artifacts: { - sourceSha256: "c0082d4b366ffdb860d8154ee7f402b8f8be1f09d4eda277eda6965198ab6a75", - x64Sha256: "69e2865c722606a71501bc38d8b9c4c2c397e748327a8077e51e040319d1280d", - arm64Sha256: "16a3f89d84b7139d556b070a626c75ece224d5e05c52d48e045b38086c0a0382" + sourceSha256: "356a8e56e44dca9ab4784ac343587c0fc4d57e23f961124d8491cf6f504f8e98", + x64Sha256: "a21efd6a47059de7c5fe6add8b23799946728dfb44845ea9b6602402e5cfad02", + arm64Sha256: "a8bf311ca82ed004dd4efd69d1b7ae9edc1264876ca9bbb94c77226d40c75381" } } as const; export const AGY_SUBSCRIPTION_REALM = { diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 4d83329..0863d31 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -77,6 +77,25 @@ model (`grok-4.6-build` → `grok-4.6`), otherwise the turn fails as rejected an is still metered. Control protocol v2 is refused-v1 on the wire because both ends ship in this package. +A failed brokered turn also carries the worker's own last words. The launcher +gives the worker one pipe for stdout and stderr and publishes no output for a +failure, so a `worker_failed` turn used to reach the host as nothing but +`exit=1` — the reason the worker printed died with the container's tmpfs. +`DBL_MAX_DIAGNOSTIC` (512 bytes) is now the launcher's bounded tail of that +pipe, sent beside the fixed result frame in `diagnostic_length` and kept only +for a worker that exited on its own account: an output-limit tail would be the +very payload the bound refused, a cancelled turn has no reader left, and a +prelaunch failure ran nothing. `engineBrokerNativeClient.ts` redacts that tail +exactly as the CLI child path redacts a failed engine child +(`redactCredentialText` with the turn's own provider/MCP capabilities as exact +secrets, the same `CLI_ENGINE_MAX_DIAGNOSTIC_BYTES` bound) and flattens it to +one line as `diagnostic.reason`. It is an optional, control-character-free +member of the sealed terminal response's closed diagnostic — admitted by +`engineBrokerProtocol.ts` only for the statuses where a worker ran and spoke — +so it replays with the sealed record and reaches the operator through +`engineBrokerControlClient.ts`'s failure message. Nothing new is written to +disk: the reason travels inside the response the broker already seals. + Evaluator inference grants (`grokInferenceGrants.ts`) let Paideia judges and the DSPy optimizer — uid 2000, the trusted evaluator side — spend the broker's Grok credential without holding it. `request_inference_grant {model, diff --git a/src/runtime/engineBrokerControlClient.ts b/src/runtime/engineBrokerControlClient.ts index 6d5c845..a02d758 100644 --- a/src/runtime/engineBrokerControlClient.ts +++ b/src/runtime/engineBrokerControlClient.ts @@ -45,6 +45,6 @@ export class EngineBrokerControlClient implements EngineBrokerTurnClient { const turnId=createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"),requestId=randomUUID();const request={version:ENGINE_BROKER_VERSION,kind:"start_turn",requestId,turnId,agentId,wakeId,prompt,mcpEndpoint,...(options.limits===undefined?{}:{limits:options.limits})} as const;const socket=createConnection({path:this.socketPath});const decoder=new EngineBrokerFrameDecoder(); return new Promise((resolve,reject)=>{let accepted=false,settled=false;const fail=()=>{if(settled)return;settled=true;cleanup();reject(new Error("engine broker unavailable"));};const cleanup=()=>{signal?.removeEventListener("abort",abort);socket.destroy();};const abort=()=>fail();signal?.addEventListener("abort",abort,{once:true});if(signal?.aborted)return abort();socket.once("connect",()=>socket.write(encodeEngineBrokerFrame(request)));socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const response=parseEngineBrokerResponse(value);if((response.kind!=="accepted"&&response.kind!=="completed"&&response.kind!=="failed")||response.requestId!==requestId||response.turnId!==turnId)throw new Error();if(response.kind==="accepted"){if(accepted)throw new Error();accepted=true;continue;}if(!accepted||settled)throw new Error();settled=true;cleanup(); if(options.model!==undefined&&response.model!==options.model){reject(new Error(`engine broker turn used model ${response.model}, not the declared ${options.model}`));return;} - if(response.kind==="completed")resolve(response.text);else reject(new Error(`engine broker turn failed (${response.code}${response.limitReason==="none"?"":`; limit=${response.limitReason}`}${response.diagnostic ? `; ${response.diagnostic.stage}/${response.diagnostic.failureClass}; exit=${response.diagnostic.exitCode}; signal=${response.diagnostic.termSignal}` : ""})`));}}catch{fail();}});socket.once("error",fail);socket.once("close",()=>{if(!settled)fail();});}); + if(response.kind==="completed")resolve(response.text);else reject(new Error(`engine broker turn failed (${response.code}${response.limitReason==="none"?"":`; limit=${response.limitReason}`}${response.diagnostic ? `; ${response.diagnostic.stage}/${response.diagnostic.failureClass}; exit=${response.diagnostic.exitCode}; signal=${response.diagnostic.termSignal}${response.diagnostic.reason===undefined?"":`; reason=${response.diagnostic.reason}`}` : ""})`));}}catch{fail();}});socket.once("error",fail);socket.once("close",()=>{if(!settled)fail();});}); } } diff --git a/src/runtime/engineBrokerNativeClient.test.ts b/src/runtime/engineBrokerNativeClient.test.ts index 754dcfa..f960d63 100644 --- a/src/runtime/engineBrokerNativeClient.test.ts +++ b/src/runtime/engineBrokerNativeClient.test.ts @@ -1,10 +1,11 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { decodeNativeBrokerResult,encodeNativeBrokerTurn,ENGINE_BROKER_NATIVE_RESULT_BYTES,NativeBrokerTurnFailure } from "./engineBrokerNativeClient.js"; +import { CLI_ENGINE_MAX_DIAGNOSTIC_BYTES } from "../pi/cliChildOutput.js"; +import { decodeNativeBrokerResult,encodeNativeBrokerTurn,ENGINE_BROKER_NATIVE_DIAGNOSTIC_BYTES,ENGINE_BROKER_NATIVE_RESULT_BYTES,NativeBrokerTurnFailure } from "./engineBrokerNativeClient.js"; const turnId="turn-1"; -function frame(values:Readonly<{status?:number;uid?:number;pid?:number;exit?:number;signal?:number;ticks?:bigint;stage?:number;failure?:number;profile?:number;reserved?:number;text?:string}>={}):Buffer{ - const text=Buffer.from(values.text??"");const out=Buffer.alloc(ENGINE_BROKER_NATIVE_RESULT_BYTES+text.length);out.writeUInt32LE(2,0);out.writeUInt32LE(values.status??0,4);out.writeUInt32LE(values.uid??2200,8);out.writeUInt32LE(text.length,12);out.writeInt32LE(values.pid??42,16);out.writeInt32LE(values.exit??0,20);out.writeInt32LE(values.signal??0,24);out.writeBigUInt64LE(values.ticks??123n,32);out.write(turnId,40);out.writeUInt32LE(values.stage??7,108);out.writeUInt32LE(values.failure??0,112);out.writeUInt32LE(values.profile??0,116);out.writeUInt32LE(values.reserved??0,120);text.copy(out,ENGINE_BROKER_NATIVE_RESULT_BYTES);return out; +function frame(values:Readonly<{status?:number;uid?:number;pid?:number;exit?:number;signal?:number;ticks?:bigint;stage?:number;failure?:number;profile?:number;diagnosticLength?:number;diagnostic?:string;text?:string}>={}):Buffer{ + const text=Buffer.from(values.text??""),diagnostic=Buffer.from(values.diagnostic??"");const out=Buffer.alloc(ENGINE_BROKER_NATIVE_RESULT_BYTES+text.length+diagnostic.length);out.writeUInt32LE(2,0);out.writeUInt32LE(values.status??0,4);out.writeUInt32LE(values.uid??2200,8);out.writeUInt32LE(text.length,12);out.writeInt32LE(values.pid??42,16);out.writeInt32LE(values.exit??0,20);out.writeInt32LE(values.signal??0,24);out.writeBigUInt64LE(values.ticks??123n,32);out.write(turnId,40);out.writeUInt32LE(values.stage??7,108);out.writeUInt32LE(values.failure??0,112);out.writeUInt32LE(values.profile??0,116);out.writeUInt32LE(values.diagnosticLength??diagnostic.length,120);text.copy(out,ENGINE_BROKER_NATIVE_RESULT_BYTES);diagnostic.copy(out,ENGINE_BROKER_NATIVE_RESULT_BYTES+text.length);return out; } test("encodes ABI v2 and decodes a closed successful result",()=>{ @@ -21,7 +22,22 @@ test("returns bounded typed diagnostics for closed native failures",()=>{ ])assert.throws(()=>decodeNativeBrokerResult(frame(value),turnId),(error:unknown)=>error instanceof NativeBrokerTurnFailure&&error.diagnostic.stage!=="none"); }); -test("rejects unknown, reserved, output-bearing, and cross-class failure frames",()=>{ - for(const value of [{status:9},{reserved:1},{status:1,stage:7,failure:4,pid:0,uid:0,ticks:0n},{status:2,stage:6,failure:6,text:"secret"}])assert.throws(()=>decodeNativeBrokerResult(frame(value),turnId),/^Error: engine broker turn failed$/u); +test("rejects unknown, diagnostic-bearing success, output-bearing, and cross-class failure frames",()=>{ + for(const value of [{status:9},{diagnostic:"late words"},{status:1,stage:4,failure:4,pid:0,uid:0,ticks:0n,exit:-1,diagnostic:"no worker ran"},{status:2,stage:6,failure:5,exit:1,diagnosticLength:ENGINE_BROKER_NATIVE_DIAGNOSTIC_BYTES+1},{status:1,stage:7,failure:4,pid:0,uid:0,ticks:0n},{status:2,stage:6,failure:6,text:"secret"}])assert.throws(()=>decodeNativeBrokerResult(frame(value),turnId),/^Error: engine broker turn failed$/u); for(const offset of [28,31,105,107,124,127]){const hostile=frame({text:"done"});hostile[offset]=1;assert.throws(()=>decodeNativeBrokerResult(hostile,turnId),/^Error: engine broker turn failed$/u);} }); + +test("a failed worker's own last words cross as a redacted, bounded reason",()=>{ + const words=`{"type":"error","message":"session store unwritable"}\nBearer provider-cap-secret-value\ngrok: exiting 1\n`; + assert.throws(()=>decodeNativeBrokerResult(frame({status:2,stage:6,failure:5,exit:1,diagnostic:words}),turnId,["provider-cap-secret-value"]),(error:unknown)=>{ + assert.ok(error instanceof NativeBrokerTurnFailure); + const reason=error.diagnostic.reason; + assert.ok(reason!==undefined,"the worker's own reason must reach the diagnostic"); + assert.match(reason,/session store unwritable/u); + assert.doesNotMatch(reason,/provider-cap-secret-value/u,"the turn capability must never reach a diagnostic"); + assert.doesNotMatch(reason,/[\n\r\u0000-\u001f]/u,"the reason is one bounded line"); + assert.ok(Buffer.byteLength(reason,"utf8")<=CLI_ENGINE_MAX_DIAGNOSTIC_BYTES); + return true; + }); + assert.throws(()=>decodeNativeBrokerResult(frame({status:2,stage:6,failure:5,exit:1}),turnId,[]),(error:unknown)=>error instanceof NativeBrokerTurnFailure&&error.diagnostic.reason===undefined,"a worker that said nothing reports no reason rather than an empty one"); +}); diff --git a/src/runtime/engineBrokerNativeClient.ts b/src/runtime/engineBrokerNativeClient.ts index 8d905bd..18fff46 100644 --- a/src/runtime/engineBrokerNativeClient.ts +++ b/src/runtime/engineBrokerNativeClient.ts @@ -1,14 +1,18 @@ import { spawn } from "node:child_process"; +import { redactCredentialText } from "../core/credentialRedaction.js"; +import { CLI_ENGINE_MAX_DIAGNOSTIC_BYTES } from "../pi/cliChildOutput.js"; import { terminateChild, trackCliChild } from "../pi/cliProcess.js"; export const ENGINE_BROKER_NATIVE_REQUEST_BYTES = 396; export const ENGINE_BROKER_NATIVE_RESULT_BYTES = 128; +/** `DBL_MAX_DIAGNOSTIC`: the launcher's bounded tail of a failed worker's own merged stdout/stderr. */ +export const ENGINE_BROKER_NATIVE_DIAGNOSTIC_BYTES = 512; const MAX_PROMPT = 65_536, MAX_CAPABILITY = 4_096, MAX_OUTPUT = 65_536; const statuses = ["ok", "prelaunch_failed", "worker_failed", "output_failed", "cancelled"] as const; const stages = ["none", "peer", "request", "registration", "executable", "exec", "wait", "output", "attestation"] as const; const failures = ["none", "peer", "protocol", "registration", "executable", "exec", "wait", "output_limit", "cancelled", "profile_missing", "profile_invalid"] as const; -export interface NativeBrokerDiagnostic { exitCode:number;failureClass:typeof failures[number];profileApplied:boolean;stage:typeof stages[number];startTicks:string;status:typeof statuses[number];termSignal:number;workerPid:number;workerUid:number } +export interface NativeBrokerDiagnostic { exitCode:number;failureClass:typeof failures[number];profileApplied:boolean;reason?:string;stage:typeof stages[number];startTicks:string;status:typeof statuses[number];termSignal:number;workerPid:number;workerUid:number } export class NativeBrokerTurnFailure extends Error { constructor(readonly diagnostic:NativeBrokerDiagnostic){super("engine broker turn failed");} } export type NativeBrokerTurn = Readonly<{slot:number;requestId:string;turnId:string;agentId:string;wakeId:string;prompt:string;providerCapability:string;mcpCapability:string}>; export interface NativeBrokerTurnResult {text:string;workerPid:number;workerUid:number;startTicks:bigint;diagnostic:NativeBrokerDiagnostic} @@ -16,22 +20,40 @@ export interface NativeBrokerTurnResult {text:string;workerPid:number;workerUid: export async function runNativeBrokerTurn(executable:string,input:NativeBrokerTurn,signal?:AbortSignal):Promise>{ const frame=encodeNativeBrokerTurn(input),child=trackCliChild(spawn(executable,["--client"],{detached:process.platform!=="win32",env:{LANG:"C",LC_ALL:"C",TZ:"UTC"},stdio:["pipe","pipe","ignore"],...(signal===undefined?{}:{signal})}));const chunks:Buffer[]=[];let bytes=0; child.stdout!.on("data",(chunk:Buffer)=>{bytes+=chunk.length;if(bytes<=ENGINE_BROKER_NATIVE_RESULT_BYTES+MAX_OUTPUT)chunks.push(chunk);});child.stdin!.end(frame);frame.fill(0); - try{const code=await new Promise((resolve,reject)=>{child.once("error",reject);child.once("exit",resolve);});if(code!==0||bytes>ENGINE_BROKER_NATIVE_RESULT_BYTES+MAX_OUTPUT)throw new Error();return decodeNativeBrokerResult(Buffer.concat(chunks),input.turnId);}catch(error){if(error instanceof NativeBrokerTurnFailure)throw error;throw new Error("engine broker turn failed");}finally{await terminateChild(child).catch(()=>undefined);} + try{const code=await new Promise((resolve,reject)=>{child.once("error",reject);child.once("exit",resolve);});if(code!==0||bytes>ENGINE_BROKER_NATIVE_RESULT_BYTES+MAX_OUTPUT)throw new Error();return decodeNativeBrokerResult(Buffer.concat(chunks),input.turnId,[input.providerCapability,input.mcpCapability]);}catch(error){if(error instanceof NativeBrokerTurnFailure)throw error;throw new Error("engine broker turn failed");}finally{await terminateChild(child).catch(()=>undefined);} } export function encodeNativeBrokerTurn(input:NativeBrokerTurn):Buffer{if(!Number.isInteger(input.slot)||input.slot<0)throw new TypeError("invalid engine broker turn");const p=Buffer.from(input.prompt),provider=Buffer.from(input.providerCapability),mcp=Buffer.from(input.mcpCapability);if(p.length<1||p.length>MAX_PROMPT||provider.length<1||provider.length>MAX_CAPABILITY||mcp.length<1||mcp.length>MAX_CAPABILITY||provider.equals(mcp))throw new TypeError("invalid engine broker turn");const c=Buffer.alloc(4+provider.length+mcp.length);c.writeUInt16LE(provider.length,0);provider.copy(c,2);c.writeUInt16LE(mcp.length,2+provider.length);mcp.copy(c,4+provider.length);const frame=Buffer.alloc(ENGINE_BROKER_NATIVE_REQUEST_BYTES+8+p.length+c.length);frame.writeUInt32LE(2,0);frame.writeUInt32LE(input.slot,4);field(frame,8,65,input.requestId);field(frame,73,65,input.turnId);field(frame,138,129,input.agentId);field(frame,267,129,input.wakeId);let o=ENGINE_BROKER_NATIVE_REQUEST_BYTES;frame.writeUInt32LE(p.length,o);o+=4;p.copy(frame,o);o+=p.length;frame.writeUInt32LE(c.length,o);o+=4;c.copy(frame,o);p.fill(0);provider.fill(0);mcp.fill(0);c.fill(0);return frame;} -export function decodeNativeBrokerResult(output:Buffer,turnId:string):NativeBrokerTurnResult{ +export function decodeNativeBrokerResult(output:Buffer,turnId:string,secrets:readonly string[]=[]):NativeBrokerTurnResult{ if(output.lengthbytes.every((byte)=>byte===0)); - if(output.length!==ENGINE_BROKER_NATIVE_RESULT_BYTES+length||output.readUInt32LE(0)!==2||status>=statuses.length||stage>=stages.length||failure>=failures.length||profile>1||reserved!==0||!paddingZero||observed!==turnId||length>MAX_OUTPUT)throw new Error("engine broker turn failed"); - const diagnostic:NativeBrokerDiagnostic={status:statuses[status]!,stage:stages[stage]!,failureClass:failures[failure]!,profileApplied:profile===1,exitCode,termSignal,workerPid:pid,workerUid:uid,startTicks:ticks.toString()}; - const success=status===0&&stage===7&&failure===0&&profile===0&&pid>0&&uid>=2200&&ticks>0n&&exitCode===0&&termSignal===0; - const prelaunch=status===1&&stage>=1&&stage<=5&&failure>=1&&failure<=5&&profile===0&&pid===0&&uid===0&&ticks===0n; + if(output.length!==ENGINE_BROKER_NATIVE_RESULT_BYTES+length+diagnosticLength||output.readUInt32LE(0)!==2||status>=statuses.length||stage>=stages.length||failure>=failures.length||profile>1||!paddingZero||observed!==turnId||length>MAX_OUTPUT||diagnosticLength>ENGINE_BROKER_NATIVE_DIAGNOSTIC_BYTES)throw new Error("engine broker turn failed"); + const reason=workerReason(output.subarray(ENGINE_BROKER_NATIVE_RESULT_BYTES+length),secrets); + const diagnostic:NativeBrokerDiagnostic={status:statuses[status]!,stage:stages[stage]!,failureClass:failures[failure]!,profileApplied:profile===1,...(reason===undefined?{}:{reason}),exitCode,termSignal,workerPid:pid,workerUid:uid,startTicks:ticks.toString()}; + const success=status===0&&stage===7&&failure===0&&profile===0&&pid>0&&uid>=2200&&ticks>0n&&exitCode===0&&termSignal===0&&diagnosticLength===0; + const prelaunch=status===1&&stage>=1&&stage<=5&&failure>=1&&failure<=5&&profile===0&&pid===0&&uid===0&&ticks===0n&&diagnosticLength===0; const worker=status===2&&stage===6&&(failure===5||failure===6)&&profile===0&&pid>0&&uid>=2200&&ticks>0n; const outputFailure=status===3&&stage===7&&failure===7&&profile===0&&pid>0&&uid>=2200&&ticks>0n; const cancelled=status===4&&stage===6&&failure===8&&profile===0&&pid>0&&uid>=2200&&ticks>0n; if(!success){if(length!==0||(!prelaunch&&!worker&&!outputFailure&&!cancelled))throw new Error("engine broker turn failed");throw new NativeBrokerTurnFailure(diagnostic);} return{text:output.subarray(ENGINE_BROKER_NATIVE_RESULT_BYTES).toString("utf8"),workerUid:uid,workerPid:pid,startTicks:ticks,diagnostic}; } +/** + * The worker's own last words, fit to cross a boundary. + * + * A failed brokered turn otherwise reports nothing but `exit=1`: the launcher + * merges the worker's stdout and stderr into one pipe and publishes no output + * for a failure, so this bounded tail is the only account of why it failed. + * It is worker-controlled text, so it is redacted exactly as the CLI child + * path redacts a failed engine child (`redactCredentialText` with the turn's + * own capabilities as exact secrets, the same diagnostic bound) and flattened + * to one line, because it travels inside a failure message. + */ +function workerReason(tail:Buffer,secrets:readonly string[]):string|undefined{ + if(tail.length===0)return undefined; + const flattened=tail.toString("utf8").replace(/[\u0000-\u001f\u007f]+/gu," ").replace(/\s+/gu," ").trim(); + const reason=redactCredentialText(flattened,secrets,CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); + return reason.length===0?undefined:reason; +} function field(target:Buffer,offset:number,length:number,value:string):void{if(!/^[A-Za-z0-9][A-Za-z0-9._:-]*$/u.test(value)||Buffer.byteLength(value)>=length)throw new TypeError("invalid engine broker turn");target.write(value,offset,"utf8");} diff --git a/src/runtime/engineBrokerProtocol.test.ts b/src/runtime/engineBrokerProtocol.test.ts index ae762bc..2ce17e4 100644 --- a/src/runtime/engineBrokerProtocol.test.ts +++ b/src/runtime/engineBrokerProtocol.test.ts @@ -62,3 +62,18 @@ test("start_turn limits are an optional closed subset inside their bounds", () = assert.throws(() => parseEngineBrokerRequest({ ...start, limits }), /invalid broker frame/u); } }); + +test("a failed worker's redacted reason is an optional bounded member of its diagnostic",()=>{ + const worker={status:"worker_failed",stage:"wait",failureClass:"exec",profileApplied:false,exitCode:1,termSignal:0,workerPid:31,workerUid:2200,startTicks:"9"} as const; + const failed={version:start.version,kind:"failed",requestId:"request-1",turnId:"turn-1",code:"engine_failed",outcome:"failed",usage:null,model:"grok-4.6",requests:4,limitReason:"none"} as const; + const named={...failed,diagnostic:{...worker,reason:"grok: session store unwritable"}} as const; + assert.deepEqual(parseEngineBrokerResponse(named),named); + const prelaunch={status:"prelaunch_failed",stage:"executable",failureClass:"executable",profileApplied:false,exitCode:-1,termSignal:0,workerPid:0,workerUid:0,startTicks:"0"} as const; + for(const bad of [ + {...failed,diagnostic:{...worker,reason:""}}, + {...failed,diagnostic:{...worker,reason:"x".repeat(769)}}, + {...failed,diagnostic:{...worker,reason:"line\nbreak"}}, + {...failed,diagnostic:{...worker,reason:7}}, + {...failed,diagnostic:{...prelaunch,reason:"no worker ran"}} + ])assert.throws(()=>parseEngineBrokerResponse(bad),/invalid broker frame/u); +}); diff --git a/src/runtime/engineBrokerProtocol.ts b/src/runtime/engineBrokerProtocol.ts index 9b9f644..eedf428 100644 --- a/src/runtime/engineBrokerProtocol.ts +++ b/src/runtime/engineBrokerProtocol.ts @@ -20,7 +20,15 @@ export type EngineBrokerRequest = | Readonly<{ version: typeof VERSION; kind: "cancel_turn"; requestId: string; turnId: string }> | EngineBrokerInferenceRequest; -export interface EngineBrokerFailureDiagnostic { status:string;stage:string;failureClass:string;profileApplied:boolean;exitCode:number;termSignal:number;workerPid:number;workerUid:number;startTicks:string } +/** + * `reason` is the worker's own last words (`engineBrokerNativeClient.ts`), + * already redacted and flattened to one bounded line by the broker. It is the + * only field a failed turn carries that the worker itself wrote, so it is + * optional, bounded, control-character free, and admitted only for the + * statuses where a worker actually ran and spoke. + */ +export interface EngineBrokerFailureDiagnostic { status:string;stage:string;failureClass:string;profileApplied:boolean;reason?:string;exitCode:number;termSignal:number;workerPid:number;workerUid:number;startTicks:string } +export const ENGINE_BROKER_MAX_DIAGNOSTIC_REASON_BYTES = 768; export type EngineBrokerResponse = | Readonly<{ version: typeof VERSION; kind: "ready"; requestId: string; brokerUid: 2100; providerProxyPort: 43123; mcpFacadePort: 43124; registrations: number; credentialStale: false; realmLease: true; workerIsolation: true }> @@ -105,7 +113,7 @@ function parseTerminal(input: JsonRecord, expected: typeof VERSION | typeof V1): const codes: readonly string[] = expected === VERSION ? ENGINE_BROKER_FAILURE_CODES : ENGINE_BROKER_FAILURE_CODES.filter((code) => code !== "limit_exceeded"); if (!codes.includes(input.code as string)) throw new TypeError("invalid broker frame"); let diagnostic:EngineBrokerFailureDiagnostic|undefined; - if(input.diagnostic!==undefined){const value=record(input.diagnostic);exact(value,["status","stage","failureClass","profileApplied","exitCode","termSignal","workerPid","workerUid","startTicks"]);const status=["prelaunch_failed","worker_failed","output_failed","cancelled"],stage=["peer","request","registration","executable","exec","wait","output","attestation"],failureClass=["peer","protocol","registration","executable","exec","wait","output_limit","cancelled","profile_missing","profile_invalid"];if(!status.includes(value.status as string)||!stage.includes(value.stage as string)||!failureClass.includes(value.failureClass as string)||typeof value.profileApplied!=="boolean"||![value.exitCode,value.termSignal,value.workerPid,value.workerUid].every(Number.isSafeInteger)||typeof value.startTicks!=="string"||!/^(0|[1-9][0-9]*)$/u.test(value.startTicks)||!closedDiagnostic(value))throw new TypeError("invalid broker frame");diagnostic=value as unknown as EngineBrokerFailureDiagnostic;} + if(input.diagnostic!==undefined){const value=record(input.diagnostic);const fields=["status","stage","failureClass","profileApplied","exitCode","termSignal","workerPid","workerUid","startTicks"];exact(value,value.reason===undefined?fields:[...fields,"reason"]);if(value.reason!==undefined&&(typeof value.reason!=="string"||value.reason.length===0||Buffer.byteLength(value.reason,"utf8")>ENGINE_BROKER_MAX_DIAGNOSTIC_REASON_BYTES||/[\u0000-\u001f\u007f]/u.test(value.reason)))throw new TypeError("invalid broker frame");const status=["prelaunch_failed","worker_failed","output_failed","cancelled"],stage=["peer","request","registration","executable","exec","wait","output","attestation"],failureClass=["peer","protocol","registration","executable","exec","wait","output_limit","cancelled","profile_missing","profile_invalid"];if(!status.includes(value.status as string)||!stage.includes(value.stage as string)||!failureClass.includes(value.failureClass as string)||typeof value.profileApplied!=="boolean"||![value.exitCode,value.termSignal,value.workerPid,value.workerUid].every(Number.isSafeInteger)||typeof value.startTicks!=="string"||!/^(0|[1-9][0-9]*)$/u.test(value.startTicks)||!closedDiagnostic(value))throw new TypeError("invalid broker frame");diagnostic=value as unknown as EngineBrokerFailureDiagnostic;} const base = { kind: "failed", requestId: id(input.requestId), turnId: id(input.turnId), code: input.code as EngineBrokerFailureCode, ...(diagnostic ? { diagnostic } : {}) } as const; if (expected === V1) return { version: V1, ...base } as V1Failed; const accountingValue = parseEngineBrokerTurnAccounting(input, "failed"); @@ -115,6 +123,9 @@ function parseTerminal(input: JsonRecord, expected: typeof VERSION | typeof V1): function closedDiagnostic(value:JsonRecord):boolean{ if(value.profileApplied!==false||(value.workerPid as number)<0||(value.workerUid as number)<0)return false; + // Only a worker that ran and wrote something can have said why it failed: + // no prelaunch failure and no attestation refusal carries worker words. + if(value.reason!==undefined&&!((value.status==="worker_failed"&&value.stage==="wait")||value.status==="output_failed"||value.status==="cancelled"))return false; const noWorker=value.workerPid===0&&value.workerUid===0&&value.startTicks==="0"; const worker=(value.workerPid as number)>0&&(value.workerUid as number)>=2200&&value.startTicks!=="0"; if(value.status==="prelaunch_failed")return noWorker&&({peer:"peer",request:"protocol",registration:"registration",executable:"executable",exec:"exec"} as Record)[value.stage as string]===value.failureClass; diff --git a/src/runtime/engineBrokerService.test.ts b/src/runtime/engineBrokerService.test.ts index d33f2bf..c48a192 100644 --- a/src/runtime/engineBrokerService.test.ts +++ b/src/runtime/engineBrokerService.test.ts @@ -56,3 +56,9 @@ test("a limit failure reaches the client with its code and limit reason", async await assert.rejects(client.turn("agent-a","wake-a","hello","http://127.0.0.1:44001/mcp"),/engine broker turn failed \(limit_exceeded; limit=requests\)/u); },async()=>{throw new EngineBrokerTurnFailure("limit_exceeded",undefined,{outcome:"failed",usage:{input:1,cacheRead:0,cacheWrite:0,output:1,total:2},model:"grok-4.6",requests:3,limitReason:"requests"});}); }); + +test("a failed worker's own reason reaches the client instead of a bare exit code", async () => { + await withService(async (client) => { + await assert.rejects(client.turn("agent-a","wake-a","hello","http://127.0.0.1:44001/mcp"),/engine broker turn failed \(engine_failed; wait\/exec; exit=1; signal=0; reason=grok: session store unwritable\)/u); + },async()=>{throw new EngineBrokerTurnFailure("engine_failed",{status:"worker_failed",stage:"wait",failureClass:"exec",profileApplied:false,exitCode:1,termSignal:0,workerPid:31,workerUid:2200,startTicks:"9",reason:"grok: session store unwritable"},{outcome:"failed",usage:null,model:"grok-4.6",requests:4,limitReason:"none"});}); +}); diff --git a/src/runtime/native/AGENTS.md b/src/runtime/native/AGENTS.md index 8fd8f77..5e8f4f9 100644 --- a/src/runtime/native/AGENTS.md +++ b/src/runtime/native/AGENTS.md @@ -39,3 +39,10 @@ Holding one verified descriptor and `execveat`-ing it would not make a replaced binary unrunnable: Grok 1.0.34 re-executes itself inside bubblewrap by path (`/usr/local/bin/grok`), so the image path's root ownership, not the launcher descriptor, is what protects the sandboxed process. + +The result frame's last word is `diagnostic_length`, not padding: on +`DBL_STATUS_WORKER_FAILED` the supervisor keeps the last `DBL_MAX_DIAGNOSTIC` +bytes of the worker's merged stdout/stderr and sends them after the fixed +frame, while `output_length` stays 0 as before. Every other failure sends none, +and `closed_result` refuses a frame that mixes the two. The bytes are the +worker's own, so the broker redacts them before they cross any boundary. diff --git a/src/runtime/native/artifacts/daimon-engine-broker-arm64 b/src/runtime/native/artifacts/daimon-engine-broker-arm64 index 6c393474c393788fcddb983d037e3d6986f75d10..8f89d08f2297daaa569b69b164f914dd6d52455a 100755 GIT binary patch delta 8940 zcmc&)eRNdSwLfR>Op+l9BqU)bBw=O}zA}-IKqenT%$LsbYOqgaXtk0&X+hw@^wl=0t&g_ws!|CV&>iKx>SHb<8qJt3NIm zBa;ym=EQgz_*~F*P&Q}=C6$GTL$XlOY^x5IiK(cMN}AFRkT z3aM4 zs#aoaFwDyfX;E}N7m9d6Ml6yR#};}V-$4gP%{PgBx;m;>rc!l=EjPYkSkD_eA_Yl8 ze=)7ju$A(9jHiKwQZ){OZyS!bGNF}1yE1HaJR>cAX98-W6F`T6?oB|GX&C4YnQ7@q z6HqHn$+YE;NYH;Qp)`%Yk!hnHXmct79Z8P?{T0yj31|kr3iLCe0}1G8`W$Hf=(O~d zFTb)on+itT=oX;%1au571iBgMgKEbRmOA<+5+G@nibP0LD4Uz~uB zqx38r%?7$60d-O}&{aS;B%p<~8t8LCwPOx;Ft`K*M8fg@04T zQ^T8j_I4{vPsaV-x9K;XeOO+ZznwRjVWO!vTk7k`6_#sGljGz;)ux(BQ>IfO@0L-Q zzSKJcR^h?Hvj4EjLm%W#@VD!FN53N6-N^$LVI`;QIOMPhH2w@c0z9l_b$y{o_aD*L zlqNDG%9yUOlI{Kr>r#Y=+iXc{Ym{iSkK3A}m@E%8S|R%kIomBwfyN%R2}>@|azBnT zhf^8%4l7e+5c^o57fO2XMZp*7IG_xaKT5mui~Xq@8{q+tQ?1N}!Lsb~rE1f;17L;> z+!ajKN?5U3+~#CFJeY1{ z*Xc){KW9U5$v~qMl~cPP?`KV!(8R+GU(qXYk$B^&-|{fCyRNh-oeL;;T#>&3H7*%5Fs7>( zTB6>!!TVY6e}e3@uqU&-OjnITG)`dK%u$TJ--bG>A!oUzInY?emPBXForJoPttKr1 zOU8h1MHxpyP>3==#(Oder~0@89KG+2D=KDJbr0)hD-E~gxIcx=R`{92#aCMDI(MV& z6Pn;Ck>}}lN1i-Ds~vg8@1utEQ{BT65>ZTuCX@RFu)z8Mi5;Q?4hOCYZ=z)neeRek z50T3`Nq&zSoOx4968gCr{hS@f$RtmDwtGCv5~p!5X-=E(b8IaEh0#)?45cAJ3<_~3_Rb-8d1m$(Y#$JF4OHEJ44j8MF$SDamzM&z0R zkAz$fIgZ|Qjn6pMUCqSrmvFi$>7#f5nV+hL*;&OWO|C(MlS{|SieKSIrg z6}ekbBP<5WgC>}hAzHh?)Qz~@Ne2p-W^O>~B@8v9)iZ7rx2mP$BccE9-H8o zuTuKNr8(8m+zkq&o;W{$P{kk*QQO2LvXH7K-621s$0pU_T6O~Y56Lz8pnQVfnLJ7U zfQBdEXBxgn-z#1w7tx!=B{@I1hQ%WXj&KN{!9ZMXfkrcBx*b_ZQG(?#dH@Dsb9zVL zI)>`p)j93JO=_T*H@iX+yeMJ%cGBD>at^)fwppfPIR)rFchQq2GjJ`PSyJj>i6)L@ zQR*|I%5)o5Gb zy(@@Cbx`+&x2mCtzPr(67Pl`Ia^>Cuz3FmnmnekJK}B?l0iDj4+-X?`c^_5ZFyFr( zxnPbs98RqbIZ{NcH%B!23WerM3G{vd&E46e%r{XQ^1Iv<%WjkA$_Vtn#pPL$r$UOv;pohCqgBa=~#%XvTKIO4xN3 zZJeV9DvUT19xc*h(rPV%#)mhn`&u&8!7qRM=oMk+0$>Eo^5(8u?Q&|gJL>ozl1y&q*}(K;FVbL+jK zb1Ow?w^Ow4RYcR-P2SMi~y*}S2TT14nKs@l5OB$}GxrSMNgvsMV#9fJ{q z54cgNH0qY(b9oDkoufoqkFdD8182HuVt0o}fTxL8oK;P{)&nW?Hu_TPH>!}sbt&4` zBBAk_6;blLzG)VtEQmqso|+?rbaHB~e=EksN7J zZ2eYP{HYPdJv(?1%h!p+ETZHYav=h9FMJTVq95i-;0bVIitmEK!E3tbES$$}&tL)= zUnWecC)g0XXpLObn_9BP?SA-a8}f03rf@!H&>#BQAb-@NAT06%_z(&nLG-tPcEg#= zcFx(j?TcumGZ>W+COle5claV!3q}*Xs2~1wZ1moXsP`LnPymZ}xbMPO9X4t>YrHHaf3&I9iJ)JIfnED6l%hJ20izb=OJ~3sEADS zKIAYL+j2j`)xEXr>Y7#5M*z2%Im*1P9R~^!u zX0rwNsUf~a?ZMu3o5!u7NwvcQb5 zep8D7f=vzdu79%Q#|O}O9WKW{n;P=rn&Qh-L%6G!@iyw%>m62O25{#Wxg*A3@D2wO zaK3~w&Q7_4W1Ah~c-cGbhv5&vCx+p00k2+*o1Ptg7aLwqId9_#6BpwQ8e{T zz%~cJvObOhC-l9@bxVXN3!Jygpj$L?KLT_sFsECTd-|bB*{o7>=Oev%Z_!ulE7vZy zOxiB7_wreM9_7rr^TAlBDz#Q-HGBsu@c3ec=w(HAjf9L4z3}-&(Nwdtyb|}PE*B1& z4B+ZSaY&}9Rd{))MJ9-{WiUa(Q-jP`E1T&=rDHA+oV7)2#2~v$ju(d_XQgJ6f!;wr zl8Sf%Mz7PlKczDWTY@u%2Ls|~i(VeMlS->>{w3Vs4BWYv)EZWm-WpnDRR+l^arAD6R?j`yJaGst1cyg7M6LC}A2ofLop2&d@8jr0BA z1LRP~u9Vn=@cy$8Z-)9-dMmUAZ50~c1oz=(ZeMYvLQ8Q~YIrBwhZk;rpZF@Z*MpTB zUc>g`r9!iFYIIcsJW#+-A{I_E@w} z@-jQ|YONiQrHk7?>9%OEizMx%tkIo#4c$H-^rA0GdnAygalGRtQeSPjTyu#E?b^H~ zZQ!R#8X^rk%#p?VaO*Bn-X-jXUWrDFCKb3@=4Bo@LXnpyI zSp3qx1!$zDc)JUVukccbvOiL1^*SY4+d!_Ha+DP9eyX}@ZgGm1BCT4?>+PQet=j8h zt9Gu=q@CTW7>>4xEseF#8|kH+3Zz26zv)NTX6_%RFQq$cY8onH1L0sQi{+bjc4i8=*)zw&E>gO^wR7_Lm)JZ$F&Y3y) zkG!klMM4C2V3X-8LHodv?%9ol_6+X9QnVYJlEmYI=pLbQAAf-!Os`wTLpb;GFnu~_ zsVt|4xwCSGNykVOltUl3Ypa=I_Y-u)CU9_*Nx)Vzq^7bbU?cxhSe&qjh4Y z+s@On`p4y3x=_DV-b=SPY?oDZx?#C|;^5o`byD7R@X?z$IpqTfEvu4~{pwy_{|sBl zqW!wQ8C!}tsOvXimHWU|@OE$&3UKEkUEcw&f(OADfgj~^aNZq!hjl%hS3fv^ z9a!~my1o`%9MSa`z}LL0>vrrI?cmcD;TPs#>-u5XJGCeQ3hZjKY)U&q2^!6)sb!{P@+6&fAk}uQT zbrq(*gY@vaiL=*m9eA_}o?XKO@Qw_o){YGR$y~SVP{Z>F8*1s5hNliR)IGhw;hA4F z%Jv*BO58caVQ@!q2Oqxo{=28%Ir-Mxr~fiicvGFw_8cl@ zP|tdYr5m|(^wIhb3s3lYYTK~hatgUyXn4c=DNUH{z<=ucEoj4ueYyhJuM)6Dz%~OL zpgj-WHsuZE%x`=(Bi!N~uzoyz;UqN@f2AA%_8ApyoK!F#eqQtDS1q^-&*N<1JE?hN z*4QTGf){jsJQpFcKREo3?>+eVMpK|U{*@~JQEJB|qnf}(8fgaH#3?TG8sU@{WkR(@ zGwCt8e^oRS|4Ld<2d+8fVtVsh4L0q}A%{Flr9;y)*Tld2#M*73)*(mcF&;7!!rVyh zv}foxIh`(`-A>9J_WSWA zepyPjBO)ap7?V4endH<=JQf=%&X0ENG0AC}JNd5wNOAs5M+C5SI{!}sX?jPlA}g%z zF|oa43B>DR8xP+N9eWfsQe!ivcD%>6{`bVK>2R6Rqu@UraEkH%j%L8t_(JOMc-AZ{ zMr5pxR?1l2@xED>Td)5gq$Ae?-KW+@i{pHMN3#V=;@_m}JDx=?Yy6Az;*R%Ny#7y< Wj@%^Zh%d=C!%CgmZuE%#6;x-*>`S*F^9r9_JxL z6c30@Zp~a7tFu^U!n(Xk1JMu~-E* z2Kn)z3{V589zs7cE)HW}Ok$^_X!rJ!k`c(Nl?3mOZW3!-F-GWzvtS?4zZn}Ga? znA{lSRp2v0<)AE3B`6u>0M&t}gD4e68LI)G6U*m-TS2KY`KB1J0G}1_$pv8sXp&Ic zmWhQ!Cjb0oo@O2$tD7cH@O*B@uQf9rSV;`e2cM`M%(}HW2IR*0s2C?!21<#^Bf)95 z-Joia2UG|u0Of%s&}>jPh>3%SIou=$4JG`)gu`e>o_^G);19u9$!> zpV8X(YQH^HU#x~P2T&RSUx@)6H#hlZBR7azV>aKfseDwc-Yk}wfKiA5h z)<&$;YMc_6QgSs^(P}MB&E-Ry+^3Zr$QRIgF!gB9BWcAt(Vg)q!jI$61$!xo?I6GM zG-k-i<^iL`nVvagD zexTt5v|M})^fb`E1hi75=fKZ7@N+CeC=#kf9ne~!mITx-9sqhjPqr+wA)HhudE78rC!A13lY4+;-%pkbc?8K45(a-OM*< z4_b=E_pR2Rr>xTrPZrD{!By2So0%K?^?}2>DdNt8alYq)UBR9ZN*<^Q>vOt4fE;FW z^Dn?7z{C2i?#p`aJb6R)>(W?eL_exKtj~6S23giKH?`TC)ZWa|CP-~{QH)p1%{h>< zk)Tetlk4Q>V`vlRBxgCFLm7rJ=6x#E^FXtX$-Pk0yAlPL(Xp%_sD4Tm7MA)<3K`)6 zj?>2Fz}QH3cuh(zbpTA0fja^wrG^w6*mv!Or_0Sa^0WVdj&Yo?S;bjP07M3uQLhL_ zQ=g2PVWL<>Usr zc@|j`oiXP~)QxP@DKab>1-b)e6a@jsTt1BVWD*n8ZH0WKSZpgUC0B)pRo5!bEjiAs zs7+SvAPyJzw>EZlqU^HRXDj1Z#0gtIe@$Gp<(K{)HSFhAH$_N9uR}EHoTq@vpZ-rQ zBr5GT{wJ}>p3jeo2kccmB=*`%_-65rJ%3V7LO(a7pD%_nGMsZ1Iu$5OoW?^U(_uBg zK>dbQcbcel)9x9`%dMO87A;Xp?MfFjfRYw$lQf!3R%@A37Yk7X92&$iEct zIHr%7juI^tud6j@?>399qH*xZv?3dKip51^EiZ-YXj(~$xg9|*r!~nO3_pv?9{Q~uTU0Z?2r~}i0pTVFlgXs zpo{6`-m7=&y883kARiWg96S2v8z^qqhLF`Q(6oP>QP~@lpMcB}_n7JKoybRG_z#g^ z9>X6+9tV9#Yp7gQjhj|^5c(n*@)_(B6k9pe4tbcD59k*ZC0Szb3`yQO*Y;54TBOVZGIP`4&&t)jJ!yXb}yr zn%RXu74>FGp}ibSM%fD2^;0+xs_yVMDHu^Zn{@`ZXc^8ov*Dzj$rp9N#wNq#SlQoZ1kynb?|0Hf!+~F~wn$dq09^Z#Hvz$8$~oGvsg< zWT#GXSmfUKsGQ`<22FmK%11KS8L~v5*Ny4wVBSiFHlG1lat4blf^AI01C>gds)}us z=FYWSSc?Om)8aC=13+qQab+EP-)A44`pgn`RNV$~8TFAwLB=9uIso+@LvT62auc>LX z3_RRuQnnQ{g$^krg6%E?8xheLbdR>YQA>l!xBR{q?mIwRN}l8yruv=rY_P!cW^JM6 ztykrN9>eqMPF%@x3`ATu5-0G`(HX%iAa7s8LPn&y zBl>(zE<%(Z#;i#F>S3A@3}i;H9TR>#1<-Yzm~bgIChV4&>2b4Z2u2v^D8M=&x0Mq2l>S5egtet4(xQu$QOM~Gr)x+`O?E3PX z;gnJyZNit-Ltz^%DUJpP8&qBYDkfE0&EsJcon`yQg35fpPuyGC$XmqAl~xmN1VrME z%0S`KK~%$j-_Ex8{wHGIYhtcQE20C~QqexerZZm?Kbdj|zex;FS&@T9>&K&gHtmPB zT)Zhv`IXpI^^oyCluQzrs>Ye8D2y@O*s6rhRmrc4MXs#UC2+5Vik{OP~5651FHX3L=Dxu4sJK#Wlw=e&;mGyh0&f5 zi>|4z= z*dTw$70v_AYfx^7JOZ2!7xK)_H(@g!hrJ8lCVe!&b`0gWRJV+|E5K?03prT}2hKpN zfJsi~b7vx>-y(_RuI;^etp2)H`9ICTahpy@Rde zchJH)KMxoMb2_X$K~D>xHV8rYb|<5R(+J>* zDa)6O((5SQrq4%zI3;7P&f@f=d7<-{u&_SYN!rqY%>+hgd3c2sjMqPh(@~ozoQmg0 zEGX48y|a}WoM%hvv9Z7jr!*I$tyh!toy#HT4XAhd!5H6we zTb1BznCn*CzO;Qkw*z#Flt6Z(HP_=Rg*S-N2j@pb`i$w0*P+mi#Yz>oVr4{HOLx>k zaX&3qDEqs(cg8xsLD?-LGjjAs;rA9}vSXN%H?lDbcE*sNs~pxMszSD~~` z3QD?x>h&5n%6X13A27%ffM1Sk1JTSchMaf*wZ(>g6<$R%*BGwoRy!f3|~)ud6|?u zG@q=(I%P?)3WxUvv3zDD|GC&Vv)=sH6)Ka0v&2PUE>StFYSxFe%Oyo)cZOmhom`o;^Kh1hvN*NKZHM zK?z8t8e4E`fi!f%h%&<56*9b`|#t@fVy5Ziuzu`@z?OORuQv zi{RT|Q&qeiU@wAC)#Iy?{Wq%W1+X1_E%?}ws{R^WI-;rujP{;4RP`S4w~wMD@X#?; zJq_-CPgOV49`+|yU5$NBdLIJ-?*M;>_BmV)E`VPF*FG`*8HQni2XJqe!21SN_4Z_B zEQ3%WPOL4iOT&vlypO`ioa8YUu!*4CfmINOM3@9x3yi*mQL=+tK+lWHb;Xm8q6D^S zDI32#K_394mdul3*`s@bQx7kKj*IqnHM;B`@ym5Ze7pE~T?xM+jO%N36^F!(_2X+d zqD}`qx*HyziGiSfAh{tU@Rp%*&y}VZE;luZFPnB>YHEC{uj%P8nr8X`(e%v4re`lS zJ@km|v&C$$m3joPm#^&Tw`wPa@+W@d?qtj{uzMeZIk zxW0JweB@sJC7h3PN;5%TV5VP*>J1IXmyq>}hc`5u`;c>CI>(B4HcXxHWXeaUPks33 z58gld-k(1Hizh>t4W10n5@BiwUWZmuy3xfS5_fGJWqc8sNo?9^Gro=7X|ZRc z-$+v)7SlFuFou!aE)H+nFlh;h{c<~JLv=DhTi>V zPN>BNU>AVJzg^O2N(TR)6Yp;>F-}BorQmH@Q^@BXmsC~z5*V91YQF^dGr-4EJ0$W8 z1)!m;XLg%T7J;_V^7yBr_?Mux60Iu3bZQBEgO=hleYT>M7G=z2jb`W*B_-GWQBGeL zDJ>LT*KOP@-nm}S-xlT}8^<@`q4La)@eeW4YM7vVhisWIk=aNHb}g+I&klW;Tf}*^ z+bGP#KA%0--?zlE3w)~v_^^(@O#{&Xw`#yR3XA{G11rTrwFX}=dp=XY;=I-$;oPU| zOY+~X>Ih;kH5ugWs7Np_w0<)0f>zP0TX1^zQ#TchK2>?|XmVdG7yzp7WgN zJm)zd_lmzU%-B;o(9T^g)i%A&&lBowk0VS@&agTaBgM{9<41(3Jk&Smd} z^`CJQp7L;~;=UR8SlpSob8ypT#%;x&k9!Di>d_?JdfXFn8*q=p9glkgiwoZ)=-4OW zV^hWhlMd+$xclJN>1z1zB$g2|QYd1xBeI1KRuhpV{LFtn>>V)C>c!L{!h8wKrza!X%=<3J^?tS2ahnC&j zRNmLAXD$5_Tmr(_hG-gj7%{9%mh{uY#UQRsUJ^yab`6-W-sGe~#ucX1WwHF&F|0b)Ft9P0lEE_F)J9EdD|P94ou>)X# z1#1dn)7Vw8nf(#p5OyTX>W}z>%?)8Q*aEPt!Ac?QXyygm3U*!yo5i+)Jqxxpgw1B> zzz$A8d_!0(OG!X{!LABn$Fdn*sURK9yKgiEO?;8$^_brGf+ z73k!6A7NS>13r$o6UN-q1sXYSAxsN-pn>Ce2}cmF=lBg55wxHNsyXorVaohKImgcv zrg<4C;dm`!Ol)1CnB%7iM-#Sj`~=~?gmXB4jBpHLGsjB_W1-Onj2ye}CnA;z9VZqN z772G>1Ce$oVOpdDevWS^97nj5&Yq2me*31<`T{*}t_BuuA>fS=>r3DZd;(8=+1 z!gTrw_&AgF5!~v z1{T-jh?2{E!%KurHq?^hTItp4obu_5uVPy@=-O$PD#>tP-sSO$eCc+|-zs~5QPp;u zV9L|zt0bSY%e$0YVbv*LlFxgK!;}%o0*|k9q4iSBILth%bO@@TX0 zp>VV#8mcmj`|dgum;$7gL-`nd3XB6|^|jIZqQ&n|nt9JHRB;(9)+&PN&H_S@UPBes zdmODsw#hApQX{c=%=(?A(f38VRDT?J?go*>N~h6d9z-98d3Cav2;>jU})E6 zRc)q(M`$Ja`->a30eT!3!O#GWy`l}!DH@=}pi^(5271;>F@%+NsG||jdeD5Jh&=v_ zn#0)|V$Jx0tIN1%<--`PBI`LN2~hG@Sc=2jE4z2(&_V0Dp6I!Y0=G+WHh*;yAh;g=*|)rK6-)iHt3x>R=3<@ z!0;*k(%8eJlZByd_2}W^)gM*0jW%x@!tNhg+vviX&o=3FKQ104rny1S&Qb<<>YcKGTJ26@w@x=L?{MXXUblLED)Mwyp+}!8;b`ApdrhqX`dPWuP3S5| z=M@D=s@X~1bjUr_dHVwUUG}?oSoyr3*TU!Zzoz?hIyVH6U$zE0QW)M`&6gbZK*&U>*iYKIpw|6yfJZ- zr>jC_W^1C$=#b~aK>PPt9-v@wqB$c!AZZ@W=N>s_*jApx>Jx0GQik4q?pBzY+*$^= ztg5_Sl0R1#-p~_03s4|=9hTo)wb?GoyCnoWUHR-hthvZ(B&X=P8G5y?d_eW0C!N=7 zwa`0bn;iqBJQ!ls9mR{?mJG}ZUM$Ib$f{A98mjgsU&LE2_mf%;SxFBq-f>O6gD(j3{b>j^sQKt7s0E4VG+gZQn+Cfm>l1eC zb-8cj|(FKzhYZ-~cV@(5M^7!oXc5#(}^Ekz_yqV6? zPFZcCvlO=8rpF07Ear=^A_Ea(r|fcCOxoCrFclnIa2_97G>e|UBh|2wIpiN`oh(71 z$wIEw@rBt5!p~c9T5_bQea&?0=Eg3*N$rGYF69drLJcIOS`KYacd(>)p=Lx(&S_QBHXVfjm16;i!AjG7q)wHNViuBZoZS z;uN2x(+l+GqCgOzY~y;Z7ANV|ss{1N&0y->mSPZa%~0DDEk3z{xYd@k5H+sL)G^E( zNqn*fgj82yF}B$8L}9j3%nhhZqn%5k%~6gAJC{S_o~t=*tCZlx*Qgc#$Rx=-;CI!$ zRU)7yIqKl6bWivblF+I8wTZt^4Rj-mU2rq9vTrT6lStA13M$Y-MsI{xUo*W!aEGMp z5yO6XxRI=y=tSny zDQ||3fcfX6G*anB1J%Io@DJ|w5j{&#nOrpB=s}OIJgs!%)5SnFnE!C}SsZP}m4zTE z7Cc)-cMhOKJ{q-G^rZ2}Pwn1>zBrQcBt}$SmK-hpcVox;Kk}fP)&@A6PDulQbxOgR zP(>_C!OYqn)HL(l;-^h&giqAq^0^&b`t>%9yr#=y5b+t z_N|d>%FJ(>Kkp&^)o`_Lo73}2-B98rL6e|os&?WZ>msf_%1Ul2aXC+O-&d8vN%7wf%al!m z=%Kl5{a}+x{7Z);>WfPb%S@2}23aJZwMXq2J+WBv@Mb|X!N8{hIy@Pc@^)1u;*k2h z;`s`Lh&RA`tlF8VsH3j&bGq^vnlJL4tIVBNqv}}U#L=!-zD8F(KG#VEX`;Nn_Df{J z)2Jk2u`-MA?Ud~as35x664#6)J3b?8AvN!@g@@6zES13@9NnDqcR2AmGjIUgY?+ND zk-X#jD&=T%x;K3E;u~XLe|?ut!HG?&yg)+)doeJFQ7x@1@_a42+XZ&IV08cB$yd4O zOWrNfz497soj5SB3L}r6rBkGDX`YOlJR0^^>ar3CZ63VJp~;%c>33A)NYwN)lv#dscKtjuf~R%Uy! zy1nd7dvC!~6f8suoiHWYUKWV%^~KsXL%!M@Ww|(9+G8uDadF61IQL@~P+<7tr(D80 zaR!ZG7Ah0XN9l9rbMS$I;1;v3la`K%55Bw|#L`Vi@m%bm2AwhpVWP9%zd4i@pJH03`9Sp#QVKiAj=V3m)slk~oeU$Xkz(OrDt30hkOjrI{>f*Sl4 z5)NO1T{JS09C1LW#Yi7V+xlRfHrTX#?St0Pa1fVdiHY*y^nGw>LNH4_O3En^$aJS9BH=7MdbEOE%HavsgVqgBI+ zq`5%vG{=KvERmF`MD7ZbmvKntO+)IGf1YH|Oim$Z$%-Ew|8!y3np|>~>Yn5N%<#j?CEm{}F$0Ig8ixZM}3qN%%mc$45qST7gHd=|C^3RLKvJT=q!RM8+ z{96ZQ{vK6;#zKoq{CJk|j5PU(D6woi(F*7(Wvu4b{`qFD!mYsyJt!&EGVXy`CMsHPMBlX*E07z#rJ&Ne7`^?i+QaZ?Ik`>QR*~_~1htHfW$YTx(_* zQIbYo{-f5CKUm|bU=7+*tvf+L%#Ls>#>tW9;pz z&qN>9I{0m z;nSwGl2s7yXG5l&WAE2;!AZmM?h)pkKEjcV^-(_MIZ+j%WFO(J=6Fx)a4Vj-YD8up(X=~+!oQycL7%u2^ z(U=ki`xoA51BQ)<(K&e!Egg1x$Hs^6J<1;sv*9z#@PGC7GxO8BG*ev&vX)?~^32C* zi>hn`N$)E6vhyJ`~Y>5A)2<3i9R~{B!rPU9(}K zY5eUrK{(K~Y|hL`!QHg$&X=Z-#os>QOHC|3*LGRe-mh`?I}m{LBKDfOcg(>-*03j{ z28uklHayDKzdAw4V+UVdIWU(d>Urhif2u0x%9(dDL9j|@y`IY+cx{ak+oZlG7~XBH zkBHBGLRHVzsOkpXoi0^A5yCEwejThL6=26JB+0A2x2IgrVKG1Z(s!|iZ7wz=)^|OR^H1`+gvqxK| zvcW9|p^T-qj1;!887-srrV930%S?TG1>4s$CZY=CdW2nVvFLptvf=NiB^p+!>QJ)h zh&87}^kC2}E7)D{k4h{>(G)7eCE=z#*tmkd^!{|c!NtCNe^kVNm#Tis2DYZ_{Q?uV z4TzO+Kyi{jhJ1c4TiiNJ@Ur(>Q}KJG^=^T&o3~9D@TVo)iZj-!>Lk*{Mct^z*l!Nu zO2DlHcXS=QvhALX^{R>_;1_AfP-cKbnD~{6Uo*Ig>)Fb-Y}d2@t-S~(N5RooBf7{! z6}Yd#VRG?{%2tE(gF8bLY-&`PA?%c}1COM?31wT5i+{i~UBtD5Ga@hV=8VR5f?GTv zB=$~zk>b}Nw}P7+!qF@XM-qqX#UOD@z?FvRnZdP&a1NA>1DF0Hd%k_DsS#nMd4|h| zyB;O`n^bihEAS0xgMBH&c9!qE6)V`|z88eAS#n3A@D01OqtJy-dKlHhMJ`Ins|s-R zU5_r}=7TH$4Ob4X>^GbnTq(FDUQ?^PA!sj@+YGKcShs21fSM>nmR7^ZUjOiJ{GUtI zezS3tMhy(w+Q9Pmry~9TXMdq!X3yby7Hi(09oKVGQ$SAgYaYA2KQ+!sZsWo{`gg$C;79boI(v zOgNMsSKd!!d*$QIekePtJIKQPp$SnX!N`@c=0g*%JJRdWJSH4=p&K=avsw3HBl8{3 ziPu>)tG)6iH`9NT9Tf~f!zPyh$)%{?KYc*IFzpC~9#Bgh^+S z4LxqPee6#Fe%DtE`A6#`8N%sJ}cTwdpehHXZukvMkiV zOQ}YV?=N*4NIKkGw!AAB$p3Hc;y~?UiU@g1E}%srgA#8+)kNH*15M|`&Nnx-9d*#vr@+a z@x@xKE#vNkTas$TZ$6C-8zJY@tguY7><^{7uvqyt)r7^%i|E6!OxZ-o@VrYfaTL>M ziu_0D#>;C3vrqgU6~D{)ml`7zXm#I&6@3%r_XUyFH;byn6XZKYwbk@sWTrfupQ$S% zL5`%eeKTdJklifUBIrIX@JjJJL>RDDbn&>*=z&ItkZFQd1ZziY9RkZj_LS(oDI!ar zD{vRdk(pEmEw_*~_DvwmzEIj5F$S&iAm1b#Y^xMfbDvOoD1B&(rPP}iv@E~r4ig>e z*Wbm#j$6V1YVi031Z^pm8%k}trmT>c{5+eg|Jz%2rYN&5o0QQ%OiCxAZ^I7B)b zz!gc4L~yh&0Q1$Gl^k7(Jkf(9syQieHPiCw_?Y#9tQB&wQ|MpOaBXyW%%{JQ)RL^k zuTa7h5Bx&V(Jb1NdMH(iPt<-PUHy?EJwvmUc=;kdqSVScbPCV;lx&Wt!RDxbmLrBv zUlb;q4%P)WD~JuFMPRGI+Jo3|dLQf>uth;^Bz+C`FxcWCHj4Vi#8XsERKKzyHk$Im z=7FsaV$HM=Y$@2rAhtif4)!&$O+jodeGax0Y;zDBM_0jK0qYB56DXs9JdNoe)vqgv zO``k2&I8*M#15tgu*<-n31WxP2C(g5rK2}|W}#DHzXxj$Vuw=l0Qh$R{2Rok(R8pb zuvtOu2wDWT3amYd9ZBzlT?4i#h#gH|gFOtkIEc-lezEXxEc_e9W>P-bJh0V4tc@0e zEd|>c#NI-$gMAHbQxH3zJ_p+gwmFEMKv%(D0qYB5b17pW{5uf-4Px_J9vt}BNGcnc zx55(Z_DGUiyES|Zdru$#%?A9Y9mlVdV#gEX1uXLS=(y4jGUln|?-ID1F|UPwpTN5q zhcIpycr#;Oy8TT8w=oW5+$iu`Hxs;o`l|);E@RGnf0@8}ZW6eSaT?>stLR@(Tg$|7 zCaQ(PyNpLLE))1o#_5cU1%8F`NXA71uVBnuwcjrA9~kqYz@H`XV#d76`z-=L$2fzr z`6~L?)1G8vEEAGYc$9G_<1<%S{t#n6H~4!5zK1cNFZ^8s&t}Z$3%^g`X^i=F;%^rC zHpYB%@HYvZ%b3q3{>IUQxS5GtnWz?c6yphu%dR&tzmYBuTOgOxgVtpC_6A9Ec~6{s zyFZ5mAVZu9t*rMy78y#LX{QF$XzCT%yyrFxG#yxX)|kS^~9@9Bo35N$pp z?(tPTWV_Hd4zr9a4KS#g9nEN!iJJ|gpo3?D_q5i{QA4G%PJpM;@O&l3DM_t-WU_VJ z(6Uy9KD=8zzGt7*Dt^}W+U(Hs9_=J7QS0vyR~yD3TZTvPQ7r+rVQ}3SXc_dL04)dT zPJ`|Y(BGj|*4`;mRa&z97GYOqW_b=;nhcp?2ZqA#*a&avnp?EwKIxwfWXPeM2xb2` zFg9NYuO){6T8!pB6S?BwQL$E&mAVWdjOYZ0ftp_+uTdNT3bpCl8#H)$f}Bm`hiA(V z&|||B+;^iMk6c;$9n4c3GXDKXc(LIlYZs3N4TW9S^;S-9xK{Q4aPBQ5Kpkii3{lIL|{p`4HvNPvY#Ua}T_$c`@9UC!N{wrM@F-X3hlF|ps|Djvc zQ{_UMo1SbMahaY;&y+9Fs`MehU3C#{BZZ$G;Hb^BhL(Sejs{#pN3X#RqGykK%C(3`+U18SGIOY0 zO&OU916Q3D9>F+!?!C`3Z;KP_6wy7IquiClWSIY&wHT)mwPB653^R6z)2g!7SaR^_ z;zw6QmGvQ4!&}xa4pYdFy57bwBen}XkLV|yJL;_4`59O3 zrqWIRkaOyL>pA4}y-sVfHjP5E^5qh8WM#O^*pi%W6@Im0rF9<6X>@serq*ti(ZJN4 zy%lGB&FZzO(9>Os5q+sS(Z8einqCg{lXj~>R|Pt)rL%1{TX~pH??oQG<37jzjt92b z#JpD9#Js-kif$FjKNgOz$M749^cqY7M0v>|P*`VN80Z>`vBl+tS~U}9g{wZwiXhK* zU|Kn9KgYcr_b%PlP-iWH7whL(RhReEsSQyvs;9d=1{+ebEzS)aW38HW8mmKntks3i z!u^}E_8IoLggq{AlvQiMN@Da;Q4cMbFe${sCRM^Do9^ALdXH%Hud68@KJ%#!PU{q# zG0|1;b`^e3&{EGBZK)nNW_T{tYTHF9O^RoXsMKyffUtBp5E`xgMw4~XLRx1{#AFas z)w_e+4A*7^X=!5dZnxgUS~c88R*-a8Kzaz$X@5Keymx9+IPagpj;Rr zs8ygG-tAg`!0U(x$B1^`JlM^iR^f?SpS;~9xpI6JXE_y=x)JEiQF|3n$#}AL-`M%t zyDkjDtZfUpg`;5h_#EEN%F>lc2=-xdD<6|xUcHTvO4w~BH$$h@a_(&;9bD`3x~y)q z25bM6v zeO;Fm3KCO5`%^sMRA*IDU-5hb0J#%DR+jMI?5uyyng{5p{TUNto;6Qd!h5r`zFlkt zj#@oZS;FgV{W9xe6g8K45O7xZGf$}X+%}eC{o^ zew@`fT7AQ=g=ftig@m6Dhv)y(>{L9%Q6i+?T{u~Z)D-C`(ooZ}#oow-fJYV2IiAZ1 zKmw-Ef{o?A@6kAo{3?lQBx?!^*rxT^r2!&BjkXB?jVdjvcwLl~+o zNfqcckPgHLdZYSKQsp^u zPMIWZ5hp%lLKL5)4<pqezu(wQ&=2^G2kE3Y|ggwIGhn*(F%_vba(-4#2PvV z$`TJv9Fw(N=oNZSRb)@C(IwLJMlXEm-x!L==l&W_%JXin{l=|$`k<1^sjsZt>UG4T zg5k*8D(e^omY3i*>`2o3KKSup#*g;ix1vu)?6zVoy`4LFX^=3oI!ULLg297+4OKC$ ziliHcmB0-D_jGr`Fw?8Y>30Q5iLWED965{ag2UVC*ubseL^-#TJ}4OZtINGoZ&8B( z6hK=D zH$Qu-MN8iH`%ug^UI|&|@?J)grfZ)-A?%=m_?Iz>^zY6^mv{rDOitLGn-{8m<{QCPX z2-Zo#8?m;Z1IVwxF+k!q?kzAZmRLOE7)IDIu)K^uu&0^* z_wO;3p9-+yf^9!)u=j!0UcheO`wDx(=G%zj;te*?S_w2Z4hIv^5N}>E_%hl>CwXj$ z6FO~X{(jie2maMEkCOYgxbVk#tJI!2Mz7x%pLm2v_EVd=SBZxR>9Hg9xgkYk;i9_G z!9fqw50B8*+lH4K#x9+KsRAu?2dAkP8ep@*c5qakUR^tlZeY>UVN7zi^Q*wY02$3B z=NpsT0^}upR~22Ol(c^wp%-pXW@EXP9&qA~-(ht5>mJ&6``9UvI%>Deg*n$Mj^Uke z!a`nb&MChSYkC=bKfYQNqv!dm^4O=4+92)V6~yKJX}(griTN(@ISVP*IcUrSr~-5g zv{=%EXBE##lP{Q3N;fmDfu6jOYMcXd(~SzZ2P#~IWRjt|rMJrhV1AeOVML-kw(a6e zwA&fyhG=Td*5z!VQo5B-9ofP*9>`6BdVGN6D(x@v)HL47Ar3XV*~Szf=s;a6HG1j^ z)L0Ry!Dk$s4?S#WJ9vNN_NrdOe=UOC!6YxyOqQVvxJ}HN16+y0ZD!5_PHQ<#uNJ0- zqc^4qkwf-Y(V#>?XB zLq^JgW_L*0cCJf&G|@rY6v@rp!ux6iqfAgs@c}+gY<#=LgZJ* zz_s6eMN_60Ozw+yj~yWrUAy!Hws~YRQvS5|J%A9wrIL1}7m3-_)r)v0X`AV@sTTP_ z%lA`lviw^ba)%}Qw?;0wI2>!gB-b6ood>bFd5?MyRfcJqUy7!)JUYNo0Cm&PmVj)4 zAsZ)TeE@L460ha>2)3Wr-7yK{yl}@n`AfRf^_JBaq|O)rr5(uf(8@%z|}2kv$oYf2q|?GS zB=SGM^WhtPpkZkr2It+uONRr>{5f)mcJDqKHnUXTLya?Y)Akrmb;HRC8xwQN5&nMzf1^BDU6aUb4%$1XC`M~OG$ap7*7Oq+@D=qpeIs9*B1Ld{N zjbZ&4*XVj4?w{PcelPA}H7##`{G;s7D%Euhf^SE85eZ#fuIrzoui^?_&jw$H(uJ}M zw61SujPeZ1tY>uHj!bVxS%%VFt?NFNc9eUWM=5dcqwiRh&5L!N|9YYcxw>3u%euY>W!Ik(Mr<(VS9EvbrbQNG+4FPP>x&=H7gls=R_DEFc)dJ_$! z^r5_h(*8GHH}jgeUf1WMw11-OrHps!dNY?;FwUdQ!n<-KjwL-PyHJ*4;V?&H#3-#O zn^9(?^r3VO#6uSzicy|HS&q_thK9F}k{fAO>n-^8v<^!Q$0>}v=)I5~(mdQ2+SEG0 zH25icu5B_6ZVQ*kP+HpvxtylAjW!)%dWK1RigvY)4a-Jg=h4+RtEqVh4f`Z5POZ}Q zRBq52ZAnLQ5m7IxqWeD?6}K9q$t=R9;^ss+UPXWTWSU9+J$?7dsIVxc(2JDVo^JA) zNZt@1g-Jh*-kYZibj&I7qBKS9z``mzb1$) z2FKrs4iH`5@bJh#={j;iT&x|%i2)8?64w~~TEOM~iJtGsbieq&+9i-20GANZ<`yc! z9S5fbIF>B~*8`6KN<*3w5fUD9RNjI|)?W?TM@Yp7@yr)&6_e*_#>BJj`gVnOgTwpys`1 z(~(*-?aPb^47_O- 0 && r.worker_uid == 2200 && + r.start_ticks > 0 && r.output_length == 0 && + r.diagnostic_length > 0 && + r.diagnostic_length <= DBL_MAX_DIAGNOSTIC, + "worker failure diagnostic length"); + char *reason = calloc(1, r.diagnostic_length + 1); + check(read_all(s, reason, r.diagnostic_length) && + strstr(reason, "grok: session store unwritable"), + "worker failure reason"); + char extra; + check(read(s, &extra, 1) == 0, "worker failure EOF"); + free(reason); + close(s); + close(p); + close(c); +} static void org_cases(void) { check(!setgid(DBL_BROKER_UID) && !setuid(DBL_BROKER_UID), "drop broker uid"); client_case(); @@ -90,6 +118,7 @@ static void org_cases(void) { client_input_reject(5, 1); output_boundary_case("exact-output", 0); output_boundary_case("overflow-output", 1); + worker_failure_case(); int s = connect_socket(), p = sealed("prompt"), c = sealed_bundle("provider.Token-1", "mcp.Token-2"); struct dbl_request q = request(); diff --git a/src/runtime/native/engineBrokerLauncherIntegrationMain.inc b/src/runtime/native/engineBrokerLauncherIntegrationMain.inc index f773c55..ef59692 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationMain.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationMain.inc @@ -114,7 +114,8 @@ int main(void) { DBL_RESULT_FAILURE_CLASS_OFFSET && offsetof(struct dbl_result, profile_applied) == DBL_RESULT_PROFILE_APPLIED_OFFSET && - offsetof(struct dbl_result, reserved) == DBL_RESULT_RESERVED_OFFSET, + offsetof(struct dbl_result, diagnostic_length) == + DBL_RESULT_DIAGNOSTIC_LENGTH_OFFSET, "result ABI"); mkdir("/run/daimon-engine-broker", 0755); mkdir("/etc/daimon-engine-broker", 0755); diff --git a/src/runtime/native/engineBrokerLauncherModes.inc b/src/runtime/native/engineBrokerLauncherModes.inc index ac78544..012da6b 100644 --- a/src/runtime/native/engineBrokerLauncherModes.inc +++ b/src/runtime/native/engineBrokerLauncherModes.inc @@ -2,7 +2,7 @@ static int client_mode(void) { struct dbl_request request; uint32_t prompt_length = 0, capability_length = 0; unsigned char *prompt = NULL, capability[DBL_MAX_CAPABILITY_BUNDLE] = {0}, - output[DBL_MAX_OUTPUT] = {0}, extra; + output[DBL_MAX_OUTPUT + DBL_MAX_DIAGNOSTIC] = {0}, extra; struct dbl_result result; struct sockaddr_un a = {.sun_family = AF_UNIX}; int s = -1, p = -1, c = -1, ok = -1; @@ -37,7 +37,8 @@ static int client_mode(void) { 1) || client_send(s, &request, p, c) || full_read(s, &result, sizeof(result)) || !closed_result(&result, request.turn_id) || - full_read(s, output, result.output_length) || read(s, &extra, 1) != 0) { + full_read(s, output, result.output_length + result.diagnostic_length) || + read(s, &extra, 1) != 0) { struct dbl_result failure; memset(&failure, 0, sizeof(failure)); failure.version = DBL_VERSION; @@ -50,8 +51,11 @@ static int client_mode(void) { ok = 0; goto done; } + /* The trailer is the turn's output on success and the worker's bounded + diagnostic tail on failure; `closed_result` keeps the two exclusive. */ if (full_write(STDOUT_FILENO, &result, sizeof(result)) || - full_write(STDOUT_FILENO, output, result.output_length)) + full_write(STDOUT_FILENO, output, + result.output_length + result.diagnostic_length)) goto done; ok = 0; done: diff --git a/src/runtime/native/engineBrokerLauncherServer.inc b/src/runtime/native/engineBrokerLauncherServer.inc index 75f452b..1d4cd76 100644 --- a/src/runtime/native/engineBrokerLauncherServer.inc +++ b/src/runtime/native/engineBrokerLauncherServer.inc @@ -225,12 +225,25 @@ static void supervise(int client, pid_t pid, int output, out->output_length = (uint32_t)used; } if (out->status != DBL_STATUS_OK) { + /* A failed turn publishes no output, but a worker that exited on its own + account said why on the pipe it shares with stdout, and that tail is the + only reason the host can ever see: without it a failure reads `exit=1`. + Keep a bounded tail of it and erase the rest here; the broker redacts it + before it crosses any boundary. The other failures get none: an + output-limit tail is the very payload the bound refused to publish, a + cancelled turn has no reader left, and a prelaunch failure ran nothing. */ + size_t keep = out->status != DBL_STATUS_WORKER_FAILED ? 0 + : used > DBL_MAX_DIAGNOSTIC ? DBL_MAX_DIAGNOSTIC + : used; + memmove(bytes, bytes + (used - keep), keep); + erase(bytes + keep, sizeof(bytes) - keep); out->output_length = 0; - erase(bytes, sizeof(bytes)); + out->diagnostic_length = (uint32_t)keep; + used = keep; } if (!disconnected) { full_write(client, out, sizeof(*out)); - if (out->status == DBL_STATUS_OK) + if (used) full_write(client, bytes, used); } erase(bytes, sizeof(bytes)); @@ -334,7 +347,8 @@ static int client_send(int socket_fd, const struct dbl_request *r, int prompt, return sendmsg(socket_fd, &m, MSG_NOSIGNAL) == (ssize_t)sizeof(*r) ? 0 : -1; } static int closed_result(const struct dbl_result *r, const char turn_id[65]) { - if (r->version != DBL_VERSION || r->reserved || r->profile_applied > 1 || + if (r->version != DBL_VERSION || r->diagnostic_length > DBL_MAX_DIAGNOSTIC || + r->profile_applied > 1 || r->stage > DBL_STAGE_ATTESTATION || r->failure_class > DBL_FAILURE_ATTESTATION_PROFILE_INVALID || r->output_length > DBL_MAX_OUTPUT || memcmp(r->turn_id, turn_id, 65)) @@ -343,12 +357,14 @@ static int closed_result(const struct dbl_result *r, const char turn_id[65]) { return r->stage == DBL_STAGE_OUTPUT && r->failure_class == DBL_FAILURE_NONE && r->profile_applied == 0 && r->worker_pid > 0 && r->worker_uid >= 2200 && r->start_ticks && - r->exit_code == 0 && r->term_signal == 0; + r->exit_code == 0 && r->term_signal == 0 && + r->diagnostic_length == 0; if (r->status == DBL_STATUS_PRELAUNCH_FAILED) return r->stage >= DBL_STAGE_PEER && r->stage <= DBL_STAGE_EXEC && r->failure_class >= DBL_FAILURE_PEER && r->failure_class <= DBL_FAILURE_EXEC && r->worker_pid == 0 && - r->worker_uid == 0 && r->start_ticks == 0 && r->output_length == 0; + r->worker_uid == 0 && r->start_ticks == 0 && r->output_length == 0 && + r->diagnostic_length == 0; if (r->status == DBL_STATUS_WORKER_FAILED) return r->stage == DBL_STAGE_WAIT && (r->failure_class == DBL_FAILURE_EXEC || @@ -358,10 +374,12 @@ static int closed_result(const struct dbl_result *r, const char turn_id[65]) { if (r->status == DBL_STATUS_OUTPUT_FAILED) return r->stage == DBL_STAGE_OUTPUT && r->failure_class == DBL_FAILURE_OUTPUT_LIMIT && r->worker_pid > 0 && - r->worker_uid >= 2200 && r->start_ticks && r->output_length == 0; + r->worker_uid >= 2200 && r->start_ticks && r->output_length == 0 && + r->diagnostic_length == 0; if (r->status == DBL_STATUS_CANCELLED) return r->stage == DBL_STAGE_WAIT && r->failure_class == DBL_FAILURE_CANCELLED && r->worker_pid > 0 && - r->worker_uid >= 2200 && r->start_ticks && r->output_length == 0; + r->worker_uid >= 2200 && r->start_ticks && r->output_length == 0 && + r->diagnostic_length == 0; return 0; } diff --git a/src/runtime/native/fixtureWorker.c b/src/runtime/native/fixtureWorker.c index cb92d4e..d2f3674 100644 --- a/src/runtime/native/fixtureWorker.c +++ b/src/runtime/native/fixtureWorker.c @@ -6,4 +6,4 @@ #include #include #include "engineBrokerLauncher.h" -int main(int argc,char**argv){unsigned char bundle[8200]={0};ssize_t size=pread(4,bundle,sizeof(bundle),0);if(size<6)return 21;size_t offset=0;unsigned provider_length=bundle[0]|(bundle[1]<<8);offset=2;if(offset+provider_length+2>(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output");if(!strcmp(provider,"sleep"))sleep(30);if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;id_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s tmpdir=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target,getenv("TMPDIR")?getenv("TMPDIR"):"unset");for(int i=0;i(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output");if(!strcmp(provider,"sleep"))sleep(30);if(!strcmp(provider,"stderr-failure")){fprintf(stderr,"grok: session store unwritable\n");return 1;}if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;id_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s tmpdir=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target,getenv("TMPDIR")?getenv("TMPDIR"):"unset");for(int i=0;i Date: Thu, 17 Sep 2026 23:33:39 +0200 Subject: [PATCH 23/69] fix: stop naming a healthy turn's own two proxy requests as refusals --- src/runtime/AGENTS.md | 15 +++++++++++++-- src/runtime/grokBrokerProxy.test.ts | 17 +++++++++++++++++ src/runtime/grokBrokerProxy.ts | 29 ++++++++++++++++++++++++----- 3 files changed, 54 insertions(+), 7 deletions(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 0863d31..7487b5d 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -163,8 +163,19 @@ refuses a turn whose worker config does not hash to the declared one. Three capability reaches the model through `env_key = "DAIMON_PROVIDER_CAPABILITY"` set by the native launcher (as exposed as `DAIMON_MCP_CAPABILITY`); the per-turn `session_title` request cannot be disabled by any key, so -`[models] session_summary` points it at a hidden model on closed loopback port -9; and effort is only sent when the model declares it, so the declared effort is +`[models] session_summary` points it at a hidden model +(`GROK_SESSION_TITLE_SINK_MODEL_ID`) whose `base_url` is the broker's own +provider proxy and whose `api_key` is a placeholder too short to ever be a turn +capability — so the request does reach the proxy and is refused there, before +any capability lookup, isolation guard, credential read or upstream call, and +Grok falls back to the truncated prompt as the title. That refusal and a bare +unauthenticated `GET /` probe are the two requests a healthy turn always makes +and the proxy never forwards; neither prints a `refused:` line, because for as +long as they did, every healthy turn read as broken. The sink keeps its 503 +shape because every live capture was taken with it: forcing 400 and 503 there +were both observed to end the turn `exit=0, result: success`, so a hard 4xx on +that request does *not* end Grok's session. And effort is only sent when the +model declares it, so the declared effort is the model's single `reasoning_efforts` entry. HTTP MCP needs CA certificates in the image even for a loopback `http://` URL ("Failed to build HTTP client"). diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index 79e8685..05caadf 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -95,3 +95,20 @@ test("the isolation guard is awaited before the first upstream call, and a faili assert.deepEqual(order, ["guard-start", "guard-end", "credential", "upstream"]); } finally { await proxy.close(); } }); + +test("the two requests every healthy turn makes are not named as refusals", async () => { + const lines: string[] = []; const original = process.stderr.write.bind(process.stderr); + process.stderr.write = ((chunk: string | Uint8Array, ...rest: unknown[]) => { lines.push(String(chunk)); return original(chunk as string, ...rest as []); }) as typeof process.stderr.write; + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => ({ status: 200, headers: { "content-type": "application/json" }, body: Buffer.from("{}") })); + try { + arm(proxy, async () => undefined); + const title = await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`, { method: "POST", headers: { authorization: `Bearer ${GROK_SESSION_TITLE_SINK_KEY}`, "content-type": "application/json" }, body: leanBody() }); + assert.equal(title.status, 503, "the title sink keeps its transient shape"); + const probe = await fetch(`http://127.0.0.1:${proxy.port}/`); + assert.equal(probe.status, 400, "the unauthenticated probe keeps its non-retryable shape"); + assert.deepEqual(lines, [], "expected per-turn traffic must not read as a refusal on the broker's stderr"); + const miss = await fetch(`http://127.0.0.1:${proxy.port}/v1/chat/completions`, { method: "POST", headers: { authorization: `Bearer ${"z".repeat(48)}`, "content-type": "application/json" }, body: leanBody() }); + assert.equal(miss.status, 400); + assert.deepEqual(lines, ["[grok-proxy] refused: unknown_capability\n"], "a genuine policy miss is still named with its reason code"); + } finally { process.stderr.write = original; await proxy.close(); } +}); diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index 1b89d58..e13d5ca 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -72,12 +72,15 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori } catch (error) { settle?.(undefined); // Name the refusal on the broker's own stderr (reason code only, never a body - // or a token) so a failing turn is diagnosable without a stub harness. + // or a token) so a failing turn is diagnosable without a stub harness — + // except for the two requests every healthy turn makes anyway. const refusal = error instanceof GrokBrokerProxyRefusal ? error.reason : "broker_unavailable"; - process.stderr.write(`[grok-proxy] refused: ${refusal}\n`); - // Grok's own session-title call is refused by design. It must keep the transient - // 503 shape it has always had: a hard 4xx on that internal request ends Grok's - // session, which surfaces as the worker exiting 1 mid-turn. + if (!titleSink && !expectedWorkerProbe(request)) process.stderr.write(`[grok-proxy] refused: ${refusal}\n`); + // Grok's own session-title call is refused by design, and keeps the transient + // 503 shape it has always had. Forcing 400 and 503 on it were both observed + // to end the turn `exit=0, result: success`, so the shape is kept because it + // is the one every live capture was taken with, not because a 4xx there ends + // Grok's session — it does not. if (titleSink) { response.writeHead(503, { "content-type": "application/json", "cache-control": "no-store" }); response.end('{"error":"broker unavailable"}'); @@ -93,6 +96,22 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori } } +/** + * The unauthenticated connectivity probe Grok sends before its own requests: a + * bare `GET /` with no Authorization header, which has no capability to look up + * and answers 400. + * + * It and the per-turn `session_title` POST are the only two requests a healthy + * turn makes that this proxy does not forward, and both used to print the same + * `refused: unknown_capability` line as a real policy miss — so every healthy + * turn read as two refusals and cost a live investigation. They answer exactly + * as before; they simply stop claiming a refusal on the broker's stderr, which + * is left for the misses that are actually worth reading. + */ +const expectedWorkerProbe = (request: IncomingMessage): boolean => + request.headers.authorization === undefined && (request.method ?? "") === "GET" && + new URL(request.url ?? "/", "http://127.0.0.1").pathname === "/"; + /** The body gate, refused non-retryably: a rejected body is a policy miss, never a transient fault. */ function authorizeRequestOrRefuse(...args: Parameters): ReturnType { try { return authorizeGrokBrokerProxyRequest(...args); } From 081b3cecd0ae8d2ee304a4ef73d2e12e5c7a0ce7 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Thu, 17 Sep 2026 23:38:09 +0200 Subject: [PATCH 24/69] test: pin the no-active-turn refusal to its non-retryable shape --- src/runtime/grokBrokerTurnMeter.test.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/runtime/grokBrokerTurnMeter.test.ts b/src/runtime/grokBrokerTurnMeter.test.ts index 0f28a88..094ef65 100644 --- a/src/runtime/grokBrokerTurnMeter.test.ts +++ b/src/runtime/grokBrokerTurnMeter.test.ts @@ -85,13 +85,21 @@ test("a request after the elapsed deadline is refused, and every admitted reques assert.deepEqual(snapshot.timings, [{ startedAt: new Date(1_000_000).toISOString(), endedAt: new Date(1_000_000).toISOString(), usage: estimate, estimated: true }]); }); -test("a turn without a registered meter is never forwarded", async () => { +// A live capability with no registered meter means the turn is already over: the +// launcher registers a turn before it starts the worker, so nothing can arrive +// before the meter exists, and nothing can make a finished turn live again. The +// answer is therefore 400 (a named, non-retryable refusal) rather than the 503 it +// once was — a retryable shape here bought only Grok's blind retry storm, which +// spent ~141k tokens re-asking a question that could never start being answerable. +test("a turn without a registered meter is refused non-retryably and never forwarded", async () => { let calls = 0; const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { calls++; return { status: 200, headers: {}, body: Buffer.from("{}") }; }, undefined, 0); try { const token = proxy.capabilities.issue("agent", "turn"); proxy.registerIsolationGuard("turn", async () => undefined); - assert.equal((await post(proxy.port, token)).status, 503); + const answer = await post(proxy.port, token); + assert.equal(answer.status, 400); + assert.equal(JSON.parse(answer.text).reason, "no_active_turn"); assert.equal(calls, 0); } finally { await proxy.close(); } }); From 37363c398484c46a752afc595ed21afc5fea39fb Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 00:05:16 +0200 Subject: [PATCH 25/69] fix: state the Grok use_tool prefix rule once, from the worker contract both texts render --- src/contracts/grokWorkerContract.ts | 44 ++++++++++++++++++++- src/runtime/attentionDispatcher.test.ts | 35 +++++++++++++++-- src/runtime/attentionDispatcher.ts | 17 ++++++-- src/runtime/engineDispatcher.test.ts | 52 ++++++++++++++++++++++++- src/runtime/engineDispatcher.ts | 26 +++++++++---- 5 files changed, 157 insertions(+), 17 deletions(-) diff --git a/src/contracts/grokWorkerContract.ts b/src/contracts/grokWorkerContract.ts index c0cc666..fbe949d 100644 --- a/src/contracts/grokWorkerContract.ts +++ b/src/contracts/grokWorkerContract.ts @@ -14,15 +14,57 @@ * run directly and saves one `search_tool` round trip per tool (P0: 3 → 2 * requests). */ + +/** + * The atoms of that route, and its single definition. + * + * Both texts a Grok worker receives are rendered from them: the pinned system + * prompt below, and the caller's identity envelope + * ({@link grokMountedToolNamingRule}, used by `src/runtime/engineDispatcher.ts`). + * They were worded independently once - the envelope told the model to call the + * tools by their bare names - and a live turn produced zero tool calls with + * every tool correctly mounted: two authoritative naming rules, the wrong one + * last. One definition is what keeps them from diverging again. + */ +export const DAIMON_GROK_MCP_SERVER = "daimon" as const; +/** Grok's own name for an MCP tool of that server: exactly what `tool_name` must carry. */ +export const DAIMON_GROK_TOOL_PREFIX = `${DAIMON_GROK_MCP_SERVER}__` as const; +export const grokDaimonToolName = (tool: string): string => `${DAIMON_GROK_TOOL_PREFIX}${tool}`; +/** Grok's two MCP meta-tools, and the argument that names a tool for the first. */ +export const GROK_MCP_INVOKE_TOOL = "use_tool" as const; +export const GROK_MCP_SEARCH_TOOL = "search_tool" as const; +export const GROK_MCP_TOOL_NAME_ARGUMENT = "tool_name" as const; +/** Illustrative Daimon tools for the system prompt, which cannot know a wake's real mount. */ +const DAIMON_GROK_EXAMPLE_TOOLS = Object.freeze(["moltnet_read", "moltnet_send", "memory_search", "memory_register"] as const); + export const DAIMON_GROK_SYSTEM_PROMPT = [ "You are a headless Daimon agent; no human is present.", "Your identity, instructions and wake event are in the user prompt.", - "Daimon tools are MCP tools on server daimon: call a known one directly with use_tool (tool_name daimon__moltnet_read, daimon__moltnet_send, daimon__memory_search, daimon__memory_register, or another daimon__ name you were given); use search_tool only for a name you do not know.", + `Daimon tools are MCP tools on server ${DAIMON_GROK_MCP_SERVER}: call a known one directly with ${GROK_MCP_INVOKE_TOOL} (${GROK_MCP_TOOL_NAME_ARGUMENT} ${DAIMON_GROK_EXAMPLE_TOOLS.map(grokDaimonToolName).join(", ")}, or another ${DAIMON_GROK_TOOL_PREFIX} name you were given); use ${GROK_MCP_SEARCH_TOOL} only for a name you do not know.`, "If a tool result says output was saved to a file, read that path with read_file.", "If a tool fails, do not retry it in a loop: stop and report the failure.", "Your final answer is a private note to the runtime: one line, or empty." ].join(" "); +/** + * The same route, stated once for the caller's identity envelope, where a + * wake's real mounted tools are known. + * + * It asserts rather than corrects: it names the bare tool set once, states the + * `use_tool` prefix rule once with one example drawn from that set, and says + * plainly that a bare name is not callable here. It never claims the agent's + * own instructions spell a tool wrongly, never offers a shell or CLI route, and + * never repeats the tool list a second time in prefixed form. + */ +export const grokMountedToolNamingRule = (mountedToolNames: readonly string[]): string => { + const example = grokDaimonToolName(mountedToolNames[0] ?? DAIMON_GROK_EXAMPLE_TOOLS[0]); + return `Your mounted tools are exactly: ${mountedToolNames.join(", ")}. ` + + `On this engine each is an MCP tool on server ${DAIMON_GROK_MCP_SERVER}: invoke it with ${GROK_MCP_INVOKE_TOOL}, ` + + `${GROK_MCP_TOOL_NAME_ARGUMENT} = ${DAIMON_GROK_TOOL_PREFIX} (for example ${example}). ` + + "None appears in your direct tool list and none is callable by its bare name; " + + `${GROK_MCP_SEARCH_TOOL} lists them if a name is unknown.`; +}; + /** Closed declared-model vocabulary; defaults are `grok-4.6` at `low`. */ export const GROK_BROKER_MODELS = Object.freeze(["grok-4.6", "grok-4.5", "grok-build"] as const); export const GROK_BROKER_REASONING_EFFORTS = Object.freeze(["low", "medium", "high"] as const); diff --git a/src/runtime/attentionDispatcher.test.ts b/src/runtime/attentionDispatcher.test.ts index 71dce93..3e61593 100644 --- a/src/runtime/attentionDispatcher.test.ts +++ b/src/runtime/attentionDispatcher.test.ts @@ -1,5 +1,6 @@ import { randomUUID } from "node:crypto"; import { AttentionDispatcher } from "./attentionDispatcher.js"; +import { DAIMON_GROK_TOOL_PREFIX, grokDaimonToolName } from "../contracts/grokWorkerContract.js"; import { createOrganizationRuntimeHost } from "./organizationRuntimeHost.js"; import { WakeAcceptanceStore, WakeExecutionClaimLostError } from "./wakeAcceptanceStore.js"; import { parseWakeAcceptanceRequest } from "./wakeAcceptanceTypes.js"; @@ -14,7 +15,7 @@ import type { OrganizationRuntimeHost, OrganizationRuntimeWakeRequest, Organizat const token = "attention-test"; const storeOptions = { processIdentity: async () => ({ pid: 1, process_start: "test-start", boot_id: "test-boot", pid_namespace_dev: 1, pid_namespace_ino: 1 }), ownerLiveness: async () => true }; -const config = (maxExecutions = 20) => ({ version: "noopolis.daimon.organization-runtime.v1", host: { bindHost: "127.0.0.1", port: 4318, controlTokenEnv: "ATTENTION_TEST" }, agents: ["alpha", "beta"].map((id) => ({ id, name: id, instructions: "Act", workspacePath: `/workspace/${id}`, runtimeHomePath: `/home/${id}`, engine: { kind: "codex" }, attention: { maxBatchMessages: 3, maxExecutions } })) }); +const config = (maxExecutions = 20, engine: "codex" | "grok" = "codex") => ({ version: "noopolis.daimon.organization-runtime.v1", host: { bindHost: "127.0.0.1", port: 4318, controlTokenEnv: "ATTENTION_TEST" }, agents: ["alpha", "beta"].map((id) => ({ id, name: id, instructions: "Act", workspacePath: `/workspace/${id}`, runtimeHomePath: `/home/${id}`, engine: { kind: engine }, attention: { maxBatchMessages: 3, maxExecutions } })) }); const request = (id: string, agent_id = "alpha") => ({ token, agent_id, delivery_id: id, event: { version: "noopolis.daimon.wake.v2", kind: "message", text: `Handle ${id}`, occurred_at: "2026-09-11T00:00:00.000Z" } }); const pause = (ms = 5) => new Promise((resolve) => setTimeout(resolve, ms)); async function until(test: () => boolean | Promise): Promise { for (let n = 0; n < 200; n++) { if (await test()) return; await pause(); } throw new Error("expected side effect did not appear"); } @@ -29,13 +30,13 @@ class Core implements OrganizationRuntimeHost { async activity() { return { version: "noopolis.daimon.organization-runtime-activity.v1" as const, items: [] }; } async stop() { this.stops++; this.releases.forEach((release, index) => release({ version: "noopolis.daimon.wake-result.v1", status: "stopped", agentId: this.wakes[index]!.agentId, wakeId: this.wakes[index]!.event.id, code: "active_wake_aborted" })); return { version: "noopolis.daimon.organization-runtime-stop.v1" as const, state: "stopped" as const }; } } -async function fixture(limit = 20, maxWakes = 100, claimTtlMs = 240000) { +async function fixture(limit = 20, maxWakes = 100, claimTtlMs = 240000, engine: "codex" | "grok" = "codex") { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-attention-")); await chmod(root, 0o700); const usage = await mkdtemp(path.join(os.tmpdir(), "daimon-attention-usage-")); await writeFile(path.join(usage, "usage.jsonl"), ""); const registry: AttentionRegistry = new Map(); const core = new Core(); const options = { acceptanceStorePath: root, controlToken: token, storeOptions: { ...storeOptions, claimTtlMs }, attentionRegistryForTest: registry, fuseEnvironment: { DAIMON_WAKE_FUSE_DIRECTORY: usage, DAIMON_WAKE_FUSE_EPOCH: "attention", DAIMON_WAKE_FUSE_MAX_WAKES: String(maxWakes), DAIMON_WAKE_FUSE_MAX_TOKENS: "10000", DAIMON_TURN_USAGE_LEDGER_PATH: path.join(usage, "usage.jsonl") } }; - const control = createOrganizationRuntimeControlHostWithCoreForTest(config(limit), core, options); await control.start(); + const control = createOrganizationRuntimeControlHostWithCoreForTest(config(limit, engine), core, options); await control.start(); return { root, usage, registry, core, options, control, cleanup: async () => { await control.stop(); await rm(root, { recursive: true, force: true }); await rm(usage, { recursive: true, force: true }); } }; } @@ -214,3 +215,31 @@ test("an inbox turn leads with each delivery's own text and keeps the accounting assert.ok(text.indexOf("Machine-readable payload:") > accounting, "payload stays a trailing appendix"); } finally { await f.cleanup(); } }); + +/** + * `daimon_inbox_disposition` is the tool that records a finished wake as + * complete; an agent that cannot name it leaves its work recorded as deferred. + * On Grok the bare name reaches nothing, so the inbox prompt must name the + * `daimon__` form the engine can actually invoke. + */ +test("a Grok inbox turn names both inbox tools the way use_tool can call them", async () => { + const f = await fixture(20, 100, 240000, "grok"); + try { + await f.control.accept(request("g-1")); await until(() => f.core.wakes.length === 1); + const text = f.core.wakes[0]!.event.text!; + assert.ok(text.includes(grokDaimonToolName("daimon_inbox_disposition")), "disposition tool carries the daimon__ prefix"); + assert.ok(text.includes(grokDaimonToolName("daimon_inbox")), "inbox tool carries the daimon__ prefix"); + // No bare occurrence survives: every mention is the prefixed one. + assert.equal(text.split("daimon_inbox").length - 1, text.split(DAIMON_GROK_TOOL_PREFIX).length - 1); + } finally { await f.cleanup(); } +}); + +test("every other engine's inbox turn keeps the bare tool names", async () => { + const f = await fixture(); + try { + await f.control.accept(request("c-1")); await until(() => f.core.wakes.length === 1); + const text = f.core.wakes[0]!.event.text!; + assert.ok(text.includes("with daimon_inbox_disposition (complete)")); + assert.equal(text.includes(DAIMON_GROK_TOOL_PREFIX), false); + } finally { await f.cleanup(); } +}); diff --git a/src/runtime/attentionDispatcher.ts b/src/runtime/attentionDispatcher.ts index 208bd5d..5d842b5 100644 --- a/src/runtime/attentionDispatcher.ts +++ b/src/runtime/attentionDispatcher.ts @@ -6,6 +6,7 @@ import { WakeAcceptanceStore, WakeExecutionClaimLostError, type WakeExecutionCla import type { StoredWakeAcceptanceRecord } from "./wakeAcceptanceRecord.js"; import { WakeFuse } from "./wakeFuse.js"; import { ORGANIZATION_RUNTIME_MAX_STRING_CODEPOINTS, ORGANIZATION_RUNTIME_MAX_WAKE_TEXT_BYTES } from "../contracts/organizationRuntimeContract.js"; +import { grokDaimonToolName } from "../contracts/grokWorkerContract.js"; type Claimed = { record: StoredWakeAcceptanceRecord; claim: WakeExecutionClaim; done: boolean }; type Options = Readonly<{ store: WakeAcceptanceStore; host: OrganizationRuntimeHost; fuse: WakeFuse; agents: readonly OrganizationRuntimeAgentConfig[]; registry: AttentionRegistry; token: string | undefined; onIdle(agentId: string): void }>; @@ -111,7 +112,7 @@ export class AttentionDispatcher { result = await host.wake({ token, agentId: agent.id, event: { version: "noopolis.daimon.wake.v1", id: agent.attention === undefined ? first.delivery_id : executionId, kind: first.event.kind, occurredAt: first.event.occurred_at, - text: agent.attention === undefined ? first.event.text : inboxPrompt(messages, agent.attention.maxBatchBytes) + text: agent.attention === undefined ? first.event.text : inboxPrompt(messages, agent.engine.kind, agent.attention.maxBatchBytes) } }); } catch (error) { result = { version: "noopolis.daimon.wake-result.v1", status: "failed", agentId: agent.id, wakeId: executionId, code: "engine_failed", detail: engineFailureDetail(error) }; @@ -193,11 +194,19 @@ function deliveryBlock(message: unknown, index: number): string | undefined { * rendered as labelled blocks and the `daimon_inbox` accounting follows them as * what to do *after* the work, with the machine-readable payload kept as a * trailing appendix while it fits the same budget. + * + * Both tools are named the way the agent's own engine can call them. On Grok a + * Daimon tool is an MCP tool of server `daimon` and its bare name reaches + * nothing (`grokDaimonToolName`, the same contract module the worker's system + * prompt and identity envelope render from), so an agent handed the bare name + * cannot mark its work complete — and an unmarked, finished wake is recorded as + * deferred. */ -function inboxPrompt(messages: readonly unknown[], maxBytes = 12000): string { +function inboxPrompt(messages: readonly unknown[], engine: OrganizationRuntimeAgentConfig["engine"]["kind"], maxBytes = 12000): string { const body = JSON.stringify(messages); const blocks = messages.map(deliveryBlock).filter((block): block is string => block !== undefined); - const accounting = "\nWhen the work above is done, record each delivery with daimon_inbox_disposition (complete), or defer the ones you could not finish; use daimon_inbox for deliveries and remaining allowances. Reading or ending this turn never completes a delivery, and deferred work waits for a later external wake.\n"; + const tool = (name: string): string => engine === "grok" ? grokDaimonToolName(name) : name; + const accounting = `\nWhen the work above is done, record each delivery with ${tool("daimon_inbox_disposition")} (complete), or defer the ones you could not finish; use ${tool("daimon_inbox")} for deliveries and remaining allowances. Reading or ending this turn never completes a delivery, and deferred work waits for a later external wake.\n`; const header = blocks.length === 1 ? "Carry out this delivery.\n" : `Carry out these ${blocks.length} deliveries.\n`; const fits = (value: string): boolean => Buffer.byteLength(value) <= ORGANIZATION_RUNTIME_MAX_WAKE_TEXT_BYTES && [...value].length <= ORGANIZATION_RUNTIME_MAX_STRING_CODEPOINTS; @@ -209,7 +218,7 @@ function inboxPrompt(messages: readonly unknown[], maxBytes = 12000): string { if (Buffer.byteLength(body) <= maxBytes && fits(withPayload)) return withPayload; if (fits(task)) return task; } - const prefix = "Handle this inbox turn. Use daimon_inbox for deliveries and remaining allowances. Explicitly call daimon_inbox_disposition for each handled delivery (complete) or unfinished delivery (defer). Reading or ending this turn never completes a delivery. Deferred work waits for a later external wake.\n"; + const prefix = `Handle this inbox turn. Use ${tool("daimon_inbox")} for deliveries and remaining allowances. Explicitly call ${tool("daimon_inbox_disposition")} for each handled delivery (complete) or unfinished delivery (defer). Reading or ending this turn never completes a delivery. Deferred work waits for a later external wake.\n`; const prompt = prefix + body; if (Buffer.byteLength(body) > maxBytes || !fits(prompt)) { return prefix + "The selected payload exceeds the prompt budget; read it with daimon_inbox."; diff --git a/src/runtime/engineDispatcher.test.ts b/src/runtime/engineDispatcher.test.ts index 0dae9a8..62f37fc 100644 --- a/src/runtime/engineDispatcher.test.ts +++ b/src/runtime/engineDispatcher.test.ts @@ -4,7 +4,8 @@ import os from "node:os"; import path from "node:path"; import test from "node:test"; -import { codexSandboxProtectedPaths, codexSandboxReadablePaths, grokSandboxProtectedPaths, startOrganizationRuntimeEngine } from "./engineDispatcher.js"; +import { codexSandboxProtectedPaths, codexSandboxReadablePaths, grokSandboxProtectedPaths, identityEnvelope, startOrganizationRuntimeEngine } from "./engineDispatcher.js"; +import { DAIMON_GROK_MCP_SERVER, DAIMON_GROK_SYSTEM_PROMPT, DAIMON_GROK_TOOL_PREFIX, GROK_MCP_INVOKE_TOOL, GROK_MCP_SEARCH_TOOL, GROK_MCP_TOOL_NAME_ARGUMENT, grokDaimonToolName } from "../contracts/grokWorkerContract.js"; import { AGY_SUBSCRIPTION_REALM, GROK_SUBSCRIPTION_REALM } from "./contractManifest.js"; import type { EngineBrokerTurnClient } from "./engineBrokerControlClient.js"; import { ORGANIZATION_RUNTIME_VERSION, type OrganizationRuntimeAgentConfig } from "./organizationRuntime.js"; @@ -71,6 +72,55 @@ test("Codex strict sandbox protects current credentials, ingress, shared roots, assert.deepEqual(codexSandboxReadablePaths(current), [path.join(current.runtimeHomePath, "tool-output")]); }); +/** + * The envelope's Grok wording is the second naming rule a Grok worker reads, + * after the pinned system prompt. When the two disagreed the later, more + * emphatic one won and a live turn made zero tool calls with every tool + * correctly mounted, so what is asserted here is agreement: the same route, + * stated once, and no instruction to use a bare name. + */ +const mounted = ["moltnet_read", "moltnet_send", "memory_search"] as const; +const envelopeToolSentence = (kind: OrganizationRuntimeAgentConfig["engine"]["kind"]): string => + identityEnvelope(rootConfig("/private/org", kind), mounted).split("\n").find((line) => line.startsWith("Your mounted tools are exactly"))!; + +test("the Grok envelope states the use_tool prefix rule once and never countermands the system prompt", () => { + const sentence = envelopeToolSentence("grok"); + // The route, asserted: one bare catalogue, one prefix rule, one example. + assert.equal(sentence.includes(`Your mounted tools are exactly: ${mounted.join(", ")}.`), true); + assert.match(sentence, new RegExp(`MCP tool on server ${DAIMON_GROK_MCP_SERVER}`, "u")); + assert.match(sentence, new RegExp(`invoke it with ${GROK_MCP_INVOKE_TOOL}, ${GROK_MCP_TOOL_NAME_ARGUMENT} = ${DAIMON_GROK_TOOL_PREFIX}`, "u")); + assert.match(sentence, new RegExp(`for example ${grokDaimonToolName(mounted[0])}`, "u")); + assert.match(sentence, /none is callable by its bare name/u); + assert.match(sentence, new RegExp(`${GROK_MCP_SEARCH_TOOL} lists them if a name is unknown`, "u")); + assert.match(sentence, /No other tool reaches the newsroom\.$/u); + // What it must never say: the bare names are callable, the agent's own + // instructions are wrong, or a shell reaches the tools. + assert.doesNotMatch(sentence, /Call them by these names/u); + assert.doesNotMatch(sentence, /spell them differently/u); + assert.doesNotMatch(sentence, /shell|terminal|CLI|run_terminal/u); + // One catalogue only: the prefixed names are a rule, not a second list. + for (const tool of mounted) assert.equal(sentence.split(tool).length - 1, tool === mounted[0] ? 2 : 1, tool); + assert.equal(sentence.split(DAIMON_GROK_TOOL_PREFIX).length - 1, 2, "prefix appears as the rule and its one example"); + // The transport prohibition is untouched and still follows the tool sentence. + assert.match(identityEnvelope(rootConfig("/private/org", "grok"), mounted), /Do not seek transport credentials or invoke a transport CLI/u); +}); + +test("the Grok envelope and the pinned worker system prompt state the same route", () => { + const sentence = envelopeToolSentence("grok"); + for (const atom of [DAIMON_GROK_MCP_SERVER, DAIMON_GROK_TOOL_PREFIX, GROK_MCP_INVOKE_TOOL, GROK_MCP_SEARCH_TOOL, GROK_MCP_TOOL_NAME_ARGUMENT]) { + assert.ok(DAIMON_GROK_SYSTEM_PROMPT.includes(atom), `system prompt states ${atom}`); + assert.ok(sentence.includes(atom), `envelope states ${atom}`); + } +}); + +test("only Grok gains the prefix rule: every other engine's envelope stays byte-identical", () => { + const unchanged = `Your mounted tools are exactly: ${mounted.join(", ")}. Call them by these names; your instructions may spell them differently. No other tool reaches the newsroom.`; + for (const kind of ["codex", "agy"] as const) assert.equal(envelopeToolSentence(kind), unchanged); + assert.notEqual(envelopeToolSentence("grok"), unchanged); + // An unmounted agent gets no tool sentence at all, on every engine. + for (const kind of ["codex", "agy", "grok"] as const) assert.doesNotMatch(identityEnvelope(rootConfig("/private/org", kind)), /Your mounted tools/u); +}); + test("production dispatcher starts each closed engine intent through Daimon", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-dispatcher-")); const priorPath = process.env.PATH; diff --git a/src/runtime/engineDispatcher.ts b/src/runtime/engineDispatcher.ts index a5864a5..24513af 100644 --- a/src/runtime/engineDispatcher.ts +++ b/src/runtime/engineDispatcher.ts @@ -1,4 +1,5 @@ import { attentionTools, type AttentionRegistry } from "./attention.js"; +import { grokMountedToolNamingRule } from "../contracts/grokWorkerContract.js"; import path from "node:path"; import type { AgentHandle } from "../core/types.js"; @@ -180,20 +181,29 @@ function grokBrokerTurnFor(agent: OrganizationRuntimeAgentConfig, grokBroker: En /** * The caller-owned prompt preamble. * - * It names the mounted tools explicitly. A CLI engine reaches Daimon's tools - * over MCP, and Grok exposes MCP tools only through a deferred `search_tool` - * catalog, so an agent whose instructions name another engine's tool spelling - * can finish a turn having called nothing. The declared names are the caller's - * own configuration, not engine-supplied text. + * It names the mounted tools explicitly, because a CLI engine reaches Daimon's + * tools over MCP and an agent whose instructions name another engine's tool + * spelling can finish a turn having called nothing. The declared names are the + * caller's own configuration, not engine-supplied text. + * + * On Grok the bare names are not the callable ones: every Daimon tool is a + * deferred MCP tool of server `daimon`, reached through `use_tool` with + * `tool_name` = `daimon__`. That rule is not restated here — it is + * rendered by `grokMountedToolNamingRule` in the same contract module that + * renders the worker's pinned system prompt, so this envelope can no longer + * contradict it. Every other engine's sentence is unchanged, byte for byte. */ -function identityEnvelope(agent: OrganizationRuntimeAgentConfig, mountedToolNames: readonly string[] = []): string { +export function identityEnvelope(agent: OrganizationRuntimeAgentConfig, mountedToolNames: readonly string[] = []): string { return [ "", JSON.stringify({ id: agent.id, name: agent.name, instructions: agent.instructions }), "", ...(mountedToolNames.length === 0 ? [] : [ - `Your mounted tools are exactly: ${mountedToolNames.join(", ")}. Call them by these names; ` - + "your instructions may spell them differently. No other tool reaches the newsroom." + (agent.engine.kind === "grok" + ? grokMountedToolNamingRule(mountedToolNames) + : `Your mounted tools are exactly: ${mountedToolNames.join(", ")}. Call them by these names; ` + + "your instructions may spell them differently.") + + " No other tool reaches the newsroom." ]), "Colleagues only hear you when you call moltnet_send; your terminal response is a private note to the runtime, not a message to anyone — keep it to one line or leave it empty. " + "Do not seek transport credentials or invoke a transport CLI unless the caller explicitly mounted an authenticated transport tool.", From cb7a745c2de357ea57ad80843cce9c058dec5259 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 00:10:46 +0200 Subject: [PATCH 26/69] fix: name the prefixed Grok tool form as the only valid one and add no third search_tool voice --- src/contracts/grokWorkerContract.ts | 37 +++++--- src/runtime/engineDispatcher.test.ts | 23 +++-- src/runtime/engineDispatcher.ts | 12 ++- src/runtime/grokBrokerProxy.ts | 17 +++- src/runtime/grokBrokerTurnMeter.test.ts | 54 ++++++++++- src/runtime/grokBrokerTurnMeter.ts | 110 ++++++++++++++++++---- src/runtime/grokEngineBrokerTurn.ts | 18 +++- src/runtime/grokEngineBrokerUsage.test.ts | 51 +++++++++- src/runtime/turnRequestLedger.ts | 24 ++++- 9 files changed, 289 insertions(+), 57 deletions(-) diff --git a/src/contracts/grokWorkerContract.ts b/src/contracts/grokWorkerContract.ts index fbe949d..2c49b12 100644 --- a/src/contracts/grokWorkerContract.ts +++ b/src/contracts/grokWorkerContract.ts @@ -21,10 +21,12 @@ * Both texts a Grok worker receives are rendered from them: the pinned system * prompt below, and the caller's identity envelope * ({@link grokMountedToolNamingRule}, used by `src/runtime/engineDispatcher.ts`). - * They were worded independently once - the envelope told the model to call the - * tools by their bare names - and a live turn produced zero tool calls with - * every tool correctly mounted: two authoritative naming rules, the wrong one - * last. One definition is what keeps them from diverging again. + * They were worded independently once, and the envelope told the model to call + * the tools by their bare names - which Grok 1.0.34 refuses outright, before + * any HTTP: `'moltnet_read' is not a valid MCP tool name. Tool names must be + * qualified as \`server__tool\`` (local rig, real CLI, real rendered config). + * There is exactly one valid spelling, so two independently worded naming rules + * are one rule too many; this is the single definition both render from. */ export const DAIMON_GROK_MCP_SERVER = "daimon" as const; /** Grok's own name for an MCP tool of that server: exactly what `tool_name` must carry. */ @@ -50,19 +52,28 @@ export const DAIMON_GROK_SYSTEM_PROMPT = [ * The same route, stated once for the caller's identity envelope, where a * wake's real mounted tools are known. * - * It asserts rather than corrects: it names the bare tool set once, states the - * `use_tool` prefix rule once with one example drawn from that set, and says - * plainly that a bare name is not callable here. It never claims the agent's - * own instructions spell a tool wrongly, never offers a shell or CLI route, and - * never repeats the tool list a second time in prefixed form. + * It contributes exactly what the pinned prompt cannot know - the wake's real + * mounted names - and the one rule that makes them callable. It asserts rather + * than corrects: one bare catalogue, one prefix rule, one example, and the + * prefixed form named as the *only* valid form, because that is the CLI's own + * verdict on a bare name rather than a preference. + * + * What it deliberately leaves out is as load bearing. It never claims the + * agent's own instructions spell a tool wrongly, never offers a shell or CLI + * route, never repeats the catalogue in prefixed form - and never restates the + * `search_tool` rule. A Grok worker already reads two authoritative sentences + * about `search_tool`: the pinned prompt's ("only for a name you do not know") + * and Grok's own injected notice, which says the model MUST call it before any + * MCP tool. Observed on the rig: that contradiction is not enforced, and + * `use_tool` works with no prior `search_tool`. A third wording would only add + * a voice, so this sentence stays out of that argument entirely. */ export const grokMountedToolNamingRule = (mountedToolNames: readonly string[]): string => { const example = grokDaimonToolName(mountedToolNames[0] ?? DAIMON_GROK_EXAMPLE_TOOLS[0]); return `Your mounted tools are exactly: ${mountedToolNames.join(", ")}. ` - + `On this engine each is an MCP tool on server ${DAIMON_GROK_MCP_SERVER}: invoke it with ${GROK_MCP_INVOKE_TOOL}, ` - + `${GROK_MCP_TOOL_NAME_ARGUMENT} = ${DAIMON_GROK_TOOL_PREFIX} (for example ${example}). ` - + "None appears in your direct tool list and none is callable by its bare name; " - + `${GROK_MCP_SEARCH_TOOL} lists them if a name is unknown.`; + + `On this engine each is an MCP tool on server ${DAIMON_GROK_MCP_SERVER}, and its only valid tool name is ` + + `${DAIMON_GROK_TOOL_PREFIX}: invoke it with ${GROK_MCP_INVOKE_TOOL}, ${GROK_MCP_TOOL_NAME_ARGUMENT} = ${example}. ` + + "A bare name is not a valid MCP tool name and reaches nothing."; }; /** Closed declared-model vocabulary; defaults are `grok-4.6` at `low`. */ diff --git a/src/runtime/engineDispatcher.test.ts b/src/runtime/engineDispatcher.test.ts index 62f37fc..dea7789 100644 --- a/src/runtime/engineDispatcher.test.ts +++ b/src/runtime/engineDispatcher.test.ts @@ -74,30 +74,33 @@ test("Codex strict sandbox protects current credentials, ingress, shared roots, /** * The envelope's Grok wording is the second naming rule a Grok worker reads, - * after the pinned system prompt. When the two disagreed the later, more - * emphatic one won and a live turn made zero tool calls with every tool - * correctly mounted, so what is asserted here is agreement: the same route, - * stated once, and no instruction to use a bare name. + * after the pinned system prompt. It used to instruct the bare form, which + * Grok 1.0.34 refuses outright as an invalid MCP tool name, so what is asserted + * here is agreement: the same route, stated once, the prefixed form named as + * the only valid one, and no third voice about `search_tool`. */ const mounted = ["moltnet_read", "moltnet_send", "memory_search"] as const; const envelopeToolSentence = (kind: OrganizationRuntimeAgentConfig["engine"]["kind"]): string => identityEnvelope(rootConfig("/private/org", kind), mounted).split("\n").find((line) => line.startsWith("Your mounted tools are exactly"))!; -test("the Grok envelope states the use_tool prefix rule once and never countermands the system prompt", () => { +test("the Grok envelope names the prefixed form as the only valid one, once", () => { const sentence = envelopeToolSentence("grok"); // The route, asserted: one bare catalogue, one prefix rule, one example. assert.equal(sentence.includes(`Your mounted tools are exactly: ${mounted.join(", ")}.`), true); assert.match(sentence, new RegExp(`MCP tool on server ${DAIMON_GROK_MCP_SERVER}`, "u")); - assert.match(sentence, new RegExp(`invoke it with ${GROK_MCP_INVOKE_TOOL}, ${GROK_MCP_TOOL_NAME_ARGUMENT} = ${DAIMON_GROK_TOOL_PREFIX}`, "u")); - assert.match(sentence, new RegExp(`for example ${grokDaimonToolName(mounted[0])}`, "u")); - assert.match(sentence, /none is callable by its bare name/u); - assert.match(sentence, new RegExp(`${GROK_MCP_SEARCH_TOOL} lists them if a name is unknown`, "u")); + assert.match(sentence, new RegExp(`only valid tool name is ${DAIMON_GROK_TOOL_PREFIX}`, "u")); + assert.match(sentence, new RegExp(`invoke it with ${GROK_MCP_INVOKE_TOOL}, ${GROK_MCP_TOOL_NAME_ARGUMENT} = ${grokDaimonToolName(mounted[0])}`, "u")); + // A bare name is invalid, not merely discouraged: that is the CLI's own verdict. + assert.match(sentence, /A bare name is not a valid MCP tool name and reaches nothing\./u); assert.match(sentence, /No other tool reaches the newsroom\.$/u); // What it must never say: the bare names are callable, the agent's own // instructions are wrong, or a shell reaches the tools. assert.doesNotMatch(sentence, /Call them by these names/u); assert.doesNotMatch(sentence, /spell them differently/u); assert.doesNotMatch(sentence, /shell|terminal|CLI|run_terminal/u); + // Fewer authoritative voices: the pinned prompt and Grok's own injected + // notice already give two rules for `search_tool`. This adds no third. + assert.equal(sentence.includes(GROK_MCP_SEARCH_TOOL), false); // One catalogue only: the prefixed names are a rule, not a second list. for (const tool of mounted) assert.equal(sentence.split(tool).length - 1, tool === mounted[0] ? 2 : 1, tool); assert.equal(sentence.split(DAIMON_GROK_TOOL_PREFIX).length - 1, 2, "prefix appears as the rule and its one example"); @@ -107,7 +110,7 @@ test("the Grok envelope states the use_tool prefix rule once and never counterma test("the Grok envelope and the pinned worker system prompt state the same route", () => { const sentence = envelopeToolSentence("grok"); - for (const atom of [DAIMON_GROK_MCP_SERVER, DAIMON_GROK_TOOL_PREFIX, GROK_MCP_INVOKE_TOOL, GROK_MCP_SEARCH_TOOL, GROK_MCP_TOOL_NAME_ARGUMENT]) { + for (const atom of [DAIMON_GROK_MCP_SERVER, DAIMON_GROK_TOOL_PREFIX, GROK_MCP_INVOKE_TOOL, GROK_MCP_TOOL_NAME_ARGUMENT]) { assert.ok(DAIMON_GROK_SYSTEM_PROMPT.includes(atom), `system prompt states ${atom}`); assert.ok(sentence.includes(atom), `envelope states ${atom}`); } diff --git a/src/runtime/engineDispatcher.ts b/src/runtime/engineDispatcher.ts index 24513af..40b06f1 100644 --- a/src/runtime/engineDispatcher.ts +++ b/src/runtime/engineDispatcher.ts @@ -187,11 +187,13 @@ function grokBrokerTurnFor(agent: OrganizationRuntimeAgentConfig, grokBroker: En * caller's own configuration, not engine-supplied text. * * On Grok the bare names are not the callable ones: every Daimon tool is a - * deferred MCP tool of server `daimon`, reached through `use_tool` with - * `tool_name` = `daimon__`. That rule is not restated here — it is - * rendered by `grokMountedToolNamingRule` in the same contract module that - * renders the worker's pinned system prompt, so this envelope can no longer - * contradict it. Every other engine's sentence is unchanged, byte for byte. + * deferred MCP tool of server `daimon`, and Grok 1.0.34 refuses an unqualified + * name before any HTTP ("Tool names must be qualified as `server__tool`"). This + * envelope used to instruct exactly that refused form. The correct rule is not + * restated here — it is rendered by `grokMountedToolNamingRule` in the same + * contract module that renders the worker's pinned system prompt, so the two + * texts cannot contradict each other again. Every other engine's sentence is + * unchanged, byte for byte. */ export function identityEnvelope(agent: OrganizationRuntimeAgentConfig, mountedToolNames: readonly string[] = []): string { return [ diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index e13d5ca..c848d91 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -5,7 +5,7 @@ import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; import { DEFAULT_GROK_BROKER_MODEL_POLICY, parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; import { authorizeGrokBrokerProxyRequest } from "./grokBrokerProxyRequest.js"; import { GROK_SESSION_TITLE_SINK_KEY } from "./grokBrokerWorkerConfig.js"; -import { parseGrokUpstreamUsage, type GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; +import { parseGrokResponseToolNames, parseGrokUpstreamUsage, type GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; import { serveGrokInferenceGrant } from "./grokInferenceProxy.js"; import type { GrokInferenceGrants } from "./grokInferenceGrants.js"; @@ -49,7 +49,7 @@ export class GrokBrokerProxyRefusal extends Error { } async function serve(request: IncomingMessage, response: ServerResponse, authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream, capabilities: EngineBrokerCapabilities,guards:MapPromise>,turns:Map,fallback:GrokBrokerModelPolicy,grants?:GrokInferenceGrants): Promise { - let settle:((usage:ReturnType)=>void)|undefined; + let settle:((usage:ReturnType,toolCalls?:readonly string[])=>void)|undefined; let titleSink = false; try { const body = await readBody(request); const headers = Object.fromEntries(Object.entries(request.headers).map(([key, value]) => [key, Array.isArray(value) ? value[0] : value])); @@ -64,10 +64,14 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori const admission=turn.meter.admit(); if("refused" in admission){response.writeHead(429,{"content-type":"application/json","cache-control":"no-store"});response.end(JSON.stringify({error:"turn limit reached",limit:admission.refused}));return;} if("busy" in admission){response.writeHead(429,{"content-type":"application/json","cache-control":"no-store"});response.end('{"error":"turn request in flight"}');return;} - settle=(usage)=>{turn.meter.settle(admission.index,usage,body.byteLength);settle=undefined;}; + settle=(usage,toolCalls)=>{turn.meter.settle(admission.index,usage,body.byteLength,toolCalls);settle=undefined;}; let result = await upstream(prepared,admission.signal); if (result.status === 401) { token = authority.refreshAfterRejection?await authority.refreshAfterRejection(rejectedDigest):await authority.accessToken(true);const refreshedDigest=createHash("sha256").update(token).digest("hex"); prepared = { ...prepared, headers: { ...prepared.headers, authorization: `Bearer ${token}` } }; token = ""; result = await upstream(prepared,admission.signal);if(result.status===401)await authority.markRejected(refreshedDigest); } - settle?.(parseGrokUpstreamUsage(result.body,result.headers["content-type"])); + // Names only, bounded, and never a reason to fail the request: the response + // is already buffered here for its usage block, so what the model tried to + // call is in hand. A decoder fault records no attempt rather than a false + // empty one, and never disturbs the turn. + settle?.(parseGrokUpstreamUsage(result.body,result.headers["content-type"]),toolCallsOrNothing(result.body,result.headers["content-type"])); response.writeHead(result.status, { "content-type": result.headers["content-type"] ?? "application/json", "cache-control": "no-store" }); response.end(result.body); } catch (error) { settle?.(undefined); @@ -112,6 +116,11 @@ const expectedWorkerProbe = (request: IncomingMessage): boolean => request.headers.authorization === undefined && (request.method ?? "") === "GET" && new URL(request.url ?? "/", "http://127.0.0.1").pathname === "/"; +/** Instrumentation must never fail a turn: a throwing decoder records nothing, exactly as an undecodable response does. */ +const toolCallsOrNothing = (body: Uint8Array, contentType: string | undefined): readonly string[] | undefined => { + try { return parseGrokResponseToolNames(body, contentType); } catch { return undefined; } +}; + /** The body gate, refused non-retryably: a rejected body is a policy miss, never a transient fault. */ function authorizeRequestOrRefuse(...args: Parameters): ReturnType { try { return authorizeGrokBrokerProxyRequest(...args); } diff --git a/src/runtime/grokBrokerTurnMeter.test.ts b/src/runtime/grokBrokerTurnMeter.test.ts index 094ef65..881b8fd 100644 --- a/src/runtime/grokBrokerTurnMeter.test.ts +++ b/src/runtime/grokBrokerTurnMeter.test.ts @@ -3,7 +3,7 @@ import { request as httpRequest } from "node:http"; import test from "node:test"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; -import { GrokBrokerTurnMeter, parseGrokUpstreamUsage } from "./grokBrokerTurnMeter.js"; +import { GROK_REQUEST_TOOL_CALLS_MAX, GROK_TOOL_CALL_INVALID, GROK_TOOL_CALL_TRUNCATED, GrokBrokerTurnMeter, parseGrokResponseToolNames, parseGrokUpstreamUsage } from "./grokBrokerTurnMeter.js"; const lean = ["run_terminal_command", "read_file", "list_dir", "grep", "search_tool", "use_tool"].map((name) => ({ type: "function", function: { name } })); const body = JSON.stringify({ model: "grok-4.6", reasoning_effort: "low", stream: true, messages: [], tools: lean }); @@ -189,3 +189,55 @@ test("an implausible per-request usage block is never added, and missing usage s }); assert.deepEqual([blind.snapshot().limitReason, blind.snapshot().tokens, blind.snapshot().estimatedRequests], ["tokens", 12_891, 3]); }); + +const events = (chunks: readonly unknown[]): Uint8Array => + Buffer.from(chunks.map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("") + "data: [DONE]\n\n"); +/** One streaming tool call: the name arrives in one delta, the arguments in the next. */ +const callDeltas = (names: readonly string[]): unknown[] => [ + ...names.map((name, index) => ({ choices: [{ index: 0, delta: { tool_calls: [{ index, id: `call-${index}`, type: "function", function: { name, arguments: "" } }] } }] })), + ...names.map((_name, index) => ({ choices: [{ index: 0, delta: { tool_calls: [{ index, function: { arguments: '{"query":"secret"}' } }] } }] })) +]; + +/** + * Two live turns ended with every tool correctly mounted and no way to tell + * whether the model had tried to call anything. These names are that answer, + * and nothing more than that answer. + */ +test("a response's tool-call names are read, bounded, and stripped of everything but the names", async () => { + // Mutation guard: passing the response through instead of the names leaks arguments here. + const names = parseGrokResponseToolNames(events([...callDeltas(["use_tool", "search_tool"]), { choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], usage: { prompt_tokens: 1, completion_tokens: 1 } }]), "text/event-stream"); + assert.deepEqual(names, ["use_tool", "search_tool"]); + assert.equal(JSON.stringify(names).includes("secret"), false); + + // A non-streaming body carries its calls on the message; one tool called + // twice is two attempts, because neither carries a streaming call index. + assert.deepEqual(parseGrokResponseToolNames(Buffer.from(JSON.stringify({ choices: [{ message: { tool_calls: [{ function: { name: "use_tool" } }, { function: { name: "use_tool" } }] } }] })), "application/json"), ["use_tool", "use_tool"]); + + // Absence stays absence: a decoded response that called nothing is `[]`, and + // an undecodable one is nothing at all. A zero-length list must never be + // invented for a response nobody could read. + assert.deepEqual(parseGrokResponseToolNames(events([{ choices: [{ index: 0, delta: { content: "x" } }] }]), "text/event-stream"), []); + assert.equal(parseGrokResponseToolNames(Buffer.from("gateway"), "text/html"), undefined); + assert.equal(parseGrokResponseToolNames(Buffer.from(""), "text/event-stream"), undefined); + assert.equal(parseGrokResponseToolNames(Buffer.from("data: not-json\n\n"), "text/event-stream"), undefined); + + // A name that is not a plain short identifier is counted, never passed through. + assert.deepEqual(parseGrokResponseToolNames(events(callDeltas(["ok_tool", "a b/c", "x".repeat(65), "inject\nline"])), "text/event-stream"), ["ok_tool", GROK_TOOL_CALL_INVALID, GROK_TOOL_CALL_INVALID, GROK_TOOL_CALL_INVALID]); + + // Mutation guard: unbounded, a pathological response writes 400 names into one row. + const many = parseGrokResponseToolNames(events(callDeltas(Array.from({ length: 400 }, (_value, index) => `tool_${index}`))), "text/event-stream"); + assert.equal(many!.length, GROK_REQUEST_TOOL_CALLS_MAX); + assert.equal(many!.at(-1), GROK_TOOL_CALL_TRUNCATED); + assert.deepEqual(many!.slice(0, 2), ["tool_0", "tool_1"]); + // Exactly the bound is not truncated. + assert.equal(parseGrokResponseToolNames(events(callDeltas(Array.from({ length: GROK_REQUEST_TOOL_CALLS_MAX }, (_value, index) => `tool_${index}`))), "text/event-stream")!.includes(GROK_TOOL_CALL_TRUNCATED), false); +}); + +test("the meter carries each request's tool-call names without letting them touch the spend gate", async () => { + const meter = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 300_000, timeoutMs: 60_000 }); + await withProxy({ prompt_tokens: 11, completion_tokens: 2 }, meter, async (send) => { assert.equal((await send()).status, 200); }); + const [timing] = meter.snapshot().timings; + // The stub response carries no tool call, and says so rather than staying silent. + assert.deepEqual(timing!.toolCalls, []); + assert.deepEqual([meter.snapshot().tokens, meter.snapshot().limitReason, meter.snapshot().estimatedRequests], [13, "none", 0]); +}); diff --git a/src/runtime/grokBrokerTurnMeter.ts b/src/runtime/grokBrokerTurnMeter.ts index 3c14b7c..f266a16 100644 --- a/src/runtime/grokBrokerTurnMeter.ts +++ b/src/runtime/grokBrokerTurnMeter.ts @@ -1,8 +1,13 @@ import { GROK_ENGINE_BROKER } from "../contracts/runtimeContractManifest.js"; import { sumEngineBrokerTurnUsage, type EngineBrokerLimitReason, type EngineBrokerTurnLimits, type EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; -/** `estimated` marks a request whose response carried no valid usage and was charged {@link estimateGrokRequestUsage}. */ -export type GrokBrokerRequestTiming = Readonly<{ startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage; estimated?: true }>; +/** + * `estimated` marks a request whose response carried no valid usage and was + * charged {@link estimateGrokRequestUsage}. `toolCalls` are the tool-call names + * that request's response carried, names only ({@link parseGrokResponseToolNames}); + * absent means the response could not be decoded, `[]` that it called nothing. + */ +export type GrokBrokerRequestTiming = Readonly<{ startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage; estimated?: true; toolCalls?: readonly string[] }>; export type GrokBrokerTurnMeterSnapshot = Readonly<{ requests: number; tokens: number; limitReason: EngineBrokerLimitReason; usage: EngineBrokerTurnUsage | null; estimatedRequests: number; timings: readonly GrokBrokerRequestTiming[] }>; /** @@ -36,7 +41,7 @@ export type GrokBrokerTurnMeterSnapshot = Readonly<{ requests: number; tokens: n */ export class GrokBrokerTurnMeter { private readonly startedAt: number; - private readonly timings: { startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage; estimated?: true }[] = []; + private readonly timings: { startedAt: string; endedAt?: string; usage?: EngineBrokerTurnUsage; estimated?: true; toolCalls?: readonly string[] }[] = []; private tokens = 0; private reason: EngineBrokerLimitReason = "none"; private inFlight: { index: number; controller: AbortController } | undefined; @@ -60,18 +65,21 @@ export class GrokBrokerTurnMeter { } /** - * Records one admitted request's end and its usage. A response without valid - * usage (absent, malformed, implausible, or a failed/aborted call) is charged - * a conservative estimate from the request body size, so a missing `usage` - * can never silently disable the token ceiling. + * Records one admitted request's end, its usage, and the tool-call names its + * response carried. A response without valid usage (absent, malformed, + * implausible, or a failed/aborted call) is charged a conservative estimate + * from the request body size, so a missing `usage` can never silently disable + * the token ceiling. `toolCalls` is observation only: it never affects + * admission, the running total, or any limit. */ - settle(index: number, usage: EngineBrokerTurnUsage | undefined, requestBytes: number): void { + settle(index: number, usage: EngineBrokerTurnUsage | undefined, requestBytes: number, toolCalls?: readonly string[]): void { const timing = this.timings[index]; if (timing === undefined || timing.endedAt !== undefined) return; if (this.inFlight?.index === index) this.inFlight = undefined; timing.endedAt = new Date(this.now()).toISOString(); if (usage === undefined) { timing.usage = estimateGrokRequestUsage(requestBytes); timing.estimated = true; } else timing.usage = usage; + if (toolCalls !== undefined) timing.toolCalls = toolCalls; this.tokens += timing.usage.total; } @@ -118,6 +126,26 @@ const count = (value: unknown): number | undefined => typeof value === "number" * zero-filled and never added — the meter charges an estimate instead. */ export function parseGrokUpstreamUsage(body: Uint8Array, contentType: string | undefined): EngineBrokerTurnUsage | undefined { + let found: EngineBrokerTurnUsage | undefined; + for (const candidate of decodeUpstreamResponse(body, contentType) ?? []) { + if (!isRecord(candidate) || !isRecord(candidate.usage)) continue; + // Last usage block wins even when invalid: an implausible final report + // must not fall back to an earlier, smaller block (the request is then + // charged the estimate instead). + found = decodeOpenAiUsage(candidate.usage); + } + return found; +} + +/** + * Every decodable JSON object of one upstream response: each `data:` event of + * an SSE stream, or the single body of a JSON response. + * + * `undefined` means *nothing* decoded — an unparseable or non-JSON response. + * Callers must keep that distinct from a decoded response that said nothing, + * because the ledger never fabricates an observation it did not make. + */ +function decodeUpstreamResponse(body: Uint8Array, contentType: string | undefined): unknown[] | undefined { const text = Buffer.from(body.buffer, body.byteOffset, body.byteLength).toString("utf8"); const candidates: unknown[] = []; if (contentType?.includes("text/event-stream") === true || text.startsWith("data:")) { @@ -125,20 +153,68 @@ export function parseGrokUpstreamUsage(body: Uint8Array, contentType: string | u if (!line.startsWith("data:")) continue; const payload = line.slice(5).trim(); if (payload === "[DONE]" || payload.length === 0) continue; - try { candidates.push(JSON.parse(payload)); } catch { /* a non-JSON event carries no usage */ } + try { candidates.push(JSON.parse(payload)); } catch { /* a non-JSON event carries neither usage nor a tool call */ } } } else { try { candidates.push(JSON.parse(text)); } catch { return undefined; } } - let found: EngineBrokerTurnUsage | undefined; - for (const candidate of candidates) { - if (!isRecord(candidate) || !isRecord(candidate.usage)) continue; - // Last usage block wins even when invalid: an implausible final report - // must not fall back to an earlier, smaller block (the request is then - // charged the estimate instead). - found = decodeOpenAiUsage(candidate.usage); + return candidates.length === 0 ? undefined : candidates; +} + +/** At most this many names per request row; a longer list ends in {@link GROK_TOOL_CALL_TRUNCATED}. */ +export const GROK_REQUEST_TOOL_CALLS_MAX = 16; +/** A `name` that is not a plain short identifier is counted, never passed through. */ +export const GROK_TOOL_CALL_INVALID = ""; +export const GROK_TOOL_CALL_TRUNCATED = ""; +const TOOL_CALL_NAME = /^[A-Za-z0-9_.-]{1,64}$/u; + +/** + * The tool-call NAMES one upstream response carried, and nothing else. + * + * Two live turns could not answer "did the model ever try `use_tool` or + * `search_tool`", because the per-request rows recorded timings and tokens but + * never an attempt. This is that answer, under four rules: + * + * - names only. No arguments, no message content, no tokens, no header. A + * `name` that is not a plain short identifier is recorded as + * {@link GROK_TOOL_CALL_INVALID} rather than passing provider bytes through; + * - bounded. At most {@link GROK_REQUEST_TOOL_CALLS_MAX} entries, the last being + * {@link GROK_TOOL_CALL_TRUNCATED} when the response carried more, so a + * pathological response cannot write an unbounded row; + * - absence stays absence. A decoded response that called nothing returns `[]`; + * a response that could not be decoded returns `undefined` and the row records + * no field at all; + * - one streaming call names itself in one delta and streams its arguments in + * the rest, so a repeat of the same `(choice, call)` index is that same call, + * not a second attempt. + */ +export function parseGrokResponseToolNames(body: Uint8Array, contentType: string | undefined): readonly string[] | undefined { + const candidates = decodeUpstreamResponse(body, contentType); + if (candidates === undefined) return undefined; + const names: string[] = [], seen = new Set(); + scan: for (const candidate of candidates) { + if (!isRecord(candidate) || !Array.isArray(candidate.choices)) continue; + for (const choice of candidate.choices) { + if (!isRecord(choice)) continue; + for (const source of [choice.delta, choice.message]) { + if (!isRecord(source) || !Array.isArray(source.tool_calls)) continue; + for (const call of source.tool_calls) { + if (!isRecord(call) || !isRecord(call.function)) continue; + const name = call.function.name; + // An arguments-only delta names nothing; it is not an attempt of its own. + if (typeof name !== "string" || name.length === 0) continue; + if (typeof choice.index === "number" && typeof call.index === "number") { + const key = `${choice.index}:${call.index}`; + if (seen.has(key)) continue; + seen.add(key); + } + names.push(TOOL_CALL_NAME.test(name) ? name : GROK_TOOL_CALL_INVALID); + if (names.length > GROK_REQUEST_TOOL_CALLS_MAX) break scan; + } + } + } } - return found; + return names.length > GROK_REQUEST_TOOL_CALLS_MAX ? [...names.slice(0, GROK_REQUEST_TOOL_CALLS_MAX - 1), GROK_TOOL_CALL_TRUNCATED] : names; } function decodeOpenAiUsage(usage: JsonRecord): EngineBrokerTurnUsage | undefined { diff --git a/src/runtime/grokEngineBrokerTurn.ts b/src/runtime/grokEngineBrokerTurn.ts index 674b572..3c66cdc 100644 --- a/src/runtime/grokEngineBrokerTurn.ts +++ b/src/runtime/grokEngineBrokerTurn.ts @@ -120,12 +120,22 @@ function streamOrMeterUsage(stream: GrokStreamUsage | undefined, snapshot: GrokB return snapshot.usage; } -/** Per-request rows: stream usage with proxy timing when both describe the same requests, else the proxy's own measured requests. */ +/** Per-request rows: stream usage with the proxy's own observation when both describe the same requests, else the proxy's measured requests. */ function requestRows(stream: GrokStreamUsage | undefined, snapshot: GrokBrokerTurnMeterSnapshot): BrokerTurnMeteringDetail["requests"] { if (stream !== undefined && stream.requests.length > 0) { const timed = snapshot.timings.length === stream.requests.length; - return stream.requests.map((value, index) => ({ ...value, usageSource: "stream" as const, ...(timed ? clock(snapshot.timings[index]!) : {}) })); + return stream.requests.map((value, index) => ({ ...value, usageSource: "stream" as const, ...(timed ? observed(snapshot.timings[index]!) : {}) })); } - return snapshot.timings.flatMap((timing, index) => timing.usage === undefined ? [] : [{ index, ...usageOf(timing.usage), usageSource: timing.estimated === true ? "estimated" as const : "upstream" as const, ...clock(timing) }]); + return snapshot.timings.flatMap((timing, index) => timing.usage === undefined ? [] : [{ index, ...usageOf(timing.usage), usageSource: timing.estimated === true ? "estimated" as const : "upstream" as const, ...observed(timing) }]); } -const clock = (timing: GrokBrokerTurnMeterSnapshot["timings"][number]) => ({ startedAt: timing.startedAt, ...(timing.endedAt === undefined ? {} : { endedAt: timing.endedAt }) }); +/** + * What only the proxy saw of one request: its clock, and the tool-call names the + * response carried. Both are attached on the stream path only when the two + * descriptions are request-for-request aligned, because an unaligned index would + * credit one request's attempt to another. + */ +const observed = (timing: GrokBrokerTurnMeterSnapshot["timings"][number]) => ({ + startedAt: timing.startedAt, + ...(timing.endedAt === undefined ? {} : { endedAt: timing.endedAt }), + ...(timing.toolCalls === undefined ? {} : { toolCalls: timing.toolCalls }) +}); diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index 9964bd9..e316a57 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -47,10 +47,19 @@ const untilAborted = (signal: AbortSignal): Promise => new Promise((_reso * talks to the proxy exactly as the native worker does (capability bearer, * pinned client version, lean body). Only the launcher and attestation are fakes. */ -const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId: string, worker: Worker, overrides?: Parameters[6], limits?: EngineBrokerServiceRegistration["limits"], turnStore?: string, syncDirectory?: (directory: string) => Promise) => ReturnType; usageRows: () => Promise[]>; requestRows: () => Promise[]>; upstreamCalls: () => number; upstreamAborts: () => number }>) => Promise, usageLedgerPath?: string, upstreamDelayMs: (call: number) => number = () => 15): Promise => { +/** The stub provider's own SSE response: usage only, and no tool call, unless a test says otherwise. */ +const upstreamResponse = (): string => `data: ${JSON.stringify({ choices: [], usage: upstreamUsage })}\n\ndata: [DONE]\n\n`; +/** One streaming tool call per name, arguments in a following delta, then the usage event. */ +const upstreamToolCallResponse = (names: readonly string[]): string => [ + ...names.map((name, index) => ({ choices: [{ index: 0, delta: { tool_calls: [{ index, id: `call-${index}`, type: "function", function: { name, arguments: "" } }] } }] })), + ...names.map((_name, index) => ({ choices: [{ index: 0, delta: { tool_calls: [{ index, function: { arguments: '{"tool_name":"daimon__moltnet_read"}' } }] } }] })), + { choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], usage: upstreamUsage } +].map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("") + "data: [DONE]\n\n"; + +const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId: string, worker: Worker, overrides?: Parameters[6], limits?: EngineBrokerServiceRegistration["limits"], turnStore?: string, syncDirectory?: (directory: string) => Promise) => ReturnType; usageRows: () => Promise[]>; requestRows: () => Promise[]>; upstreamCalls: () => number; upstreamAborts: () => number }>) => Promise, usageLedgerPath?: string, upstreamDelayMs: (call: number) => number = () => 15, upstreamBody: (call: number) => string = upstreamResponse): Promise => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-usage-")); let calls = 0, aborted = 0; - const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async (_request, signal) => { calls++; await new Promise((resolve, reject) => { const timer = setTimeout(resolve, upstreamDelayMs(calls)); signal?.addEventListener("abort", () => { clearTimeout(timer); aborted++; reject(new Error("aborted")); }, { once: true }); }); return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(`data: ${JSON.stringify({ choices: [], usage: upstreamUsage })}\n\ndata: [DONE]\n\n`) }; }, undefined, 0); + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async (_request, signal) => { calls++; await new Promise((resolve, reject) => { const timer = setTimeout(resolve, upstreamDelayMs(calls)); signal?.addEventListener("abort", () => { clearTimeout(timer); aborted++; reject(new Error("aborted")); }, { once: true }); }); return { status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from(upstreamBody(calls)) }; }, undefined, 0); const ledger = usageLedgerPath ?? path.join(root, "usage.jsonl"); const rows = async (file: string) => (await readFile(file, "utf8").catch(() => "")).split("\n").filter((line) => line.length > 0).map((line) => JSON.parse(line) as Record); try { @@ -244,3 +253,41 @@ test("the broker meters only through the single sealing helper, on both terminal assert.equal((body.match(/finishBrokerTurnWithUsage\(/gu) ?? []).length, 2); assert.ok(body.indexOf("return replay(") < body.indexOf("finishBrokerTurnWithUsage("), "a replay returns before any metering"); }); + +/** + * The question two live turns could not answer: did the model ever *try* to + * call a tool? The rows carried timings and tokens and nothing about an + * attempt, so a turn with zero tool calls and a turn whose calls all failed + * read identically after the fact. + */ +test("each per-request row records the tool-call names that request's response carried, names only", async () => { + await withBroker(async ({ turn, requestRows }) => { + await turn("wake-tools", twoRequests); + const rows = await requestRows(); + // Mutation guard: without the field these are `[undefined, undefined]`. + assert.deepEqual(rows.map((row) => row.tool_calls), [["use_tool", "search_tool"], ["use_tool", "search_tool"]]); + const text = JSON.stringify(rows); + // Names only: no arguments, no message content, no bearer. + assert.equal(text.includes("daimon__moltnet_read"), false, "an argument value must never reach the ledger"); + assert.equal(text.includes("arguments"), false); + assert.equal(text.includes("provider-token"), false); + }, undefined, () => 1, () => upstreamToolCallResponse(["use_tool", "search_tool"])); +}); + +test("a response that called nothing records an empty list, and one that cannot be decoded records no field at all", async () => { + await withBroker(async ({ turn, requestRows }) => { + await turn("wake-silent", twoRequests); + // Decoded, and it called nothing: that is an observation, not a gap. + assert.deepEqual((await requestRows()).map((row) => row.tool_calls), [[], []]); + }, undefined, () => 1); + await withBroker(async ({ turn, requestRows }) => { + await turn("wake-undecodable", twoRequests); + const rows = await requestRows(); + // Mutation guard: a fabricated `[]` here would be byte-identical to the + // measured empty list above, and the ledger would claim an observation the + // proxy never made. + assert.deepEqual(rows.map((row) => Object.hasOwn(row, "tool_calls")), [false, false]); + assert.deepEqual(rows.map((row) => row.request), [0, 1], "the rows themselves are still written"); + }, undefined, () => 1, () => "bad gateway"); +}); + diff --git a/src/runtime/turnRequestLedger.ts b/src/runtime/turnRequestLedger.ts index bb33992..d0d2153 100644 --- a/src/runtime/turnRequestLedger.ts +++ b/src/runtime/turnRequestLedger.ts @@ -110,7 +110,12 @@ const requestClockFields = (request: Readonly<{ startedAt?: string; endedAt?: st * (the provider response the proxy saw), or `estimated` (no valid usage; the * proxy's conservative charge, see `grokBrokerTurnMeter.ts`). */ -export type GrokTurnRequest = Readonly<{ index: number; input: number; cacheRead: number; cacheWrite: number; output: number; total: number; startedAt?: string; endedAt?: string; usageSource?: "stream" | "upstream" | "estimated" }>; +/** + * `toolCalls` are the tool-call names that request's response carried, as the + * proxy read them (`grokBrokerTurnMeter.ts`): names only, bounded, `[]` for a + * response that called nothing, and absent when no response could be decoded. + */ +export type GrokTurnRequest = Readonly<{ index: number; input: number; cacheRead: number; cacheWrite: number; output: number; total: number; startedAt?: string; endedAt?: string; usageSource?: "stream" | "upstream" | "estimated"; toolCalls?: readonly string[] }>; /** * `requestCount` is the turn's admitted request count when it exceeds the rows: * a killed turn's in-flight request was sent upstream but never reported usage, @@ -125,6 +130,22 @@ export type GrokTurnRequestEntry = Readonly<{ agent: string; wake: string; turn: * reasoning tokens, so `reasoning` is absent rather than zero. `turn` is the * broker idempotency key and `thread` the Grok session id when the stream * named one. + * + * `tool_calls` is what a turn's rows could not say before: whether the model + * ever *tried* to call anything. Two live turns ended with correctly mounted + * tools and no visible attempt, and the rows recorded timings and tokens only, + * so the question could not be answered after the fact. It is names only — + * never arguments, never message content, never a bearer — bounded, `[]` for a + * response that called nothing, and absent for a response that could not be + * decoded, because a fabricated empty list is byte-identical to a measured one. + * + * It is an additive field inside the unchanged + * `noopolis.daimon.turn-requests.v1` row, deliberately without a version bump: + * Spawnfile's usage reader (`spawnfile/src/runtime/usageLedger.ts`) drops every + * line whose `v` it does not recognise while ignoring fields it does not know, + * and Paideia only relocates this stream's path + * (`DAIMON_TURN_REQUESTS_LEDGER_PATH`). A bump is what would blind them; a new + * field is not. */ export const renderGrokTurnRequestLines = (entry: GrokTurnRequestEntry): string => { const at = entry.at ?? new Date().toISOString(); @@ -146,6 +167,7 @@ export const renderGrokTurnRequestLines = (entry: GrokTurnRequestEntry): string output: request.output, total: request.total, ...(request.usageSource === undefined ? {} : { usage_source: request.usageSource }), + ...(request.toolCalls === undefined ? {} : { tool_calls: request.toolCalls }), ...requestClockFields(request) })}\n`).join(""); }; From 1c74e716c0ec76fe55fe71a4f7ee3fb3048c7b12 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 00:11:23 +0200 Subject: [PATCH 27/69] feat: record each brokered request's tool-call names in the per-request ledger --- src/runtime/AGENTS.md | 22 ++++++++++++++++++++++ src/runtime/grokBrokerTurnMeter.test.ts | 6 ++++++ 2 files changed, 28 insertions(+) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 7487b5d..1860ba4 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -353,6 +353,28 @@ is swallowed, because instrumentation must never fail a wake. The existing ledger's version, path, and field list are untouched, so Spawnfile's `v`-pinned reader is unaffected. +Each Grok row also carries `tool_calls`: the tool-call NAMES that request's +response carried, read by the proxy from the body it already buffers for usage +(`parseGrokResponseToolNames` in `grokBrokerTurnMeter.ts`). Timings and tokens +alone cannot answer "did the model ever *try* to call `use_tool` or +`search_tool`", which is exactly the question two live turns left open. Names +only — never arguments, never message content, never a bearer; a `name` that is +not a plain short identifier is recorded as `` rather than passed +through, and the list is bounded at `GROK_REQUEST_TOOL_CALLS_MAX` (16) entries +with a `` last entry, so a pathological response cannot write an +unbounded row. Absence stays absence, as everywhere in these ledgers: a decoded +response that called nothing records `[]`, and a response that could not be +decoded records *no field at all*, because a fabricated empty list is +byte-identical to a measured one. On the stream row path the names are attached +only when the proxy's timings and the worker's stream requests are aligned +request-for-request, since an unaligned index would credit one request's attempt +to another. It is an additive field inside the unchanged +`noopolis.daimon.turn-requests.v1` row and deliberately not a version bump: +Spawnfile's reader pins `v` and ignores fields it does not know, and Paideia +only relocates this stream's path. The whole path is advisory — the parse is +wrapped, and nothing it does can refuse, delay, or fail a turn, or reach the +spend gate. + `testRuntimeSubprocess.ts` is an unexported, explicit-test-only JSONL process surface for exercising the real control, schedule, and acceptance paths with a controlled clock and deterministic scripted cognition. Its ephemeral loopback diff --git a/src/runtime/grokBrokerTurnMeter.test.ts b/src/runtime/grokBrokerTurnMeter.test.ts index 881b8fd..852b514 100644 --- a/src/runtime/grokBrokerTurnMeter.test.ts +++ b/src/runtime/grokBrokerTurnMeter.test.ts @@ -224,6 +224,12 @@ test("a response's tool-call names are read, bounded, and stripped of everything // A name that is not a plain short identifier is counted, never passed through. assert.deepEqual(parseGrokResponseToolNames(events(callDeltas(["ok_tool", "a b/c", "x".repeat(65), "inject\nline"])), "text/event-stream"), ["ok_tool", GROK_TOOL_CALL_INVALID, GROK_TOOL_CALL_INVALID, GROK_TOOL_CALL_INVALID]); + // A hostile but decodable shape yields no attempt instead of throwing: + // instrumentation must never be able to fail the turn it observes. + assert.deepEqual(parseGrokResponseToolNames(events([ + { choices: "not-an-array" }, { choices: [null, 7, { delta: { tool_calls: "no" } }, { message: { tool_calls: [null, { function: null }, { function: { name: 42 } }, { function: { name: "" } }] } }] } + ]), "text/event-stream"), []); + // Mutation guard: unbounded, a pathological response writes 400 names into one row. const many = parseGrokResponseToolNames(events(callDeltas(Array.from({ length: 400 }, (_value, index) => `tool_${index}`))), "text/event-stream"); assert.equal(many!.length, GROK_REQUEST_TOOL_CALLS_MAX); From d8a67dcee896fb9c66da2f03c0b3da4aaf2cac04 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 00:12:12 +0200 Subject: [PATCH 28/69] test: split the identity-envelope tests into their own file under the line limit --- src/runtime/engineDispatcher.test.ts | 55 +---------------- src/runtime/engineDispatcherIdentity.test.ts | 65 ++++++++++++++++++++ 2 files changed, 66 insertions(+), 54 deletions(-) create mode 100644 src/runtime/engineDispatcherIdentity.test.ts diff --git a/src/runtime/engineDispatcher.test.ts b/src/runtime/engineDispatcher.test.ts index dea7789..0dae9a8 100644 --- a/src/runtime/engineDispatcher.test.ts +++ b/src/runtime/engineDispatcher.test.ts @@ -4,8 +4,7 @@ import os from "node:os"; import path from "node:path"; import test from "node:test"; -import { codexSandboxProtectedPaths, codexSandboxReadablePaths, grokSandboxProtectedPaths, identityEnvelope, startOrganizationRuntimeEngine } from "./engineDispatcher.js"; -import { DAIMON_GROK_MCP_SERVER, DAIMON_GROK_SYSTEM_PROMPT, DAIMON_GROK_TOOL_PREFIX, GROK_MCP_INVOKE_TOOL, GROK_MCP_SEARCH_TOOL, GROK_MCP_TOOL_NAME_ARGUMENT, grokDaimonToolName } from "../contracts/grokWorkerContract.js"; +import { codexSandboxProtectedPaths, codexSandboxReadablePaths, grokSandboxProtectedPaths, startOrganizationRuntimeEngine } from "./engineDispatcher.js"; import { AGY_SUBSCRIPTION_REALM, GROK_SUBSCRIPTION_REALM } from "./contractManifest.js"; import type { EngineBrokerTurnClient } from "./engineBrokerControlClient.js"; import { ORGANIZATION_RUNTIME_VERSION, type OrganizationRuntimeAgentConfig } from "./organizationRuntime.js"; @@ -72,58 +71,6 @@ test("Codex strict sandbox protects current credentials, ingress, shared roots, assert.deepEqual(codexSandboxReadablePaths(current), [path.join(current.runtimeHomePath, "tool-output")]); }); -/** - * The envelope's Grok wording is the second naming rule a Grok worker reads, - * after the pinned system prompt. It used to instruct the bare form, which - * Grok 1.0.34 refuses outright as an invalid MCP tool name, so what is asserted - * here is agreement: the same route, stated once, the prefixed form named as - * the only valid one, and no third voice about `search_tool`. - */ -const mounted = ["moltnet_read", "moltnet_send", "memory_search"] as const; -const envelopeToolSentence = (kind: OrganizationRuntimeAgentConfig["engine"]["kind"]): string => - identityEnvelope(rootConfig("/private/org", kind), mounted).split("\n").find((line) => line.startsWith("Your mounted tools are exactly"))!; - -test("the Grok envelope names the prefixed form as the only valid one, once", () => { - const sentence = envelopeToolSentence("grok"); - // The route, asserted: one bare catalogue, one prefix rule, one example. - assert.equal(sentence.includes(`Your mounted tools are exactly: ${mounted.join(", ")}.`), true); - assert.match(sentence, new RegExp(`MCP tool on server ${DAIMON_GROK_MCP_SERVER}`, "u")); - assert.match(sentence, new RegExp(`only valid tool name is ${DAIMON_GROK_TOOL_PREFIX}`, "u")); - assert.match(sentence, new RegExp(`invoke it with ${GROK_MCP_INVOKE_TOOL}, ${GROK_MCP_TOOL_NAME_ARGUMENT} = ${grokDaimonToolName(mounted[0])}`, "u")); - // A bare name is invalid, not merely discouraged: that is the CLI's own verdict. - assert.match(sentence, /A bare name is not a valid MCP tool name and reaches nothing\./u); - assert.match(sentence, /No other tool reaches the newsroom\.$/u); - // What it must never say: the bare names are callable, the agent's own - // instructions are wrong, or a shell reaches the tools. - assert.doesNotMatch(sentence, /Call them by these names/u); - assert.doesNotMatch(sentence, /spell them differently/u); - assert.doesNotMatch(sentence, /shell|terminal|CLI|run_terminal/u); - // Fewer authoritative voices: the pinned prompt and Grok's own injected - // notice already give two rules for `search_tool`. This adds no third. - assert.equal(sentence.includes(GROK_MCP_SEARCH_TOOL), false); - // One catalogue only: the prefixed names are a rule, not a second list. - for (const tool of mounted) assert.equal(sentence.split(tool).length - 1, tool === mounted[0] ? 2 : 1, tool); - assert.equal(sentence.split(DAIMON_GROK_TOOL_PREFIX).length - 1, 2, "prefix appears as the rule and its one example"); - // The transport prohibition is untouched and still follows the tool sentence. - assert.match(identityEnvelope(rootConfig("/private/org", "grok"), mounted), /Do not seek transport credentials or invoke a transport CLI/u); -}); - -test("the Grok envelope and the pinned worker system prompt state the same route", () => { - const sentence = envelopeToolSentence("grok"); - for (const atom of [DAIMON_GROK_MCP_SERVER, DAIMON_GROK_TOOL_PREFIX, GROK_MCP_INVOKE_TOOL, GROK_MCP_TOOL_NAME_ARGUMENT]) { - assert.ok(DAIMON_GROK_SYSTEM_PROMPT.includes(atom), `system prompt states ${atom}`); - assert.ok(sentence.includes(atom), `envelope states ${atom}`); - } -}); - -test("only Grok gains the prefix rule: every other engine's envelope stays byte-identical", () => { - const unchanged = `Your mounted tools are exactly: ${mounted.join(", ")}. Call them by these names; your instructions may spell them differently. No other tool reaches the newsroom.`; - for (const kind of ["codex", "agy"] as const) assert.equal(envelopeToolSentence(kind), unchanged); - assert.notEqual(envelopeToolSentence("grok"), unchanged); - // An unmounted agent gets no tool sentence at all, on every engine. - for (const kind of ["codex", "agy", "grok"] as const) assert.doesNotMatch(identityEnvelope(rootConfig("/private/org", kind)), /Your mounted tools/u); -}); - test("production dispatcher starts each closed engine intent through Daimon", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-dispatcher-")); const priorPath = process.env.PATH; diff --git a/src/runtime/engineDispatcherIdentity.test.ts b/src/runtime/engineDispatcherIdentity.test.ts new file mode 100644 index 0000000..b82dafa --- /dev/null +++ b/src/runtime/engineDispatcherIdentity.test.ts @@ -0,0 +1,65 @@ +import assert from "node:assert/strict"; +import path from "node:path"; +import test from "node:test"; + +import { identityEnvelope } from "./engineDispatcher.js"; +import { DAIMON_GROK_MCP_SERVER, DAIMON_GROK_SYSTEM_PROMPT, DAIMON_GROK_TOOL_PREFIX, GROK_MCP_INVOKE_TOOL, GROK_MCP_SEARCH_TOOL, GROK_MCP_TOOL_NAME_ARGUMENT, grokDaimonToolName } from "../contracts/grokWorkerContract.js"; +import type { OrganizationRuntimeAgentConfig } from "./organizationRuntime.js"; + +const rootConfig = (root: string, kind: OrganizationRuntimeAgentConfig["engine"]["kind"]): OrganizationRuntimeAgentConfig => ({ + id: `${kind}-agent`, name: kind, instructions: "Reply.", + workspacePath: path.join(root, "workspace", kind), runtimeHomePath: path.join(root, "runtime", kind), + engine: { kind } +}); + +/** + * The envelope's Grok wording is the second naming rule a Grok worker reads, + * after the pinned system prompt. It used to instruct the bare form, which + * Grok 1.0.34 refuses outright as an invalid MCP tool name, so what is asserted + * here is agreement: the same route, stated once, the prefixed form named as + * the only valid one, and no third voice about `search_tool`. + */ +const mounted = ["moltnet_read", "moltnet_send", "memory_search"] as const; +const envelopeToolSentence = (kind: OrganizationRuntimeAgentConfig["engine"]["kind"]): string => + identityEnvelope(rootConfig("/private/org", kind), mounted).split("\n").find((line) => line.startsWith("Your mounted tools are exactly"))!; + +test("the Grok envelope names the prefixed form as the only valid one, once", () => { + const sentence = envelopeToolSentence("grok"); + // The route, asserted: one bare catalogue, one prefix rule, one example. + assert.equal(sentence.includes(`Your mounted tools are exactly: ${mounted.join(", ")}.`), true); + assert.match(sentence, new RegExp(`MCP tool on server ${DAIMON_GROK_MCP_SERVER}`, "u")); + assert.match(sentence, new RegExp(`only valid tool name is ${DAIMON_GROK_TOOL_PREFIX}`, "u")); + assert.match(sentence, new RegExp(`invoke it with ${GROK_MCP_INVOKE_TOOL}, ${GROK_MCP_TOOL_NAME_ARGUMENT} = ${grokDaimonToolName(mounted[0])}`, "u")); + // A bare name is invalid, not merely discouraged: that is the CLI's own verdict. + assert.match(sentence, /A bare name is not a valid MCP tool name and reaches nothing\./u); + assert.match(sentence, /No other tool reaches the newsroom\.$/u); + // What it must never say: the bare names are callable, the agent's own + // instructions are wrong, or a shell reaches the tools. + assert.doesNotMatch(sentence, /Call them by these names/u); + assert.doesNotMatch(sentence, /spell them differently/u); + assert.doesNotMatch(sentence, /shell|terminal|CLI|run_terminal/u); + // Fewer authoritative voices: the pinned prompt and Grok's own injected + // notice already give two rules for `search_tool`. This adds no third. + assert.equal(sentence.includes(GROK_MCP_SEARCH_TOOL), false); + // One catalogue only: the prefixed names are a rule, not a second list. + for (const tool of mounted) assert.equal(sentence.split(tool).length - 1, tool === mounted[0] ? 2 : 1, tool); + assert.equal(sentence.split(DAIMON_GROK_TOOL_PREFIX).length - 1, 2, "prefix appears as the rule and its one example"); + // The transport prohibition is untouched and still follows the tool sentence. + assert.match(identityEnvelope(rootConfig("/private/org", "grok"), mounted), /Do not seek transport credentials or invoke a transport CLI/u); +}); + +test("the Grok envelope and the pinned worker system prompt state the same route", () => { + const sentence = envelopeToolSentence("grok"); + for (const atom of [DAIMON_GROK_MCP_SERVER, DAIMON_GROK_TOOL_PREFIX, GROK_MCP_INVOKE_TOOL, GROK_MCP_TOOL_NAME_ARGUMENT]) { + assert.ok(DAIMON_GROK_SYSTEM_PROMPT.includes(atom), `system prompt states ${atom}`); + assert.ok(sentence.includes(atom), `envelope states ${atom}`); + } +}); + +test("only Grok gains the prefix rule: every other engine's envelope stays byte-identical", () => { + const unchanged = `Your mounted tools are exactly: ${mounted.join(", ")}. Call them by these names; your instructions may spell them differently. No other tool reaches the newsroom.`; + for (const kind of ["codex", "agy"] as const) assert.equal(envelopeToolSentence(kind), unchanged); + assert.notEqual(envelopeToolSentence("grok"), unchanged); + // An unmounted agent gets no tool sentence at all, on every engine. + for (const kind of ["codex", "agy", "grok"] as const) assert.doesNotMatch(identityEnvelope(rootConfig("/private/org", kind)), /Your mounted tools/u); +}); From 964dda31f76f40604445c8805c6af3813e1ec90d Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 00:14:12 +0200 Subject: [PATCH 29/69] fix: prefix the inbox prompt's oversized-payload branch for Grok too --- src/runtime/attentionDispatcher.test.ts | 22 +++++++++++++++++++++- src/runtime/attentionDispatcher.ts | 4 ++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/runtime/attentionDispatcher.test.ts b/src/runtime/attentionDispatcher.test.ts index 3e61593..6e1c0b6 100644 --- a/src/runtime/attentionDispatcher.test.ts +++ b/src/runtime/attentionDispatcher.test.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { AttentionDispatcher } from "./attentionDispatcher.js"; +import { AttentionDispatcher, inboxPrompt } from "./attentionDispatcher.js"; import { DAIMON_GROK_TOOL_PREFIX, grokDaimonToolName } from "../contracts/grokWorkerContract.js"; import { createOrganizationRuntimeHost } from "./organizationRuntimeHost.js"; import { WakeAcceptanceStore, WakeExecutionClaimLostError } from "./wakeAcceptanceStore.js"; @@ -243,3 +243,23 @@ test("every other engine's inbox turn keeps the bare tool names", async () => { assert.equal(text.includes(DAIMON_GROK_TOOL_PREFIX), false); } finally { await f.cleanup(); } }); + +/** + * Every branch of the inbox prompt, not only the one a small delivery takes: + * the oversized-payload fallback is the branch a busy agent meets, and it names + * `daimon_inbox` too. + */ +test("every branch of the inbox prompt names its tools the way the engine can call them", () => { + const delivery = { acceptance_id: "a-1", delivery_id: "d-1", kind: "message", text: "Do the thing", occurred_at: "2026-09-11T00:00:00.000Z" }; + const oversized = { ...delivery, text: "x".repeat(2_000) }; + for (const [messages, budget] of [[[delivery], 12_000], [[oversized], 64], [[oversized], 8]] as const) { + const grok = inboxPrompt(messages, "grok", budget), codex = inboxPrompt(messages, "codex", budget); + // Mutation guard: an unprefixed branch leaves a bare name in the Grok text. + assert.equal(grok.split("daimon_inbox").length - 1, grok.split(DAIMON_GROK_TOOL_PREFIX).length - 1, grok); + assert.ok(grok.includes(grokDaimonToolName("daimon_inbox")), grok); + assert.equal(codex.includes(DAIMON_GROK_TOOL_PREFIX), false, codex); + assert.ok(codex.includes("daimon_inbox"), codex); + } + // The smallest budget is the fallback that only points at the tool. + assert.match(inboxPrompt([oversized], "grok", 8), new RegExp(`exceeds the prompt budget; read it with ${grokDaimonToolName("daimon_inbox")}\\.$`, "u")); +}); diff --git a/src/runtime/attentionDispatcher.ts b/src/runtime/attentionDispatcher.ts index 5d842b5..ad204fc 100644 --- a/src/runtime/attentionDispatcher.ts +++ b/src/runtime/attentionDispatcher.ts @@ -202,7 +202,7 @@ function deliveryBlock(message: unknown, index: number): string | undefined { * cannot mark its work complete — and an unmarked, finished wake is recorded as * deferred. */ -function inboxPrompt(messages: readonly unknown[], engine: OrganizationRuntimeAgentConfig["engine"]["kind"], maxBytes = 12000): string { +export function inboxPrompt(messages: readonly unknown[], engine: OrganizationRuntimeAgentConfig["engine"]["kind"], maxBytes = 12000): string { const body = JSON.stringify(messages); const blocks = messages.map(deliveryBlock).filter((block): block is string => block !== undefined); const tool = (name: string): string => engine === "grok" ? grokDaimonToolName(name) : name; @@ -221,7 +221,7 @@ function inboxPrompt(messages: readonly unknown[], engine: OrganizationRuntimeAg const prefix = `Handle this inbox turn. Use ${tool("daimon_inbox")} for deliveries and remaining allowances. Explicitly call ${tool("daimon_inbox_disposition")} for each handled delivery (complete) or unfinished delivery (defer). Reading or ending this turn never completes a delivery. Deferred work waits for a later external wake.\n`; const prompt = prefix + body; if (Buffer.byteLength(body) > maxBytes || !fits(prompt)) { - return prefix + "The selected payload exceeds the prompt budget; read it with daimon_inbox."; + return prefix + `The selected payload exceeds the prompt budget; read it with ${tool("daimon_inbox")}.`; } return prompt; } From 23b28e97556c58fb8f1b552c19fad7d86cf7fe15 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 00:16:57 +0200 Subject: [PATCH 30/69] fix: forward the MCP session and protocol headers and the GET/DELETE routes through the broker facade --- src/runtime/engineBrokerMcpFacade.test.ts | 231 +++++++++++++++++++++- src/runtime/engineBrokerMcpFacade.ts | 202 ++++++++++++++++++- 2 files changed, 424 insertions(+), 9 deletions(-) diff --git a/src/runtime/engineBrokerMcpFacade.test.ts b/src/runtime/engineBrokerMcpFacade.test.ts index 5244b4f..c19272a 100644 --- a/src/runtime/engineBrokerMcpFacade.test.ts +++ b/src/runtime/engineBrokerMcpFacade.test.ts @@ -1,10 +1,235 @@ import assert from "node:assert/strict"; -import { createServer } from "node:http"; +import { createServer, type Server as HttpServer, type IncomingMessage } from "node:http"; +import { randomUUID } from "node:crypto"; import test from "node:test"; -import { startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; + +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; +import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; +import { ToolListChangedNotificationSchema } from "@modelcontextprotocol/sdk/types.js"; +import { Type } from "@earendil-works/pi-ai"; +import { defineTool } from "@earendil-works/pi-coding-agent"; + +import { createPiToolMcpServer } from "../mcp/toolServer.js"; +import { ENGINE_BROKER_MCP_FACADE_PORT, startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; + +const FACADE_URL = `http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}/mcp`; test("MCP facade routes only valid active capabilities to the registered mount", async () => { let calls=0;const target=createServer((_request,response)=>{calls++;response.writeHead(200,{"content-type":"application/json"});response.end('{"ok":true}');});await new Promise((resolve)=>target.listen(0,"127.0.0.1",resolve));const address=target.address();if(address===null||typeof address==="string")throw new Error(); - const facade=await startEngineBrokerMcpFacade();const token=facade.register("agent","turn",`http://127.0.0.1:${address.port}/mcp`);const call=(value:string)=>fetch("http://127.0.0.1:43124/mcp",{method:"POST",headers:{authorization:`Bearer ${value}`,"content-type":"application/json"},body:"{}"}); + const facade=await startEngineBrokerMcpFacade();const token=facade.register("agent","turn",`http://127.0.0.1:${address.port}/mcp`);const call=(value:string)=>fetch(FACADE_URL,{method:"POST",headers:{authorization:`Bearer ${value}`,"content-type":"application/json"},body:"{}"}); try{assert.equal((await call("wrong-token-abcdefghijklmnopqrstuvwxyz0123456789")).status,403);assert.equal((await call(token)).status,200);assert.equal(calls,1);facade.revoke("turn");assert.equal((await call(token)).status,403);assert.equal(calls,1);}finally{await facade.close();await new Promise((resolve)=>target.close(()=>resolve()));} }); + +/** + * The brokered worker's real route: a real Daimon MCP mount behind a real + * Streamable HTTP transport, reached by a real MCP client through the facade. + * Asserting that a header is copied would pass while the route stayed broken, + * so every case below drives the transport end to end. + */ +type Rig = Readonly<{ + facade: Awaited>; + mount: HttpServer; + transport: StreamableHTTPServerTransport; + server: ReturnType; + capability: string; + observed: IncomingMessage[]; + close: () => Promise; +}>; + +const echoTool = defineTool({ + name: "moltnet_read", + label: "Read a scoped Moltnet surface", + description: "Reads the fixture room.", + parameters: Type.Object({ target: Type.String() }, { additionalProperties: false }), + async execute(_toolCallId: string, params: { target: string }) { + return { content: [{ type: "text" as const, text: `read ${params.target}` }], details: { target: params.target } }; + } +}); + +const startRig = async (): Promise => { + const server = createPiToolMcpServer([echoTool], {}); + const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: () => randomUUID() }); + await server.connect(transport); + const observed: IncomingMessage[] = []; + const mount = createServer((request, response) => { + observed.push(request); + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.on("end", () => { + const raw = Buffer.concat(chunks); + let parsed: unknown; + try { parsed = raw.length === 0 ? undefined : JSON.parse(raw.toString("utf8")); } catch { parsed = undefined; } + void transport.handleRequest(request, response, parsed); + }); + }); + await new Promise((resolve) => mount.listen(0, "127.0.0.1", resolve)); + const address = mount.address(); + if (address === null || typeof address === "string") throw new Error("mount address unavailable"); + const facade = await startEngineBrokerMcpFacade(); + const capability = facade.register("alpha", "turn-1", `http://127.0.0.1:${address.port}/mcp`); + return { + facade, mount, transport, server, capability, observed, + close: async () => { + facade.revoke("turn-1"); + await facade.close(); + await new Promise((resolve) => mount.close(() => resolve())); + await transport.close().catch(() => undefined); + await server.close().catch(() => undefined); + } + }; +}; + +const connectClient = async (capability: string): Promise<{ client: Client; transport: StreamableHTTPClientTransport }> => { + const transport = new StreamableHTTPClientTransport(new URL(FACADE_URL), { + requestInit: { headers: { authorization: `Bearer ${capability}` } } + }); + const client = new Client({ name: "daimon-facade-test-client", version: "0.1.0" }); + await client.connect(transport); + return { client, transport }; +}; + +test("a brokered worker completes the whole MCP handshake through the facade and calls a mounted tool", async () => { + const rig = await startRig(); + try { + // connect() is initialize + notifications/initialized: before the session + // header was forwarded the notification came back HTTP 400. + const { client, transport } = await connectClient(rig.capability); + try { + assert.equal(typeof transport.sessionId, "string", "the mount's session id must reach the client"); + const listed = await client.listTools(); + assert.deepEqual(listed.tools.map((tool) => tool.name), ["moltnet_read"]); + const called = await client.callTool({ name: "moltnet_read", arguments: { target: "room:desk" } }); + assert.deepEqual(called.structuredContent, { target: "room:desk" }); + assert.equal(called.isError, undefined); + } finally { + await client.close(); + } + } finally { + await rig.close(); + } +}); + +test("the facade carries the mount's server-initiated SSE stream, which only the GET route provides", async () => { + const rig = await startRig(); + try { + const { client, transport } = await connectClient(rig.capability); + const notified = new Promise((resolve) => client.setNotificationHandler(ToolListChangedNotificationSchema, () => resolve())); + try { + // The standalone GET stream is the only route a server notification can + // take; a POST-only facade answers it 403 and this never arrives. + await new Promise((resolve) => setTimeout(resolve, 150)); + rig.server.sendToolListChanged(); + await Promise.race([ + notified, + new Promise((_resolve, reject) => setTimeout(() => reject(new Error("no server notification reached the client")), 4_000)) + ]); + assert.ok(rig.observed.some((request) => request.method === "GET"), "the mount must have seen the GET stream"); + assert.equal(typeof transport.sessionId, "string"); + } finally { + await client.close(); + } + } finally { + await rig.close(); + } +}); + +test("the facade carries the session-closing DELETE, and the mount then refuses the stale session", async () => { + const rig = await startRig(); + try { + const { client, transport } = await connectClient(rig.capability); + const sessionId = transport.sessionId; + assert.equal(typeof sessionId, "string"); + await transport.terminateSession(); + assert.equal(transport.sessionId, undefined, "DELETE must be accepted, not 403ed"); + assert.ok(rig.observed.some((request) => request.method === "DELETE"), "the mount must have seen the DELETE"); + const stale = await fetch(FACADE_URL, { + method: "POST", + headers: { + authorization: `Bearer ${rig.capability}`, + "content-type": "application/json", + accept: "application/json, text/event-stream", + "mcp-session-id": sessionId! + }, + body: JSON.stringify({ jsonrpc: "2.0", id: 9, method: "tools/list", params: {} }) + }); + assert.ok(stale.status >= 400, `a terminated session must not still route (got ${stale.status})`); + await stale.body?.cancel(); + await client.close().catch(() => undefined); + } finally { + await rig.close(); + } +}); + +test("the facade forwards a closed header allowlist and never the worker's bearer", async () => { + const rig = await startRig(); + try { + const { client } = await connectClient(rig.capability); + await client.listTools(); + await client.close(); + const forwarded = rig.observed.flatMap((request) => Object.keys(request.headers)); + assert.equal(forwarded.includes("authorization"), false, "the turn capability must never reach the mount"); + assert.equal(forwarded.includes("cookie"), false); + const allowed = new Set(["host", "connection", "content-length", "transfer-encoding", "accept-encoding", "accept-language", "user-agent", "content-type", "accept", "mcp-session-id", "mcp-protocol-version", "last-event-id", + // undici's own outbound headers, set by the facade's fetch rather than forwarded from the worker. + "sec-fetch-mode"]); + const unexpected = [...new Set(forwarded)].filter((name) => !allowed.has(name)); + assert.deepEqual(unexpected, [], `unexpected headers reached the mount: ${unexpected.join(",")}`); + } finally { + await rig.close(); + } +}); + +test("the facade withholds a mount response header that is not on the allowlist", async () => { + const target = createServer((_request, response) => { + response.writeHead(200, { + "content-type": "application/json", + "mcp-session-id": "session-from-mount", + "set-cookie": "leak=1", + "www-authenticate": "Bearer realm=\"mount\"", + "x-mount-internal": "private" + }); + response.end('{"ok":true}'); + }); + await new Promise((resolve) => target.listen(0, "127.0.0.1", resolve)); + const address = target.address(); + if (address === null || typeof address === "string") throw new Error("target address unavailable"); + const facade = await startEngineBrokerMcpFacade(); + const capability = facade.register("alpha", "turn-1", `http://127.0.0.1:${address.port}/mcp`); + try { + const answered = await fetch(FACADE_URL, { + method: "POST", + headers: { authorization: `Bearer ${capability}`, "content-type": "application/json" }, + body: "{}" + }); + assert.equal(answered.status, 200); + assert.equal(answered.headers.get("mcp-session-id"), "session-from-mount"); + assert.equal(answered.headers.get("cache-control"), "no-store"); + assert.equal(answered.headers.get("set-cookie"), null); + assert.equal(answered.headers.get("www-authenticate"), null); + assert.equal(answered.headers.get("x-mount-internal"), null); + await answered.body?.cancel(); + } finally { + facade.revoke("turn-1"); + await facade.close(); + await new Promise((resolve) => target.close(() => resolve())); + } +}); + +test("the facade still refuses every route and method outside the MCP surface", async () => { + const facade = await startEngineBrokerMcpFacade(); + const capability = facade.register("alpha", "turn-1", "http://127.0.0.1:1/mcp"); + const call = (method: string, path: string) => fetch(`http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}${path}`, { + method, headers: { authorization: `Bearer ${capability}` } + }); + try { + assert.equal((await call("GET", "/")).status, 403); + assert.equal((await call("GET", "/mcp?probe=1")).status, 403); + assert.equal((await call("PUT", "/mcp")).status, 403); + assert.equal((await call("PATCH", "/mcp")).status, 403); + assert.equal((await fetch(`http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}/mcp`, { method: "GET" })).status, 403); + } finally { + facade.revoke("turn-1"); + await facade.close(); + } +}); diff --git a/src/runtime/engineBrokerMcpFacade.ts b/src/runtime/engineBrokerMcpFacade.ts index 156506b..ee1defd 100644 --- a/src/runtime/engineBrokerMcpFacade.ts +++ b/src/runtime/engineBrokerMcpFacade.ts @@ -1,10 +1,200 @@ -import { createServer } from "node:http"; +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import { Readable } from "node:stream"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; +/** + * The brokered worker's only route to its own per-wake Daimon MCP mount. The + * worker holds a turn capability and nothing else: it never learns the mount's + * address, and the mount never learns the capability. Every header crossing + * either way is rebuilt from a closed allowlist, so this stays a boundary and + * not a transparent proxy — a blanket passthrough would hand the mount the + * worker's bearer and hand the worker whatever the mount chose to say. + * + * The allowlists are exactly the Streamable HTTP transport's own routing + * headers. Anything outside them (authorization above all, cookies, auth + * challenges, forwarding and tracing headers) is dropped in both directions. + * + * Client -> mount: + * - `content-type`: the JSON-RPC body's media type; the mount refuses a POST + * without it. + * - `accept`: the transport negotiates `application/json, text/event-stream` + * per request and the mount answers 406 when a POST does not accept both. + * - `mcp-session-id`: the opaque session the mount issued on `initialize`. + * Dropping it made the mount answer every later request with HTTP 400 + * `Mcp-Session-Id header is required`, so no tool was ever reachable. It is + * a routing value, not a secret — and not a value to log either. + * - `mcp-protocol-version`: the version the handshake settled on. The mount + * validates it and otherwise assumes a default that can disagree with what + * the client negotiated. + * - `last-event-id`: SSE resumability. A reconnecting stream replays from the + * last event it saw; without it the mount cannot tell where to resume. + * + * Mount -> client: + * - `content-type`: tells the client whether it got JSON or an SSE stream. + * - `mcp-session-id`: the id minted on `initialize`. The client must learn it + * or it can never make a second request. + * - `mcp-protocol-version`: the version the mount confirms for the session. + * - `cache-control: no-store` is the facade's own, not the mount's. + * + * Methods are the three the transport uses: POST for JSON-RPC, GET for the + * server-to-client SSE stream (notifications and progress arrive only there), + * and DELETE to end a session. POST alone left the GET stream answering 403. + */ +const FORWARDED_REQUEST_HEADERS = ["content-type", "accept", "mcp-session-id", "mcp-protocol-version", "last-event-id"] as const; +const FORWARDED_RESPONSE_HEADERS = ["content-type", "mcp-session-id", "mcp-protocol-version"] as const; +const FORWARDED_METHODS = new Set(["POST", "GET", "DELETE"]); +const MAX_REQUEST_BYTES = 1024 * 1024; +export const ENGINE_BROKER_MCP_FACADE_PORT = 43_124; + +class FacadeRefusal extends Error {} + export async function startEngineBrokerMcpFacade() { - const capabilities=new EngineBrokerCapabilities();const targets=new Map(); - const server=createServer((request,response)=>{void(async()=>{try{if(request.url!=="/mcp"||request.method!=="POST")throw new Error();const match=request.headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u);if(!match)throw new Error();const scope=capabilities.authorizeToken(match[1]!);if(!scope)throw new Error();const target=targets.get(scope.turnId);if(!target)throw new Error();const body=await bounded(request);const payload=body.buffer.slice(body.byteOffset,body.byteOffset+body.byteLength) as ArrayBuffer;const upstream=await fetch(target,{method:"POST",headers:{"content-type":request.headers["content-type"]??"application/json","accept":request.headers.accept??"application/json, text/event-stream"},body:payload});response.writeHead(upstream.status,{"content-type":upstream.headers.get("content-type")??"application/json","cache-control":"no-store"});response.end(Buffer.from(await upstream.arrayBuffer()));}catch{response.writeHead(403,{"content-type":"application/json","cache-control":"no-store"});response.end('{"error":"forbidden"}');}})();}); - await new Promise((resolve,reject)=>{server.once("error",reject);server.listen(43_124,"127.0.0.1",()=>{server.off("error",reject);resolve();});}); - return {register(agentId:string,turnId:string,endpoint:string){const url=new URL(endpoint);if(url.protocol!=="http:"||url.hostname!=="127.0.0.1"||url.pathname!=="/mcp")throw new TypeError("invalid scoped MCP mount");if(targets.has(turnId))throw new Error("MCP turn already registered");targets.set(turnId,url.href);return capabilities.issue(agentId,turnId,15*60_000,128);},revoke(turnId:string){targets.delete(turnId);capabilities.revoke(turnId);},close:()=>new Promise((resolve,reject)=>server.close((error)=>error?reject(error):resolve()))}; + const capabilities = new EngineBrokerCapabilities(); + const targets = new Map(); + /** In-flight upstream calls per turn, so a revoke or a close tears down any open SSE tunnel. */ + const inflight = new Map>(); + + const server = createServer((request, response) => { + void route(request, response).catch((error: unknown) => { + if (response.headersSent || response.destroyed) { response.destroy(); return; } + const status = error instanceof FacadeRefusal ? 403 : 502; + const body = error instanceof FacadeRefusal ? '{"error":"forbidden"}' : '{"error":"bad_gateway"}'; + response.writeHead(status, { "content-type": "application/json", "cache-control": "no-store" }); + response.end(body); + }); + }); + + async function route(request: IncomingMessage, response: ServerResponse): Promise { + const method = request.method ?? ""; + if (request.url !== "/mcp" || !FORWARDED_METHODS.has(method)) throw new FacadeRefusal(); + const match = request.headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u); + if (!match) throw new FacadeRefusal(); + const scope = capabilities.authorizeToken(match[1]!); + if (!scope) throw new FacadeRefusal(); + const target = targets.get(scope.turnId); + if (target === undefined) throw new FacadeRefusal(); + + // Only POST carries a JSON-RPC body; drain anything else so the socket + // never stalls waiting for a body the facade will not forward. + const body = method === "POST" ? await bounded(request) : (request.resume(), undefined); + const payload = body === undefined ? undefined : (body.buffer.slice(body.byteOffset, body.byteOffset + body.byteLength) as ArrayBuffer); + + const controller = new AbortController(); + const open = inflight.get(scope.turnId) ?? new Set(); + open.add(controller); + inflight.set(scope.turnId, open); + const abort = (): void => controller.abort(); + response.on("close", abort); + try { + await forward(target, method, headersFor(method, request), payload, controller.signal, response); + } finally { + response.off("close", abort); + open.delete(controller); + if (open.size === 0) inflight.delete(scope.turnId); + } + } + + function endTurnStreams(turnId: string): void { + for (const controller of inflight.get(turnId) ?? []) controller.abort(); + inflight.delete(turnId); + } + + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(ENGINE_BROKER_MCP_FACADE_PORT, "127.0.0.1", () => { server.off("error", reject); resolve(); }); + }); + + return { + register(agentId: string, turnId: string, endpoint: string): string { + const url = new URL(endpoint); + if (url.protocol !== "http:" || url.hostname !== "127.0.0.1" || url.pathname !== "/mcp") throw new TypeError("invalid scoped MCP mount"); + if (targets.has(turnId)) throw new Error("MCP turn already registered"); + targets.set(turnId, url.href); + return capabilities.issue(agentId, turnId, 15 * 60_000, 128); + }, + revoke(turnId: string): void { + targets.delete(turnId); + capabilities.revoke(turnId); + endTurnStreams(turnId); + }, + close: async (): Promise => { + for (const turnId of [...inflight.keys()]) endTurnStreams(turnId); + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + // A GET SSE tunnel keeps its socket open indefinitely, and `close` + // only stops accepting; without this a shutdown would hang on it. + server.closeAllConnections(); + }); + } + }; +} + +/** The client -> mount allowlist, with the two defaults the mount requires of a POST. */ +function headersFor(method: string, request: IncomingMessage): Record { + const headers: Record = {}; + for (const name of FORWARDED_REQUEST_HEADERS) { + const value = request.headers[name]; + if (typeof value === "string" && value.length > 0) headers[name] = value; + } + if (method === "POST") { + headers["content-type"] ??= "application/json"; + headers["accept"] ??= "application/json, text/event-stream"; + } else { + delete headers["content-type"]; + } + return headers; +} + +/** + * Streams the exchange rather than buffering it: a GET stream stays open for + * the whole session, and a buffered POST would withhold progress + * notifications until the call had already finished. + */ +async function forward( + target: string, + method: string, + headers: Record, + body: ArrayBuffer | undefined, + signal: AbortSignal, + response: ServerResponse +): Promise { + const upstream = await fetch(target, { method, headers, body, signal, redirect: "manual" }); + // MCP never redirects, and following one would let the mount aim the facade + // at a host the capability was never scoped to. `manual` also reports an + // opaque redirect as status 0, which is not a status to relay at all. + const relayable = (upstream.status >= 200 && upstream.status < 300) || (upstream.status >= 400 && upstream.status <= 599); + if (!relayable) throw new Error("unexpected MCP mount status"); + const outbound: Record = { "cache-control": "no-store" }; + for (const name of FORWARDED_RESPONSE_HEADERS) { + const value = upstream.headers.get(name); + if (value !== null && value.length > 0) outbound[name] = value; + } + outbound["content-type"] ??= "application/json"; + response.writeHead(upstream.status, outbound); + if (upstream.body === null) { response.end(); return; } + const stream = Readable.fromWeb(upstream.body as Parameters[0]); + try { + for await (const chunk of stream) { + if (!response.write(chunk as Uint8Array)) await new Promise((resolve) => response.once("drain", resolve)); + } + response.end(); + } catch { + // The client hung up or the mount's stream broke: tear the tunnel down + // rather than leaving a half-written response open. + response.destroy(); + } finally { + stream.destroy(); + } +} + +async function bounded(request: AsyncIterable): Promise { + const chunks: Buffer[] = []; + let bytes = 0; + for await (const chunk of request) { + const value = Buffer.from(chunk as Uint8Array); + bytes += value.length; + if (bytes > MAX_REQUEST_BYTES) throw new FacadeRefusal(); + chunks.push(value); + } + return Buffer.concat(chunks); } -async function bounded(request:AsyncIterable):Promise{const chunks:Buffer[]=[];let bytes=0;for await(const chunk of request){const value=Buffer.from(chunk as Uint8Array);bytes+=value.length;if(bytes>1024*1024)throw new Error();chunks.push(value);}return Buffer.concat(chunks);} From 8d5c26c2cee1e486fca02f2df326282c67373ebf Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 00:16:57 +0200 Subject: [PATCH 31/69] docs: record the broker MCP facade's closed header allowlist and supported methods --- src/runtime/AGENTS.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 1860ba4..6276db1 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -179,6 +179,26 @@ model declares it, so the declared effort is the model's single `reasoning_efforts` entry. HTTP MCP needs CA certificates in the image even for a loopback `http://` URL ("Failed to build HTTP client"). +`engineBrokerMcpFacade.ts` is the worker's only route to its per-wake MCP mount +and rebuilds every header from a closed allowlist in both directions, so the +worker's bearer never reaches the mount and no mount header reaches the worker +uninvited. That allowlist must include the Streamable HTTP transport's own +routing headers or the route does not exist: forwarding only +`content-type`/`accept` destroyed `Mcp-Session-Id`, so `initialize` returned 200 +while every request after it — `notifications/initialized`, `tools/list`, +`tools/call` — came back HTTP 400 `Mcp-Session-Id header is required`, and the +model saw `search_tool` answer `{"results":[],"total_hidden_tools":0,"status": +"partial"}`. Client to mount: `content-type`, `accept`, `mcp-session-id`, +`mcp-protocol-version`, `last-event-id`. Mount to client: `content-type`, +`mcp-session-id`, `mcp-protocol-version`, plus the facade's own +`cache-control: no-store`. The session id is an opaque routing value and is +never logged or ledgered. The facade also carries the three methods the +transport uses — POST, the standalone `GET` SSE stream that is the only route a +server notification or progress frame can take, and the `DELETE` that ends a +session — and streams each body rather than buffering it, because a GET tunnel +stays open for the whole session. Never widen it into a transparent proxy: the +whole point of the boundary is that the allowlist is closed. + Worker `GROK_HOME` layout the deployment must provision (attested before every turn by `grokWorkerHomeAttestation.ts`, recorded in `GROK_ENGINE_BROKER.worker.home`): `$GROK_HOME` and `$GROK_HOME/sessions` `root: 1771`; `config.toml`, From a43d62c1b977b016f633d0aa6821c84271f28498 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 00:18:03 +0200 Subject: [PATCH 32/69] fix: name the transport send tool the way Grok can call it --- src/runtime/engineDispatcher.test.ts | 2 +- src/runtime/engineDispatcher.ts | 18 ++++++++++-- src/runtime/engineDispatcherIdentity.test.ts | 29 ++++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/runtime/engineDispatcher.test.ts b/src/runtime/engineDispatcher.test.ts index 0dae9a8..ca1b5e1 100644 --- a/src/runtime/engineDispatcher.test.ts +++ b/src/runtime/engineDispatcher.test.ts @@ -290,7 +290,7 @@ test("Daimon frames one escaped identity envelope for every production engine", const envelope = JSON.stringify({ id: config.id, name: identity.name, instructions: identity.instructions }); assert.equal(result.text.split(envelope).length - 1, 1); assert.match(result.text, //); - assert.match(result.text, /Colleagues only hear you when you call moltnet_send/u); + assert.ok(result.text.includes(`Colleagues only hear you when you call ${kind === "grok" ? "daimon__moltnet_send" : "moltnet_send"};`), `${kind} must name the send tool the way it can call it`); assert.match(result.text, /Do not seek transport credentials or invoke a transport CLI/u); assert.match(result.text, /payload/); await handle.stop(); diff --git a/src/runtime/engineDispatcher.ts b/src/runtime/engineDispatcher.ts index 40b06f1..9b72149 100644 --- a/src/runtime/engineDispatcher.ts +++ b/src/runtime/engineDispatcher.ts @@ -1,5 +1,5 @@ import { attentionTools, type AttentionRegistry } from "./attention.js"; -import { grokMountedToolNamingRule } from "../contracts/grokWorkerContract.js"; +import { grokDaimonToolName, grokMountedToolNamingRule } from "../contracts/grokWorkerContract.js"; import path from "node:path"; import type { AgentHandle } from "../core/types.js"; @@ -207,12 +207,26 @@ export function identityEnvelope(agent: OrganizationRuntimeAgentConfig, mountedT + "your instructions may spell them differently.") + " No other tool reaches the newsroom." ]), - "Colleagues only hear you when you call moltnet_send; your terminal response is a private note to the runtime, not a message to anyone — keep it to one line or leave it empty. " + // The one tool that reaches colleagues is named the way this engine can + // call it. Meaning and prohibition are unchanged; only the spelling is. + `Colleagues only hear you when you call ${engineToolName(agent, "moltnet_send")}; your terminal response is a private note to the runtime, not a message to anyone — keep it to one line or leave it empty. ` + "Do not seek transport credentials or invoke a transport CLI unless the caller explicitly mounted an authenticated transport tool.", "The following is the current wake event." ].join("\n") + "\n"; } +/** + * One Daimon tool name, spelled the way this agent's engine accepts it. + * + * On Grok the bare form is refused as an invalid MCP tool name, so any + * engine-facing sentence that *names* a tool renders it through the contract's + * `grokDaimonToolName`; every other engine keeps the bare name byte for byte. + * Grok's own native tools (`read_file`, `search_tool`, `use_tool`) are not + * Daimon tools and never take the prefix. + */ +const engineToolName = (agent: OrganizationRuntimeAgentConfig, tool: string): string => + agent.engine.kind === "grok" ? grokDaimonToolName(tool) : tool; + function cliHarness( agent: OrganizationRuntimeAgentConfig, sessionFactory: ReturnType, diff --git a/src/runtime/engineDispatcherIdentity.test.ts b/src/runtime/engineDispatcherIdentity.test.ts index b82dafa..13b1607 100644 --- a/src/runtime/engineDispatcherIdentity.test.ts +++ b/src/runtime/engineDispatcherIdentity.test.ts @@ -56,10 +56,39 @@ test("the Grok envelope and the pinned worker system prompt state the same route } }); +/** + * The transport sentence names the one tool an agent needs to reach its + * colleagues. A bare `moltnet_send` one line under "a bare name is not a valid + * MCP tool name and reaches nothing" is the same self-contradiction, on the + * tool that matters most. + */ +test("the transport sentence names the send tool the way Grok can call it, prohibition unchanged", () => { + const grok = identityEnvelope(rootConfig("/private/org", "grok"), mounted); + const transport = grok.split("\n").find((line) => line.startsWith("Colleagues only hear you"))!; + // Mutation guard: un-prefixing this name leaves the bare form the line above declares invalid. + assert.ok(transport.includes(`you call ${grokDaimonToolName("moltnet_send")};`), transport); + assert.equal(new RegExp(`(? { const unchanged = `Your mounted tools are exactly: ${mounted.join(", ")}. Call them by these names; your instructions may spell them differently. No other tool reaches the newsroom.`; for (const kind of ["codex", "agy"] as const) assert.equal(envelopeToolSentence(kind), unchanged); assert.notEqual(envelopeToolSentence("grok"), unchanged); // An unmounted agent gets no tool sentence at all, on every engine. for (const kind of ["codex", "agy", "grok"] as const) assert.doesNotMatch(identityEnvelope(rootConfig("/private/org", kind)), /Your mounted tools/u); + // The transport sentence keeps its bare spelling on every other engine. + const transport = "Colleagues only hear you when you call moltnet_send; your terminal response is a private note to the runtime, not a message to anyone — keep it to one line or leave it empty. Do not seek transport credentials or invoke a transport CLI unless the caller explicitly mounted an authenticated transport tool."; + for (const kind of ["codex", "agy"] as const) { + assert.ok(identityEnvelope(rootConfig("/private/org", kind), mounted).includes(`\n${transport}\n`), kind); + assert.equal(identityEnvelope(rootConfig("/private/org", kind), mounted).includes(DAIMON_GROK_TOOL_PREFIX), false, kind); + } }); From 9d773aede5b51c6559af29fe8d42c01f8f61aab0 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 00:38:11 +0200 Subject: [PATCH 33/69] test: share one facade across the broker MCP facade tests and cover revoke and shutdown teardown --- src/runtime/engineBrokerMcpFacade.test.ts | 136 +++++++++++++++++----- 1 file changed, 107 insertions(+), 29 deletions(-) diff --git a/src/runtime/engineBrokerMcpFacade.test.ts b/src/runtime/engineBrokerMcpFacade.test.ts index c19272a..95d0b65 100644 --- a/src/runtime/engineBrokerMcpFacade.test.ts +++ b/src/runtime/engineBrokerMcpFacade.test.ts @@ -14,11 +14,40 @@ import { createPiToolMcpServer } from "../mcp/toolServer.js"; import { ENGINE_BROKER_MCP_FACADE_PORT, startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; const FACADE_URL = `http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}/mcp`; +type Facade = Awaited>; + +/** + * One facade serves every turn of a broker, so the tests share one too. It + * also keeps the fixed port free: a facade per test would leave the HTTP + * client pooling a socket onto a server that no longer exists. + */ +let shared: Facade | undefined; +const sharedFacade = async (): Promise => (shared ??= await startFacade()); +const releaseShared = async (): Promise => { const facade = shared; shared = undefined; if (facade) await facade.close(); }; +test.after(releaseShared); + +/** + * Closing a facade destroys its sockets, and the port is fixed, so the HTTP + * client can still hold a pooled connection to the server that just went away. + * That is a test-harness artifact — one facade outlives a whole broker — so a + * fresh facade is probed until a refusal proves the route is live again. + */ +const startFacade = async (): Promise => { + const facade = await startEngineBrokerMcpFacade(); + for (let attempt = 0; attempt < 20; attempt += 1) { + try { + const probe = await fetch(FACADE_URL, { method: "PUT" }); + await probe.body?.cancel(); + if (probe.status === 403) return facade; + } catch { /* a pooled socket onto the previous facade: try the next one */ } + } + throw new Error("facade did not answer after starting"); +}; test("MCP facade routes only valid active capabilities to the registered mount", async () => { let calls=0;const target=createServer((_request,response)=>{calls++;response.writeHead(200,{"content-type":"application/json"});response.end('{"ok":true}');});await new Promise((resolve)=>target.listen(0,"127.0.0.1",resolve));const address=target.address();if(address===null||typeof address==="string")throw new Error(); - const facade=await startEngineBrokerMcpFacade();const token=facade.register("agent","turn",`http://127.0.0.1:${address.port}/mcp`);const call=(value:string)=>fetch(FACADE_URL,{method:"POST",headers:{authorization:`Bearer ${value}`,"content-type":"application/json"},body:"{}"}); - try{assert.equal((await call("wrong-token-abcdefghijklmnopqrstuvwxyz0123456789")).status,403);assert.equal((await call(token)).status,200);assert.equal(calls,1);facade.revoke("turn");assert.equal((await call(token)).status,403);assert.equal(calls,1);}finally{await facade.close();await new Promise((resolve)=>target.close(()=>resolve()));} + const facade=await sharedFacade();const token=facade.register("agent","turn-capabilities",`http://127.0.0.1:${address.port}/mcp`);const call=(value:string)=>fetch(FACADE_URL,{method:"POST",headers:{authorization:`Bearer ${value}`,"content-type":"application/json"},body:"{}"}); + try{assert.equal((await call("wrong-token-abcdefghijklmnopqrstuvwxyz0123456789")).status,403);assert.equal((await call(token)).status,200);assert.equal(calls,1);facade.revoke("turn-capabilities");assert.equal((await call(token)).status,403);assert.equal(calls,1);}finally{facade.revoke("turn-capabilities");await new Promise((resolve)=>target.close(()=>resolve()));} }); /** @@ -28,12 +57,13 @@ test("MCP facade routes only valid active capabilities to the registered mount", * so every case below drives the transport end to end. */ type Rig = Readonly<{ - facade: Awaited>; - mount: HttpServer; - transport: StreamableHTTPServerTransport; + facade: Facade; + turnId: string; server: ReturnType; capability: string; observed: IncomingMessage[]; + /** Resolves when the mount's standalone GET stream is torn down. */ + getStreamClosed: Promise; close: () => Promise; }>; @@ -47,13 +77,20 @@ const echoTool = defineTool({ } }); -const startRig = async (): Promise => { +let turns = 0; + +const startRig = async (facade?: Facade): Promise => { + const host = facade ?? await sharedFacade(); + const turnId = `turn-${++turns}`; const server = createPiToolMcpServer([echoTool], {}); const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: () => randomUUID() }); await server.connect(transport); const observed: IncomingMessage[] = []; + let noteGetStreamClosed = (): void => undefined; + const getStreamClosed = new Promise((resolve) => { noteGetStreamClosed = resolve; }); const mount = createServer((request, response) => { observed.push(request); + if (request.method === "GET") response.on("close", () => noteGetStreamClosed()); const chunks: Buffer[] = []; request.on("data", (chunk: Buffer) => chunks.push(chunk)); request.on("end", () => { @@ -66,20 +103,23 @@ const startRig = async (): Promise => { await new Promise((resolve) => mount.listen(0, "127.0.0.1", resolve)); const address = mount.address(); if (address === null || typeof address === "string") throw new Error("mount address unavailable"); - const facade = await startEngineBrokerMcpFacade(); - const capability = facade.register("alpha", "turn-1", `http://127.0.0.1:${address.port}/mcp`); + const capability = host.register("alpha", turnId, `http://127.0.0.1:${address.port}/mcp`); return { - facade, mount, transport, server, capability, observed, + facade: host, turnId, server, capability, observed, getStreamClosed, close: async () => { - facade.revoke("turn-1"); - await facade.close(); - await new Promise((resolve) => mount.close(() => resolve())); - await transport.close().catch(() => undefined); - await server.close().catch(() => undefined); + host.revoke(turnId); + await closeMount(mount, transport, server); } }; }; +const closeMount = async (mount: HttpServer, transport: StreamableHTTPServerTransport, server: ReturnType): Promise => { + mount.closeAllConnections(); + await new Promise((resolve) => mount.close(() => resolve())); + await transport.close().catch(() => undefined); + await server.close().catch(() => undefined); +}; + const connectClient = async (capability: string): Promise<{ client: Client; transport: StreamableHTTPClientTransport }> => { const transport = new StreamableHTTPClientTransport(new URL(FACADE_URL), { requestInit: { headers: { authorization: `Bearer ${capability}` } } @@ -89,6 +129,15 @@ const connectClient = async (capability: string): Promise<{ client: Client; tran return { client, transport }; }; +const withDeadline = async (work: Promise, ms: number, reason: string): Promise => { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([work, new Promise((_resolve, reject) => { timer = setTimeout(() => reject(new Error(reason)), ms); })]); + } finally { + if (timer) clearTimeout(timer); + } +}; + test("a brokered worker completes the whole MCP handshake through the facade and calls a mounted tool", async () => { const rig = await startRig(); try { @@ -120,10 +169,7 @@ test("the facade carries the mount's server-initiated SSE stream, which only the // take; a POST-only facade answers it 403 and this never arrives. await new Promise((resolve) => setTimeout(resolve, 150)); rig.server.sendToolListChanged(); - await Promise.race([ - notified, - new Promise((_resolve, reject) => setTimeout(() => reject(new Error("no server notification reached the client")), 4_000)) - ]); + await withDeadline(notified, 4_000, "no server notification reached the client"); assert.ok(rig.observed.some((request) => request.method === "GET"), "the mount must have seen the GET stream"); assert.equal(typeof transport.sessionId, "string"); } finally { @@ -170,9 +216,12 @@ test("the facade forwards a closed header allowlist and never the worker's beare const forwarded = rig.observed.flatMap((request) => Object.keys(request.headers)); assert.equal(forwarded.includes("authorization"), false, "the turn capability must never reach the mount"); assert.equal(forwarded.includes("cookie"), false); - const allowed = new Set(["host", "connection", "content-length", "transfer-encoding", "accept-encoding", "accept-language", "user-agent", "content-type", "accept", "mcp-session-id", "mcp-protocol-version", "last-event-id", - // undici's own outbound headers, set by the facade's fetch rather than forwarded from the worker. - "sec-fetch-mode"]); + const allowed = new Set([ + "host", "connection", "content-length", "transfer-encoding", "accept-encoding", "accept-language", "user-agent", + "content-type", "accept", "mcp-session-id", "mcp-protocol-version", "last-event-id", + // undici's own outbound header, set by the facade's fetch rather than forwarded from the worker. + "sec-fetch-mode" + ]); const unexpected = [...new Set(forwarded)].filter((name) => !allowed.has(name)); assert.deepEqual(unexpected, [], `unexpected headers reached the mount: ${unexpected.join(",")}`); } finally { @@ -194,8 +243,8 @@ test("the facade withholds a mount response header that is not on the allowlist" await new Promise((resolve) => target.listen(0, "127.0.0.1", resolve)); const address = target.address(); if (address === null || typeof address === "string") throw new Error("target address unavailable"); - const facade = await startEngineBrokerMcpFacade(); - const capability = facade.register("alpha", "turn-1", `http://127.0.0.1:${address.port}/mcp`); + const facade = await sharedFacade(); + const capability = facade.register("alpha", "turn-response-headers", `http://127.0.0.1:${address.port}/mcp`); try { const answered = await fetch(FACADE_URL, { method: "POST", @@ -210,15 +259,14 @@ test("the facade withholds a mount response header that is not on the allowlist" assert.equal(answered.headers.get("x-mount-internal"), null); await answered.body?.cancel(); } finally { - facade.revoke("turn-1"); - await facade.close(); + facade.revoke("turn-response-headers"); await new Promise((resolve) => target.close(() => resolve())); } }); test("the facade still refuses every route and method outside the MCP surface", async () => { - const facade = await startEngineBrokerMcpFacade(); - const capability = facade.register("alpha", "turn-1", "http://127.0.0.1:1/mcp"); + const facade = await sharedFacade(); + const capability = facade.register("alpha", "turn-refusals", "http://127.0.0.1:1/mcp"); const call = (method: string, path: string) => fetch(`http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}${path}`, { method, headers: { authorization: `Bearer ${capability}` } }); @@ -229,7 +277,37 @@ test("the facade still refuses every route and method outside the MCP surface", assert.equal((await call("PATCH", "/mcp")).status, 403); assert.equal((await fetch(`http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}/mcp`, { method: "GET" })).status, 403); } finally { - facade.revoke("turn-1"); - await facade.close(); + facade.revoke("turn-refusals"); } }); + +/** + * Last, because both cases take the fixed port for themselves: an open GET + * tunnel must not survive its own capability, and must not stall shutdown + * either — a server-to-client stream stays open for the whole session, so + * before it existed nothing could hold the listener open. + */ +test("revoking a turn tears down its open server-to-client stream, and closing never stalls on one", async () => { + await releaseShared(); + const facade = await startFacade(); + const rig = await startRig(facade); + const { client } = await connectClient(rig.capability); + await new Promise((resolve) => setTimeout(resolve, 150)); + assert.ok(rig.observed.some((request) => request.method === "GET"), "the GET stream must be open before revoking"); + + // The client's stream survives its own capability unless the facade ends the + // tunnel: the mount's GET response is the side that has to close. + facade.revoke(rig.turnId); + await withDeadline(rig.getStreamClosed, 4_000, "a revoked capability left its SSE tunnel open"); + + // A second turn's tunnel, deliberately left open, is what shutdown must not wait on. + const second = await startRig(facade); + const held = await connectClient(second.capability); + await new Promise((resolve) => setTimeout(resolve, 150)); + await withDeadline(facade.close(), 3_000, "closing the facade stalled on an open SSE tunnel"); + + await held.client.close().catch(() => undefined); + await client.close().catch(() => undefined); + await second.close().catch(() => undefined); + await rig.close().catch(() => undefined); +}); From 73cc6b6e3b8a5509e71d6fe17f2ac00e4133f63a Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 01:00:33 +0200 Subject: [PATCH 34/69] fix: name the underlying fault beside the proxy's broker_unavailable log line --- src/runtime/AGENTS.md | 14 ++++++++- src/runtime/grokBrokerProxy.test.ts | 25 ++++++++++++++++ src/runtime/grokBrokerProxy.ts | 45 ++++++++++++++++++++++++++--- 3 files changed, 79 insertions(+), 5 deletions(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 6276db1..fb1e433 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -171,7 +171,19 @@ any capability lookup, isolation guard, credential read or upstream call, and Grok falls back to the truncated prompt as the title. That refusal and a bare unauthenticated `GET /` probe are the two requests a healthy turn always makes and the proxy never forwards; neither prints a `refused:` line, because for as -long as they did, every healthy turn read as broken. The sink keeps its 503 +long as they did, every healthy turn read as broken. Every *other* refused +request does name itself on the broker's stderr, and a fault that is not a +`GrokBrokerProxyRefusal` names its own class and message beside +`broker_unavailable` — `[grok-proxy] refused: broker_unavailable (TypeError: +…)` — because the bare word carries no diagnostic content and is answered 503, +which Grok blind-retries: one live turn emitted it fifteen times over five +minutes, spent $0, and died with no account of why. That cause is the error's +class and message only (never a body, bearer, capability, session id or +header), redacted through `redactCredentialText` with that request's own +capabilities as exact secrets and the `CLI_ENGINE_MAX_DIAGNOSTIC_BYTES` bound, +flattened to one line, exactly as the failed CLI child and the launcher's +worker diagnostic are. It is a log line only: the 503 is unchanged, because a +genuinely transient fault is still transient. The sink keeps its 503 shape because every live capture was taken with it: forcing 400 and 503 there were both observed to end the turn `exit=0, result: success`, so a hard 4xx on that request does *not* end Grok's session. And effort is only sent when the diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index 05caadf..8ca502d 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -112,3 +112,28 @@ test("the two requests every healthy turn makes are not named as refusals", asyn assert.deepEqual(lines, ["[grok-proxy] refused: unknown_capability\n"], "a genuine policy miss is still named with its reason code"); } finally { process.stderr.write = original; await proxy.close(); } }); + +test("a non-refusal fault names its own class and message on one bounded line, credentials withheld", async () => { + const lines: string[] = []; const original = process.stderr.write.bind(process.stderr); + process.stderr.write = ((chunk: string | Uint8Array) => { lines.push(String(chunk)); return true; }) as typeof process.stderr.write; + const provider = "provider-vqmxdfhlzptgnbwc"; let capability = ""; + // The fault's own words carry both credentials verbatim — the worker's turn + // capability and the broker's provider bearer, neither in a shape any generic + // pattern recognises — plus a newline, a control character, and far more text + // than the bound admits. + const proxy = await startGrokBrokerProxy( + { accessToken: async () => provider, markRejected: async () => undefined }, + async () => { throw new RangeError(`socket hang up forwarding ${capability}\nwith ${provider} ${"pad ".repeat(400)}`); }); + try { + capability = proxy.capabilities.issue("agent", "turn"); arm(proxy, async () => undefined); + assert.equal(await post(proxy.port, capability, leanBody()), 503, "a genuine transient fault keeps its 503"); + assert.equal(lines.length, 1, "one line per fault"); + const line = lines[0]!; + assert.match(line, /^\[grok-proxy\] refused: broker_unavailable \(RangeError: socket hang up forwarding /u, "the fault names its own class and message"); + assert.ok(!line.includes(capability), `the worker's own capability is withheld: ${line}`); + assert.ok(!line.includes(provider), `the broker's provider bearer is withheld: ${line}`); + assert.match(line, /\[REDACTED\]/u, "the withheld values are marked, not silently dropped"); + assert.match(line, /^[^\n]+\n$/u, "one line: newlines and control characters are flattened"); + assert.ok(Buffer.byteLength(line, "utf8") <= 900, `the line stays bounded: ${Buffer.byteLength(line, "utf8")} bytes`); + } finally { process.stderr.write = original; await proxy.close(); } +}); diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index c848d91..3955938 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -1,6 +1,8 @@ import { createHash } from "node:crypto"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import type { AddressInfo } from "node:net"; +import { redactCredentialText } from "../core/credentialRedaction.js"; +import { CLI_ENGINE_MAX_DIAGNOSTIC_BYTES } from "../pi/cliChildOutput.js"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; import { DEFAULT_GROK_BROKER_MODEL_POLICY, parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; import { authorizeGrokBrokerProxyRequest } from "./grokBrokerProxyRequest.js"; @@ -51,14 +53,19 @@ export class GrokBrokerProxyRefusal extends Error { async function serve(request: IncomingMessage, response: ServerResponse, authority: GrokBrokerCredentialAuthority, upstream: GrokBrokerUpstream, capabilities: EngineBrokerCapabilities,guards:MapPromise>,turns:Map,fallback:GrokBrokerModelPolicy,grants?:GrokInferenceGrants): Promise { let settle:((usage:ReturnType,toolCalls?:readonly string[])=>void)|undefined; let titleSink = false; + // Every credential this request holds, kept only for this request and only so + // that a fault's own words can be redacted against them exactly as the CLI + // child and launcher diagnostics are. Nothing reads them but {@link brokerFaultCause}. + const secrets: string[] = []; try { const body = await readBody(request); const headers = Object.fromEntries(Object.entries(request.headers).map(([key, value]) => [key, Array.isArray(value) ? value[0] : value])); titleSink = headers.authorization === `Bearer ${GROK_SESSION_TITLE_SINK_KEY}`; const match=headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u); + if(match)secrets.push(match[1]!); if(match&&match[1]!.startsWith(GROK_ENGINE_BROKER.inferenceGrants.tokenPrefix)){if(!grants)throw new GrokBrokerProxyRefusal("inference_grants_unavailable");return await serveGrokInferenceGrant({method:request.method??"",pathname:new URL(request.url??"/","http://127.0.0.1").pathname,headers,body,token:match[1]!},response,grants,authority,upstream);} const scope=match?capabilities.inspectToken(match[1]!):undefined;if(!scope)throw new GrokBrokerProxyRefusal("unknown_capability");const guard=guards.get(scope.turnId),turn=turns.get(scope.turnId);if(!guard||!turn)throw new GrokBrokerProxyRefusal("no_active_turn"); try { await guard(); } catch (error) { throw new GrokBrokerProxyRefusal("worker_isolation_unverified"); } - let token = await authority.accessToken(false);const rejectedDigest=createHash("sha256").update(token).digest("hex"); let prepared = authorizeRequestOrRefuse({ method: request.method ?? "", pathname: new URL(request.url ?? "/", "http://127.0.0.1").pathname, headers, body }, capabilities, token, turn.policy ?? fallback); token = ""; + let token = await authority.accessToken(false);secrets.push(token);const rejectedDigest=createHash("sha256").update(token).digest("hex"); let prepared = authorizeRequestOrRefuse({ method: request.method ?? "", pathname: new URL(request.url ?? "/", "http://127.0.0.1").pathname, headers, body }, capabilities, token, turn.policy ?? fallback); token = ""; // The spend gate runs after the body is proven a real lean worker request // (a refused session-title body never counts) and before any upstream call. const admission=turn.meter.admit(); @@ -66,7 +73,7 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori if("busy" in admission){response.writeHead(429,{"content-type":"application/json","cache-control":"no-store"});response.end('{"error":"turn request in flight"}');return;} settle=(usage,toolCalls)=>{turn.meter.settle(admission.index,usage,body.byteLength,toolCalls);settle=undefined;}; let result = await upstream(prepared,admission.signal); - if (result.status === 401) { token = authority.refreshAfterRejection?await authority.refreshAfterRejection(rejectedDigest):await authority.accessToken(true);const refreshedDigest=createHash("sha256").update(token).digest("hex"); prepared = { ...prepared, headers: { ...prepared.headers, authorization: `Bearer ${token}` } }; token = ""; result = await upstream(prepared,admission.signal);if(result.status===401)await authority.markRejected(refreshedDigest); } + if (result.status === 401) { token = authority.refreshAfterRejection?await authority.refreshAfterRejection(rejectedDigest):await authority.accessToken(true);secrets.push(token);const refreshedDigest=createHash("sha256").update(token).digest("hex"); prepared = { ...prepared, headers: { ...prepared.headers, authorization: `Bearer ${token}` } }; token = ""; result = await upstream(prepared,admission.signal);if(result.status===401)await authority.markRejected(refreshedDigest); } // Names only, bounded, and never a reason to fail the request: the response // is already buffered here for its usage block, so what the model tried to // call is in hand. A decoder fault records no attempt rather than a false @@ -79,7 +86,11 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori // or a token) so a failing turn is diagnosable without a stub harness — // except for the two requests every healthy turn makes anyway. const refusal = error instanceof GrokBrokerProxyRefusal ? error.reason : "broker_unavailable"; - if (!titleSink && !expectedWorkerProbe(request)) process.stderr.write(`[grok-proxy] refused: ${refusal}\n`); + // A named refusal is its own account; anything else used to reach the log as + // the bare word `broker_unavailable`, which names nothing — so it carries the + // fault's own class and message, and nothing else, beside it. + const named = error instanceof GrokBrokerProxyRefusal ? refusal : `${refusal} (${brokerFaultCause(error, secrets)})`; + if (!titleSink && !expectedWorkerProbe(request)) process.stderr.write(`[grok-proxy] refused: ${named}\n`); // Grok's own session-title call is refused by design, and keeps the transient // 503 shape it has always had. Forcing 400 and 503 on it were both observed // to end the turn `exit=0, result: success`, so the shape is kept because it @@ -97,9 +108,35 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori } response.writeHead(503, { "content-type": "application/json", "cache-control": "no-store" }); response.end('{"error":"broker unavailable"}'); - } + } finally { secrets.length = 0; } } +/** + * A non-refusal fault, named on one bounded line. + * + * `broker_unavailable` on its own carries no diagnostic content at all, and it + * is answered 503, which Grok blind-retries: one live turn emitted it fifteen + * times over five minutes, spent $0 — so no upstream call ever succeeded — and + * died with no account of why. The error's own class and message are the whole + * of what is logged: never a request body, bearer, capability, session id or + * header. It is redacted exactly as the failed CLI child and the launcher's + * worker diagnostic are — `redactCredentialText` with this request's own + * capabilities as exact secrets and the same `CLI_ENGINE_MAX_DIAGNOSTIC_BYTES` + * bound — and flattened to one line, because it travels on a log line. Naming + * a fault must never be able to fail the response that reports it, so a value + * that cannot even be described degrades to a marker. + */ +const brokerFaultCause = (error: unknown, secrets: readonly string[]): string => { + try { + const described = error instanceof Error + ? `${error.constructor?.name ?? error.name}: ${error.message}` + : `${typeof error}: ${String(error)}`; + const flattened = described.replace(/[-]+/gu, " ").replace(/\s+/gu, " ").trim(); + const named = redactCredentialText(flattened, secrets, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); + return named.length === 0 ? "unnamed" : named; + } catch { return "unnameable"; } +}; + /** * The unauthenticated connectivity probe Grok sends before its own requests: a * bare `GET /` with no Authorization header, which has no capability to look up From 16c12241f3cbf3bbfb1dc9071c165c935cb66a6d Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 01:07:58 +0200 Subject: [PATCH 35/69] fix: name one level of a broker fault's own cause so a failed provider fetch says which fault it was --- src/runtime/AGENTS.md | 8 ++++++-- src/runtime/grokBrokerProxy.test.ts | 13 +++++++++++++ src/runtime/grokBrokerProxy.ts | 18 +++++++++++++++--- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index fb1e433..c0ce68e 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -178,8 +178,12 @@ request does name itself on the broker's stderr, and a fault that is not a …)` — because the bare word carries no diagnostic content and is answered 503, which Grok blind-retries: one live turn emitted it fifteen times over five minutes, spent $0, and died with no account of why. That cause is the error's -class and message only (never a body, bearer, capability, session id or -header), redacted through `redactCredentialText` with that request's own +class and message, plus one level of its own `cause` — every failed provider +`fetch` is `TypeError: fetch failed` and names nothing without it, so the line +reads `broker_unavailable (TypeError: fetch failed <- Error: ENOTFOUND)`, an +errno cause with no message named by its `code`. Nothing else: never a body, +bearer, capability, session id or +header. It is redacted through `redactCredentialText` with that request's own capabilities as exact secrets and the `CLI_ENGINE_MAX_DIAGNOSTIC_BYTES` bound, flattened to one line, exactly as the failed CLI child and the launcher's worker diagnostic are. It is a log line only: the 503 is unchanged, because a diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index 8ca502d..be3b2f4 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -137,3 +137,16 @@ test("a non-refusal fault names its own class and message on one bounded line, c assert.ok(Buffer.byteLength(line, "utf8") <= 900, `the line stays bounded: ${Buffer.byteLength(line, "utf8")} bytes`); } finally { process.stderr.write = original; await proxy.close(); } }); + +test("a fault's own cause is named too, because `fetch failed` on its own names nothing", async () => { + const lines: string[] = []; const original = process.stderr.write.bind(process.stderr); + process.stderr.write = ((chunk: string | Uint8Array) => { lines.push(String(chunk)); return true; }) as typeof process.stderr.write; + // Exactly the shape undici throws when the provider is unreachable. + const fault = new TypeError("fetch failed"); (fault as { cause?: unknown }).cause = Object.assign(new Error(""), { code: "ENOTFOUND" }); + const proxy = await startGrokBrokerProxy({ accessToken: async () => "provider-token", markRejected: async () => undefined }, async () => { throw fault; }); + try { + const token = proxy.capabilities.issue("agent", "turn"); arm(proxy, async () => undefined); + assert.equal(await post(proxy.port, token, leanBody()), 503); + assert.deepEqual(lines, ["[grok-proxy] refused: broker_unavailable (TypeError: fetch failed <- Error: ENOTFOUND)\n"]); + } finally { process.stderr.write = original; await proxy.close(); } +}); diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index 3955938..a92b61c 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -125,18 +125,30 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori * bound — and flattened to one line, because it travels on a log line. Naming * a fault must never be able to fail the response that reports it, so a value * that cannot even be described degrades to a marker. + * + * One level of `cause` is named too, because the fault this exists for names + * nothing without it: every failed `fetch` to the provider is `TypeError: + * fetch failed`, and which fault it was — `ENOTFOUND`, `ECONNREFUSED`, a TLS + * refusal, an abort — is only in the cause. An errno error whose message is + * empty is named by its `code`. */ const brokerFaultCause = (error: unknown, secrets: readonly string[]): string => { try { - const described = error instanceof Error - ? `${error.constructor?.name ?? error.name}: ${error.message}` - : `${typeof error}: ${String(error)}`; + const described = `${describeFault(error)}${error instanceof Error && error.cause !== undefined && error.cause !== null ? ` <- ${describeFault(error.cause)}` : ""}`; const flattened = described.replace(/[-]+/gu, " ").replace(/\s+/gu, " ").trim(); const named = redactCredentialText(flattened, secrets, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); return named.length === 0 ? "unnamed" : named; } catch { return "unnameable"; } }; +/** One value's class and words: an error's own, or the type of whatever else was thrown. */ +const describeFault = (error: unknown): string => { + if (!(error instanceof Error)) return `${typeof error}: ${String(error)}`; + const code = (error as NodeJS.ErrnoException).code; + const words = error.message.length > 0 ? error.message : typeof code === "string" ? code : "(no message)"; + return `${error.constructor?.name ?? error.name}: ${words}`; +}; + /** * The unauthenticated connectivity probe Grok sends before its own requests: a * bare `GET /` with no Authorization header, which has no capability to look up From f3d4b2bc9b4bec41aacffc0e1758c1ae9eb6e8d8 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 01:09:40 +0200 Subject: [PATCH 36/69] fix: escape the flatten ranges and the test's control byte so no source carries a raw control byte --- src/runtime/grokBrokerProxy.test.ts | 2 +- src/runtime/grokBrokerProxy.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index be3b2f4..ee9270b 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -123,7 +123,7 @@ test("a non-refusal fault names its own class and message on one bounded line, c // than the bound admits. const proxy = await startGrokBrokerProxy( { accessToken: async () => provider, markRejected: async () => undefined }, - async () => { throw new RangeError(`socket hang up forwarding ${capability}\nwith ${provider} ${"pad ".repeat(400)}`); }); + async () => { throw new RangeError(`socket hang up forwarding ${capability}\nwith ${provider}\u0007 ${"pad ".repeat(400)}`); }); try { capability = proxy.capabilities.issue("agent", "turn"); arm(proxy, async () => undefined); assert.equal(await post(proxy.port, capability, leanBody()), 503, "a genuine transient fault keeps its 503"); diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index a92b61c..47be53c 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -135,7 +135,7 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori const brokerFaultCause = (error: unknown, secrets: readonly string[]): string => { try { const described = `${describeFault(error)}${error instanceof Error && error.cause !== undefined && error.cause !== null ? ` <- ${describeFault(error.cause)}` : ""}`; - const flattened = described.replace(/[-]+/gu, " ").replace(/\s+/gu, " ").trim(); + const flattened = described.replace(/[\u0000-\u001f\u007f]+/gu, " ").replace(/\s+/gu, " ").trim(); const named = redactCredentialText(flattened, secrets, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); return named.length === 0 ? "unnamed" : named; } catch { return "unnameable"; } From 6a4fa63da47fdcf5478a0b6468f2644051edd99a Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 01:15:52 +0200 Subject: [PATCH 37/69] fix: wake the MCP facade's backpressure await on a hang-up or abort so a stalled tunnel cannot park a turn --- src/runtime/engineBrokerMcpFacade.test.ts | 50 ++++++++++++++++++++++- src/runtime/engineBrokerMcpFacade.ts | 35 +++++++++++++++- 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/src/runtime/engineBrokerMcpFacade.test.ts b/src/runtime/engineBrokerMcpFacade.test.ts index 95d0b65..057e798 100644 --- a/src/runtime/engineBrokerMcpFacade.test.ts +++ b/src/runtime/engineBrokerMcpFacade.test.ts @@ -1,5 +1,6 @@ import assert from "node:assert/strict"; import { createServer, type Server as HttpServer, type IncomingMessage } from "node:http"; +import { connect, type Socket } from "node:net"; import { randomUUID } from "node:crypto"; import test from "node:test"; @@ -11,7 +12,7 @@ import { Type } from "@earendil-works/pi-ai"; import { defineTool } from "@earendil-works/pi-coding-agent"; import { createPiToolMcpServer } from "../mcp/toolServer.js"; -import { ENGINE_BROKER_MCP_FACADE_PORT, startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; +import { awaitMcpTunnelDrain, ENGINE_BROKER_MCP_FACADE_PORT, startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; const FACADE_URL = `http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}/mcp`; type Facade = Awaited>; @@ -311,3 +312,50 @@ test("revoking a turn tears down its open server-to-client stream, and closing n await second.close().catch(() => undefined); await rig.close().catch(() => undefined); }); + +/** + * The relay parks on this await whenever a tunnel is backpressured, and a + * parked await is invisible from outside: no status, no refusal, no line. So + * the assertion is that it settles at all — on a client that hung up + * mid-write, on the turn's abort, and on a genuine drain — with a deadline + * standing in for the hang. + */ +test("a backpressured MCP tunnel always settles: on a hang-up, on the turn's abort, and on a real drain", async () => { + const parked = new Map>(); + // One controller per phase: the turn whose abort is under test must not be + // the turn that is still relaying. + const controllers = new Map(); + const server = createServer((request, response) => { + const phase = request.url ?? ""; + const controller = new AbortController(); controllers.set(phase, controller); + response.writeHead(200, { "content-type": "text/event-stream" }); + // A paused client cannot absorb this, so `write` reports backpressure and + // the relay would park exactly here. + response.write("data: open\n\n"); + if (phase === "/drain") assert.equal(response.write(Buffer.alloc(16 * 1024 * 1024, 0x61)), false, "a paused client must backpressure the tunnel"); + parked.set(phase, awaitMcpTunnelDrain(response, controller.signal).then(() => "drained", (error: Error) => error.message)); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); if (address === null || typeof address === "string") throw new Error(); + const open = (phase: string): Promise => new Promise((resolve) => { + const socket = connect(address.port, "127.0.0.1", () => socket.write(`GET ${phase} HTTP/1.1\r\nhost: facade\r\n\r\n`)); + socket.once("data", () => { socket.pause(); resolve(socket); }); + }); + const settled = (phase: string): Promise => withDeadline(parked.get(phase)!, 3_000, `the ${phase} await never settled: the relay is parked`); + const sockets: Socket[] = []; + try { + sockets.push(await open("/hangup")); + sockets[0]!.destroy(); + assert.equal(await settled("/hangup"), "MCP tunnel closed", "a client that hung up mid-write wakes the await"); + sockets.push(await open("/abort")); + controllers.get("/abort")!.abort(); + assert.equal(await settled("/abort"), "MCP tunnel aborted", "the turn's own abort wakes the await"); + const draining = await open("/drain"); + sockets.push(draining); + draining.resume(); + assert.equal(await settled("/drain"), "drained", "a client that resumes reading resolves the await"); + } finally { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve) => server.close(() => resolve())); + } +}); diff --git a/src/runtime/engineBrokerMcpFacade.ts b/src/runtime/engineBrokerMcpFacade.ts index ee1defd..03eee1f 100644 --- a/src/runtime/engineBrokerMcpFacade.ts +++ b/src/runtime/engineBrokerMcpFacade.ts @@ -175,7 +175,8 @@ async function forward( const stream = Readable.fromWeb(upstream.body as Parameters[0]); try { for await (const chunk of stream) { - if (!response.write(chunk as Uint8Array)) await new Promise((resolve) => response.once("drain", resolve)); + if (response.destroyed || response.writableEnded) throw new Error("MCP tunnel closed"); + if (!response.write(chunk as Uint8Array)) await awaitMcpTunnelDrain(response, signal); } response.end(); } catch { @@ -187,6 +188,38 @@ async function forward( } } +/** + * Waits for a backpressured tunnel to drain, or for the tunnel to end — + * whichever happens first, but always one of them. + * + * A bare `once("drain")` never settles for a client that hung up mid-write: + * `drain` cannot fire on a socket nobody is reading, and neither the + * response's own `close` nor the turn's abort woke that await. The GET SSE + * tunnel stays open for a whole session, so the handler — and the upstream + * call it was relaying — leaked for the life of the broker process, with no + * refusal, no status and no line anywhere to read. Every outcome settles this + * now, and every outcome but an actual drain *rejects*, so the caller tears + * the tunnel down instead of writing into a socket that is gone. + * + * Exported for its own test: a hang is only observable from inside. + */ +export function awaitMcpTunnelDrain(response: ServerResponse, signal: AbortSignal): Promise { + if (response.destroyed || response.writableEnded) return Promise.reject(new Error("MCP tunnel closed")); + if (signal.aborted) return Promise.reject(new Error("MCP tunnel aborted")); + return new Promise((resolve, reject) => { + const settle = (finish: () => void) => (): void => { + response.off("drain", onDrain); response.off("close", onClosed); response.off("error", onClosed); + signal.removeEventListener("abort", onAborted); + finish(); + }; + const onDrain = settle(resolve); + const onClosed = settle(() => reject(new Error("MCP tunnel closed"))); + const onAborted = settle(() => reject(new Error("MCP tunnel aborted"))); + response.once("drain", onDrain); response.once("close", onClosed); response.once("error", onClosed); + signal.addEventListener("abort", onAborted, { once: true }); + }); +} + async function bounded(request: AsyncIterable): Promise { const chunks: Buffer[] = []; let bytes = 0; From 64a30bb5d19fdc9499c008b8b06e9fb563a98032 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 01:17:52 +0200 Subject: [PATCH 38/69] fix: refuse a fenced credential realm as a named non-retryable auth_stale on the turn path --- src/runtime/AGENTS.md | 22 +++++++++++-- src/runtime/engineBrokerProtocol.ts | 9 +++++- src/runtime/grokBrokerProxy.test.ts | 50 +++++++++++++++++++++++++++++ src/runtime/grokBrokerProxy.ts | 17 ++++++++-- 4 files changed, 92 insertions(+), 6 deletions(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index c0ce68e..947682e 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -187,7 +187,18 @@ header. It is redacted through `redactCredentialText` with that request's own capabilities as exact secrets and the `CLI_ENGINE_MAX_DIAGNOSTIC_BYTES` bound, flattened to one line, exactly as the failed CLI child and the launcher's worker diagnostic are. It is a log line only: the 503 is unchanged, because a -genuinely transient fault is still transient. The sink keeps its 503 +genuinely transient fault is still transient. + +One fault is *not* transient and no longer wears that shape: a fenced +credential realm. `isStale()` is checked on the turn path before the +credential read, and the request that discovers the fence (the authority's own +generic error) is promoted to the same refusal, so a stale realm is a named +400 `auth_stale` instead of one 503 plus fourteen blind retries — the training +login expired at 22:28Z and the 22:48Z run spent five minutes and $0 learning +nothing. `ENGINE_BROKER_AUTH_STALE` (`engineBrokerProtocol.ts`) is the single +name behind the turn failure code, this refusal reason and the grant path's +401 `GROK_INFERENCE_AUTH_STALE_BODY`; the grant path keeps its own 401 shape, +and the title sink keeps its 503 on a fenced realm like everywhere else. The sink keeps its 503 shape because every live capture was taken with it: forcing 400 and 503 there were both observed to end the turn `exit=0, result: success`, so a hard 4xx on that request does *not* end Grok's session. And effort is only sent when the @@ -212,7 +223,14 @@ never logged or ledgered. The facade also carries the three methods the transport uses — POST, the standalone `GET` SSE stream that is the only route a server notification or progress frame can take, and the `DELETE` that ends a session — and streams each body rather than buffering it, because a GET tunnel -stays open for the whole session. Never widen it into a transparent proxy: the +stays open for the whole session. Streaming means backpressure, and a +backpressured tunnel must never park: `awaitMcpTunnelDrain` races the client's +`drain` against its `close`/`error` and the turn's abort, because a bare +`once("drain")` cannot fire for a client that hung up mid-write and left the +handler — and the upstream call it was relaying — awaiting for the life of the +process, with no status, no refusal and no line anywhere to read. Every +outcome but a real drain rejects, so the relay tears the tunnel down instead +of writing into a socket that is gone. Never widen it into a transparent proxy: the whole point of the boundary is that the allowlist is closed. Worker `GROK_HOME` layout the deployment must provision (attested before every diff --git a/src/runtime/engineBrokerProtocol.ts b/src/runtime/engineBrokerProtocol.ts index eedf428..224ca44 100644 --- a/src/runtime/engineBrokerProtocol.ts +++ b/src/runtime/engineBrokerProtocol.ts @@ -36,7 +36,14 @@ export type EngineBrokerResponse = | (Readonly<{ version: typeof VERSION; kind: "completed"; requestId: string; turnId: string; text: string; workerPid: number; workerUid: number; workerStartTime: string }> & EngineBrokerTurnAccounting) | (Readonly<{ version: typeof VERSION; kind: "failed"; requestId: string; turnId: string; code: EngineBrokerFailureCode; diagnostic?: EngineBrokerFailureDiagnostic }> & EngineBrokerTurnAccounting) | EngineBrokerInferenceResponse; -export const ENGINE_BROKER_FAILURE_CODES = ["auth_stale", "cancelled", "engine_failed", "invalid_request", "limit_exceeded", "turn_conflict", "unavailable"] as const; +/** + * The one name for a fenced credential realm. The turn's failure code, the + * proxy's refusal reason on a worker request, and the grant path's 401 body + * (`GROK_INFERENCE_AUTH_STALE_BODY`) all say this same word, so an operator + * greps one string across every surface instead of three spellings of it. + */ +export const ENGINE_BROKER_AUTH_STALE = "auth_stale" as const; +export const ENGINE_BROKER_FAILURE_CODES = [ENGINE_BROKER_AUTH_STALE, "cancelled", "engine_failed", "invalid_request", "limit_exceeded", "turn_conflict", "unavailable"] as const; export type EngineBrokerFailureCode = (typeof ENGINE_BROKER_FAILURE_CODES)[number]; export type EngineBrokerTerminalResponse = Extract; type V1Completed = Readonly<{ version: typeof V1; kind: "completed"; requestId: string; turnId: string; text: string; workerPid: number; workerUid: number; workerStartTime: string }>; diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index ee9270b..6c9cfd9 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -4,6 +4,8 @@ import test from "node:test"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; import { GROK_SESSION_TITLE_SINK_KEY } from "./grokBrokerWorkerConfig.js"; import { GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; +import { ENGINE_BROKER_AUTH_STALE } from "./engineBrokerProtocol.js"; +import { GROK_INFERENCE_AUTH_STALE_BODY } from "./grokInferenceProxy.js"; type Proxy = Awaited>; const arm = (proxy: Proxy, guard: () => Promise, meter = new GrokBrokerTurnMeter({ maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 })): GrokBrokerTurnMeter => { @@ -150,3 +152,51 @@ test("a fault's own cause is named too, because `fetch failed` on its own names assert.deepEqual(lines, ["[grok-proxy] refused: broker_unavailable (TypeError: fetch failed <- Error: ENOTFOUND)\n"]); } finally { process.stderr.write = original; await proxy.close(); } }); + +/** + * A fenced realm is the one fault this proxy answered worst: the credential is + * gone until an operator logs in again, and 503 made Grok retry it fifteen + * times over five minutes for nothing. It is a named, non-retryable refusal + * now — and it wears the same name as the turn failure code and the grant + * path's 401 body, so one word finds it on every surface. + */ +test("a fenced credential realm is a named 400 auth_stale, before any credential read or upstream call", async () => { + const lines: string[] = []; const original = process.stderr.write.bind(process.stderr); + process.stderr.write = ((chunk: string | Uint8Array) => { lines.push(String(chunk)); return true; }) as typeof process.stderr.write; + let reads = 0, calls = 0, stale = true; + const proxy = await startGrokBrokerProxy( + { accessToken: async () => { reads += 1; return "provider-token"; }, markRejected: async () => undefined, isStale: () => stale }, + async () => { calls += 1; return { status: 200, headers: { "content-type": "application/json" }, body: Buffer.from("{}") }; }); + try { + const token = proxy.capabilities.issue("agent", "turn"); arm(proxy, async () => undefined); + assert.equal(await post(proxy.port, token, leanBody()), 400, "a stale realm is not transient, so it must not be retryable"); + assert.deepEqual({ reads, calls }, { reads: 0, calls: 0 }, "no credential is read and nothing is forwarded for a fenced realm"); + assert.deepEqual(lines, [`[grok-proxy] refused: ${ENGINE_BROKER_AUTH_STALE}\n`]); + // The title sink keeps the 503 it has always had, fenced realm or not. + assert.equal(await post(proxy.port, GROK_SESSION_TITLE_SINK_KEY, leanBody()), 503); + // And the same capability serves the real request once the realm is healthy. + stale = false; lines.length = 0; + assert.equal(await post(proxy.port, token, leanBody()), 200); + assert.deepEqual({ reads, calls, lines }, { reads: 1, calls: 1, lines: [] }); + } finally { process.stderr.write = original; await proxy.close(); } +}); + +test("the request that discovers the fence is named auth_stale too, not one transient fault", async () => { + const lines: string[] = []; const original = process.stderr.write.bind(process.stderr); + process.stderr.write = ((chunk: string | Uint8Array) => { lines.push(String(chunk)); return true; }) as typeof process.stderr.write; + // The live shape: `accessToken` fences the realm and throws the authority's + // own generic error, which on its own reads as a transient fault. + let stale = false; + const proxy = await startGrokBrokerProxy( + { accessToken: async () => { stale = true; throw new Error("Grok broker credential authority unavailable"); }, markRejected: async () => undefined, isStale: () => stale }, + async () => ({ status: 200, headers: { "content-type": "application/json" }, body: Buffer.from("{}") })); + try { + const token = proxy.capabilities.issue("agent", "turn"); arm(proxy, async () => undefined); + assert.equal(await post(proxy.port, token, leanBody()), 400); + assert.deepEqual(lines, [`[grok-proxy] refused: ${ENGINE_BROKER_AUTH_STALE}\n`]); + } finally { process.stderr.write = original; await proxy.close(); } +}); + +test("the turn path and the grant path name a fenced realm the same way", () => { + assert.ok(GROK_INFERENCE_AUTH_STALE_BODY.includes(ENGINE_BROKER_AUTH_STALE), "one name, not three spellings"); +}); diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index 47be53c..55f335a 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -4,6 +4,7 @@ import type { AddressInfo } from "node:net"; import { redactCredentialText } from "../core/credentialRedaction.js"; import { CLI_ENGINE_MAX_DIAGNOSTIC_BYTES } from "../pi/cliChildOutput.js"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; +import { ENGINE_BROKER_AUTH_STALE } from "./engineBrokerProtocol.js"; import { DEFAULT_GROK_BROKER_MODEL_POLICY, parseGrokBrokerModelPolicy, type GrokBrokerModelPolicy } from "./grokBrokerModelPolicy.js"; import { authorizeGrokBrokerProxyRequest } from "./grokBrokerProxyRequest.js"; import { GROK_SESSION_TITLE_SINK_KEY } from "./grokBrokerWorkerConfig.js"; @@ -65,6 +66,11 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori if(match&&match[1]!.startsWith(GROK_ENGINE_BROKER.inferenceGrants.tokenPrefix)){if(!grants)throw new GrokBrokerProxyRefusal("inference_grants_unavailable");return await serveGrokInferenceGrant({method:request.method??"",pathname:new URL(request.url??"/","http://127.0.0.1").pathname,headers,body,token:match[1]!},response,grants,authority,upstream);} const scope=match?capabilities.inspectToken(match[1]!):undefined;if(!scope)throw new GrokBrokerProxyRefusal("unknown_capability");const guard=guards.get(scope.turnId),turn=turns.get(scope.turnId);if(!guard||!turn)throw new GrokBrokerProxyRefusal("no_active_turn"); try { await guard(); } catch (error) { throw new GrokBrokerProxyRefusal("worker_isolation_unverified"); } + // A fenced realm is not a transient fault: the credential is gone until an + // operator re-logs in, and 503 made Grok blind-retry it (observed: fifteen + // retries over five minutes, $0 spent, nothing learned). Named, 400, and + // checked before the credential read, so the miss costs one round trip. + if (authority.isStale?.() === true) throw new GrokBrokerProxyRefusal(ENGINE_BROKER_AUTH_STALE); let token = await authority.accessToken(false);secrets.push(token);const rejectedDigest=createHash("sha256").update(token).digest("hex"); let prepared = authorizeRequestOrRefuse({ method: request.method ?? "", pathname: new URL(request.url ?? "/", "http://127.0.0.1").pathname, headers, body }, capabilities, token, turn.policy ?? fallback); token = ""; // The spend gate runs after the body is proven a real lean worker request // (a refused session-title body never counts) and before any upstream call. @@ -85,11 +91,16 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori // Name the refusal on the broker's own stderr (reason code only, never a body // or a token) so a failing turn is diagnosable without a stub harness — // except for the two requests every healthy turn makes anyway. - const refusal = error instanceof GrokBrokerProxyRefusal ? error.reason : "broker_unavailable"; + // The request that *discovers* the fence throws an ordinary error from the + // credential authority, so it is promoted to the same named refusal: one + // stale realm must not read as one transient fault plus fourteen retries. + const fenced = !(error instanceof GrokBrokerProxyRefusal) && authority.isStale?.() === true; + const refused = error instanceof GrokBrokerProxyRefusal || fenced; + const refusal = error instanceof GrokBrokerProxyRefusal ? error.reason : fenced ? ENGINE_BROKER_AUTH_STALE : "broker_unavailable"; // A named refusal is its own account; anything else used to reach the log as // the bare word `broker_unavailable`, which names nothing — so it carries the // fault's own class and message, and nothing else, beside it. - const named = error instanceof GrokBrokerProxyRefusal ? refusal : `${refusal} (${brokerFaultCause(error, secrets)})`; + const named = refused ? refusal : `${refusal} (${brokerFaultCause(error, secrets)})`; if (!titleSink && !expectedWorkerProbe(request)) process.stderr.write(`[grok-proxy] refused: ${named}\n`); // Grok's own session-title call is refused by design, and keeps the transient // 503 shape it has always had. Forcing 400 and 503 on it were both observed @@ -101,7 +112,7 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori response.end('{"error":"broker unavailable"}'); return; } - if (error instanceof GrokBrokerProxyRefusal) { + if (refused) { response.writeHead(400, { "content-type": "application/json", "cache-control": "no-store" }); response.end(JSON.stringify({ error: "broker refused this request", reason: refusal })); return; From 8ac45f49bf0025a2cec3ac8ad5465e0d595e5dc4 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 01:18:02 +0200 Subject: [PATCH 39/69] docs: keep the session-title sink's own paragraph intact in the runtime guide --- src/runtime/AGENTS.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 947682e..baa9983 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -198,7 +198,9 @@ login expired at 22:28Z and the 22:48Z run spent five minutes and $0 learning nothing. `ENGINE_BROKER_AUTH_STALE` (`engineBrokerProtocol.ts`) is the single name behind the turn failure code, this refusal reason and the grant path's 401 `GROK_INFERENCE_AUTH_STALE_BODY`; the grant path keeps its own 401 shape, -and the title sink keeps its 503 on a fenced realm like everywhere else. The sink keeps its 503 +and the title sink keeps its 503 on a fenced realm like everywhere else. + +The sink keeps that 503 shape because every live capture was taken with it: forcing 400 and 503 there were both observed to end the turn `exit=0, result: success`, so a hard 4xx on that request does *not* end Grok's session. And effort is only sent when the From a509277d215c786650a5e1345f811255d9107791 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 01:20:17 +0200 Subject: [PATCH 40/69] fix: let a usage decoder fault fall through to the estimate instead of failing a paid request --- src/runtime/AGENTS.md | 8 +++++++- src/runtime/grokBrokerProxy.test.ts | 29 ++++++++++++++++++++++++++++- src/runtime/grokBrokerProxy.ts | 17 ++++++++++++++++- 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index baa9983..147dc5c 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -424,7 +424,13 @@ decoded records *no field at all*, because a fabricated empty list is byte-identical to a measured one. On the stream row path the names are attached only when the proxy's timings and the worker's stream requests are aligned request-for-request, since an unaligned index would credit one request's attempt -to another. It is an additive field inside the unchanged +to another. The *usage* decode beside it is wrapped the same way, and for a +sharper reason: the upstream call has already succeeded, so a decoder fault +that failed the request would throw away a response the broker paid for and +have Grok buy it again. A fault there falls through to the documented estimate +(`ceil(bodyBytes/2) + 4096`, `usage_source: "estimated"`, counted in +`estimated_requests`) — never to zero and never to absence, because the +ceiling must still count what was spent. It is an additive field inside the unchanged `noopolis.daimon.turn-requests.v1` row and deliberately not a version bump: Spawnfile's reader pins `v` and ignores fields it does not know, and Paideia only relocates this stream's path. The whole path is advisory — the parse is diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index 6c9cfd9..526bd78 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -3,7 +3,7 @@ import { request as httpRequest } from "node:http"; import test from "node:test"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; import { GROK_SESSION_TITLE_SINK_KEY } from "./grokBrokerWorkerConfig.js"; -import { GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; +import { estimateGrokRequestUsage, GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; import { ENGINE_BROKER_AUTH_STALE } from "./engineBrokerProtocol.js"; import { GROK_INFERENCE_AUTH_STALE_BODY } from "./grokInferenceProxy.js"; @@ -200,3 +200,30 @@ test("the request that discovers the fence is named auth_stale too, not one tran test("the turn path and the grant path name a fenced realm the same way", () => { assert.ok(GROK_INFERENCE_AUTH_STALE_BODY.includes(ENGINE_BROKER_AUTH_STALE), "one name, not three spellings"); }); + +/** + * A response the usage decoder cannot read is the worst case to get wrong: the + * upstream call already succeeded, so the money is spent whatever happens + * next. It must reach the worker anyway, and it must be charged — a fabricated + * zero is byte-identical to a measured one, so the documented estimate is the + * only honest landing place. (The fault is synthetic: the real one is a + * response body past the maximum string length, which is not a thing to + * allocate in a test.) + */ +test("a response whose usage cannot be decoded is still delivered, and charged the documented estimate", async () => { + const contentType = { toString: () => "application/json" } as unknown as string; + const proxy = await startGrokBrokerProxy( + { accessToken: async () => "provider-token", markRejected: async () => undefined }, + async () => ({ status: 200, headers: { "content-type": contentType }, body: Buffer.from('{"usage":{"prompt_tokens":11,"completion_tokens":3}}') })); + try { + const token = proxy.capabilities.issue("agent", "turn"); + const meter = arm(proxy, async () => undefined); + const payload = leanBody(); + assert.equal(await post(proxy.port, token, payload), 200, "a paid response must not be thrown away over its own instrumentation"); + const snapshot = meter.snapshot(); + assert.equal(snapshot.requests, 1); + assert.equal(snapshot.estimatedRequests, 1, "the row says the charge was estimated, not measured"); + assert.deepEqual(snapshot.usage, estimateGrokRequestUsage(Buffer.byteLength(payload)), "charged the documented conservative estimate, never zero and never nothing"); + assert.equal(snapshot.timings[0]?.toolCalls, undefined, "an undecodable response records no tool-call attempt either"); + } finally { await proxy.close(); } +}); diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index 55f335a..9857cff 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -84,7 +84,7 @@ async function serve(request: IncomingMessage, response: ServerResponse, authori // is already buffered here for its usage block, so what the model tried to // call is in hand. A decoder fault records no attempt rather than a false // empty one, and never disturbs the turn. - settle?.(parseGrokUpstreamUsage(result.body,result.headers["content-type"]),toolCallsOrNothing(result.body,result.headers["content-type"])); + settle?.(usageOrEstimate(result.body,result.headers["content-type"]),toolCallsOrNothing(result.body,result.headers["content-type"])); response.writeHead(result.status, { "content-type": result.headers["content-type"] ?? "application/json", "cache-control": "no-store" }); response.end(result.body); } catch (error) { settle?.(undefined); @@ -176,6 +176,21 @@ const expectedWorkerProbe = (request: IncomingMessage): boolean => request.headers.authorization === undefined && (request.method ?? "") === "GET" && new URL(request.url ?? "/", "http://127.0.0.1").pathname === "/"; +/** + * Upstream-reported usage, or the documented conservative estimate. + * + * The decoder faulting must not fail the request that already cost real + * money: the upstream call succeeded, the worker is owed its answer, and a + * 503 here would throw away a paid response and have Grok buy it again. The + * `undefined` this returns is not "no usage" — `GrokBrokerTurnMeter.settle` + * charges it `ceil(bodyBytes/2) + 4096` and marks the row + * `usage_source: "estimated"`, so the token ceiling still counts it and no + * fabricated zero ever reaches a ledger. + */ +const usageOrEstimate = (body: Uint8Array, contentType: string | undefined): ReturnType => { + try { return parseGrokUpstreamUsage(body, contentType); } catch { return undefined; } +}; + /** Instrumentation must never fail a turn: a throwing decoder records nothing, exactly as an undecodable response does. */ const toolCallsOrNothing = (body: Uint8Array, contentType: string | undefined): readonly string[] | undefined => { try { return parseGrokResponseToolNames(body, contentType); } catch { return undefined; } From 20ae57befb33baa5958dacb0aabe28a5e7b19548 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 01:39:02 +0200 Subject: [PATCH 41/69] fix: decode a failed worker's last words as text and keep both ends of the diagnostic window --- src/pi/cliChildOutput.ts | 82 ++++++++++++++++---- src/pi/cliSessionOutput.test.ts | 47 ++++++++++- src/runtime/AGENTS.md | 18 ++++- src/runtime/engineBrokerNativeClient.test.ts | 60 +++++++++++++- src/runtime/engineBrokerNativeClient.ts | 38 +++++++-- src/runtime/native/AGENTS.md | 16 ++++ src/runtime/toolResultSpill.ts | 6 +- 7 files changed, 237 insertions(+), 30 deletions(-) diff --git a/src/pi/cliChildOutput.ts b/src/pi/cliChildOutput.ts index 19b5ca2..78a87d4 100644 --- a/src/pi/cliChildOutput.ts +++ b/src/pi/cliChildOutput.ts @@ -1,13 +1,14 @@ import type { ChildProcess } from "node:child_process"; import { redactCredentialText } from "../core/credentialRedaction.js"; +import { headUtf8, tailUtf8 } from "../runtime/toolResultSpill.js"; import type { TurnUsageFailureReason } from "../runtime/turnUsageLedger.js"; import { decodeCodexTurnUsage, type CodexTurnUsage } from "./codexHeadlessResult.js"; import { terminateChild, trackCliChild } from "./cliProcess.js"; /** Maximum assistant reply bytes retained from stdout. */ export const CLI_ENGINE_MAX_OUTPUT_BYTES = 64 * 1024; -/** Tail bytes retained from stderr only for a failed-child diagnostic. */ +/** Diagnostic bytes retained from stderr for a failed child: its head and its tail together. */ export const CLI_ENGINE_MAX_DIAGNOSTIC_BYTES = 768; const CLI_ENGINE_FAILURE_SCAN_CHARS = 256; @@ -39,18 +40,36 @@ const redactChildOutput = (value: string, secretValues: readonly string[]): stri return redactCredentialText(value, secretValues, CLI_ENGINE_MAX_OUTPUT_BYTES); }; -const utf8Tail = (value: string, maxBytes: number): string => { - const bytes = Buffer.from(value, "utf8"); - if (bytes.length <= maxBytes) return value; - let result = bytes.subarray(bytes.length - maxBytes).toString("utf8"); - while (result.startsWith("\uFFFD")) result = result.slice(1); - return result; +/** + * One bounded window over a failed child's own output: its head AND its tail, + * with an explicit marker naming the bytes elided between them. + * + * A pure tail is the wrong end for the process this exists for. A worker that + * dies early prints its error first and then echoes its own input, so the tail + * is the echo: one live brokered turn reported 512 bytes of its own prompt + * read back, with the actual error already off the front and discarded. Both + * ends cost the same window, and the marker is the one oversized tool results + * already use (`toolResultSpill.ts`), so a reader meets one shape everywhere. + * + * The marker is paid for out of the same budget — it is sized against the + * largest count it could carry — so the result never exceeds `maxBytes`, and + * output that fits is returned byte-identical with no marker at all. + */ +export const boundedDiagnosticWindow = (value: string, maxBytes: number): string => { + const total = Buffer.byteLength(value, "utf8"); + if (total <= maxBytes) return value; + const budget = Math.max(0, maxBytes - Buffer.byteLength(elisionMarker(total), "utf8")); + const head = headUtf8(value, Math.floor(budget / 2)); + const tail = tailUtf8(value, budget - Buffer.byteLength(head, "utf8")); + const elided = total - Buffer.byteLength(head, "utf8") - Buffer.byteLength(tail, "utf8"); + return `${head}${elisionMarker(elided)}${tail}`; }; +const elisionMarker = (elided: number): string => `[… ${elided} bytes elided …]`; const childDiagnostic = (stdout: string, stderr: string, secretValues: readonly string[]): string => { const output = stderr.trim().length > 0 ? stderr : stdout; const redacted = redactCredentialText(output, secretValues, Number.MAX_SAFE_INTEGER).trim(); - const bounded = utf8Tail(redacted, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); + const bounded = boundedDiagnosticWindow(redacted, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); return bounded.length > 0 ? `: ${bounded}` : ""; }; @@ -104,7 +123,13 @@ export const readChild = ( const stdout: Buffer[] = []; let stdoutTail = Buffer.alloc(0); let droppingStdoutLine = false; + // Both ends of stderr, retained as it streams: the head frozen once it is + // full, the tail sliding. A single sliding tail dropped the head at capture + // time, which is where the cause of an early death lives — no later window + // can recover what was never kept. + let stderrHead = Buffer.alloc(0); let stderrTail = Buffer.alloc(0); + let stderrBytes = 0; let stdoutBytes = 0; let stdoutRemainder = Buffer.alloc(0); let droppingNdjsonLine = false; @@ -113,8 +138,18 @@ export const readChild = ( let cleanupStarted = false; let classifiedFailure: Error | undefined; let failureScanTail = ""; - const stderrRetentionBytes = CLI_ENGINE_MAX_DIAGNOSTIC_BYTES - + Math.max(0, ...secretValues.map((secret) => Buffer.byteLength(secret, "utf8"))); + /** + * Each retained end carries its reported share PLUS one whole secret, which + * is the invariant that keeps exact redaction exact: a secret that reaches + * the reported window can extend at most its own length past that window's + * cut, so retaining that much more on each side means the redactor always + * sees the secret whole. Halving one shared budget instead broke it — a + * 2000-byte secret was cut in the middle and its tail fragment + * (`…qqq-secret-end`) reached the diagnostic verbatim. + */ + const stderrSecretAllowance = Math.max(0, ...secretValues.map((secret) => Buffer.byteLength(secret, "utf8"))); + const stderrHeadBytes = Math.floor(CLI_ENGINE_MAX_DIAGNOSTIC_BYTES / 2) + stderrSecretAllowance; + const stderrTailBytes = CLI_ENGINE_MAX_DIAGNOSTIC_BYTES - Math.floor(CLI_ENGINE_MAX_DIAGNOSTIC_BYTES / 2) + stderrSecretAllowance; const settle = (action: () => void): void => { if (settled) return; settled = true; @@ -221,18 +256,33 @@ export const readChild = ( } stdout.push(value); }; - const retainStderrTail = (chunk: Buffer): void => { + const retainStderrWindow = (chunk: Buffer): void => { const value = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); classifyFailure(value); - if (value.length >= stderrRetentionBytes) { - stderrTail = Buffer.from(value.subarray(value.length - stderrRetentionBytes)); + stderrBytes += value.length; + if (stderrHead.length < stderrHeadBytes) stderrHead = Buffer.concat([stderrHead, value.subarray(0, stderrHeadBytes - stderrHead.length)]); + if (value.length >= stderrTailBytes) { + stderrTail = Buffer.from(value.subarray(value.length - stderrTailBytes)); return; } - const overflow = stderrTail.length + value.length - stderrRetentionBytes; + const overflow = stderrTail.length + value.length - stderrTailBytes; stderrTail = Buffer.concat([overflow > 0 ? stderrTail.subarray(overflow) : stderrTail, value]); }; + /** + * The retained stderr, reassembled exactly. + * + * Nothing was elided while the whole output fitted the budget, and then the + * two ends overlap: they tile the stream, so dropping the overlap from the + * tail rebuilds it byte-identically. Above the budget the ends are joined by + * the marker, which names how many bytes never reached this process at all. + */ + const retainedStderr = (): string => { + const elided = stderrBytes - stderrHead.length - stderrTail.length; + if (elided <= 0) return Buffer.concat([stderrHead, stderrTail.subarray(stderrHead.length + stderrTail.length - stderrBytes)]).toString("utf8"); + return `${stderrHead.toString("utf8")}${elisionMarker(elided)}${stderrTail.toString("utf8")}`; + }; child.stdout?.on("data", retainStdout); - child.stderr?.on("data", retainStderrTail); + child.stderr?.on("data", retainStderrWindow); const timer = timeoutMs === undefined ? undefined : setTimeout(() => abort(tagCliChildFailure(new Error(options.timeoutErrorMessage ?? "CLI engine timed out"), "wake_timeout")), timeoutMs); child.once("error", abort); child.once("close", (code, signal) => { @@ -244,7 +294,7 @@ export const readChild = ( return; } settle(() => reject(tagCliChildFailure(classifiedFailure ?? new Error(`CLI engine exited ${code ?? signal}${childDiagnostic( - retainedStdout.toString("utf8"), stderrTail.toString("utf8"), secretValues + retainedStdout.toString("utf8"), retainedStderr(), secretValues )}`), "engine_exit"))); }); }); diff --git a/src/pi/cliSessionOutput.test.ts b/src/pi/cliSessionOutput.test.ts index 3e6ec4a..65d0b2c 100644 --- a/src/pi/cliSessionOutput.test.ts +++ b/src/pi/cliSessionOutput.test.ts @@ -34,11 +34,15 @@ test("verbose progress stderr is drained without invalidating a bounded successf } }); -test("failed verbose stderr retains only a redacted bounded diagnostic tail", async () => { +test("failed verbose stderr retains a redacted bounded window of its head AND its tail", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-cli-stderr-diagnostic-")); const engine = path.join(root, "verbose-failure.mjs"); const secret = "bounded-diagnostic-secret-value"; await writeFile(engine, [ + // The real shape of an early death: the cause first, then a flood, then + // the last thing the child happened to print. A pure tail kept only the + // last of the three. + `process.stderr.write(${JSON.stringify("first-cause: profile refused\n")});`, `process.stderr.write("p".repeat(${CLI_ENGINE_MAX_OUTPUT_BYTES * 8}));`, `process.stderr.write(${JSON.stringify(` final-error ${secret}`)});`, "process.exitCode = 7;" @@ -54,7 +58,9 @@ test("failed verbose stderr retains only a redacted bounded diagnostic tail", as await assert.rejects(readChild(child, 10_000, [secret]), (error: unknown) => { assert.ok(error instanceof Error); assert.match(error.message, /CLI engine exited 7/); + assert.match(error.message, /first-cause: profile refused/, "the head of the output is the error, and must survive the bound"); assert.match(error.message, /final-error \[REDACTED\]/); + assert.match(error.message, /\[… \d+ bytes elided …\]/, "and what was dropped between the two ends is named"); assert.equal(error.message.includes(secret), false); assert.ok(Buffer.byteLength(error.message) <= CLI_ENGINE_MAX_DIAGNOSTIC_BYTES + 80); return true; @@ -64,6 +70,45 @@ test("failed verbose stderr retains only a redacted bounded diagnostic tail", as } }); +/** + * The mirror of the long-secret case, at the other cut. Retaining both ends + * only stays safe while each end carries a whole secret's worth beyond its + * reported share: sizing the two ends by halving one shared budget let a + * 2000-byte secret straddle the cut and leak its own tail verbatim. + */ +test("a secret straddling the head's own retention edge is still redacted whole", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-cli-head-edge-secret-")); + const engine = path.join(root, "head-edge-secret.mjs"); + // The first ten bytes are a distinctive token, so any surviving prefix of + // this credential is visible to the assertion rather than merely shorter. + const secret = `CREDENTIAL-${"z".repeat(200)}-END`; + // Land it across the edge of the head's REPORTED share + // (`CLI_ENGINE_MAX_DIAGNOSTIC_BYTES / 2`): 80 bytes inside the window, the + // rest past it. Only the extra secret-length the head retains beyond that + // share lets the redactor match it whole; without it those 80 bytes are a + // verbatim credential prefix in the diagnostic. + const pad = Math.floor(CLI_ENGINE_MAX_DIAGNOSTIC_BYTES / 2) - 80; + await writeFile(engine, [ + `process.stderr.write("h".repeat(${pad}));`, + `process.stderr.write(${JSON.stringify(secret)});`, + `process.stderr.write("p".repeat(${CLI_ENGINE_MAX_OUTPUT_BYTES * 8}));`, + `process.stderr.write(${JSON.stringify(" terminal-detail")});`, + "process.exitCode = 9;" + ].join("\n")); + try { + const child = spawnEngine({ + command: process.execPath, commandArgs: [engine], engine: "agy", + maxToolTurns: 1, timeoutMs: 10_000 + }, "verbose", { cwd: root }, undefined); + await assert.rejects(readChild(child, 10_000, [secret]), (error: unknown) => { + assert.ok(error instanceof Error); + assert.match(error.message, /terminal-detail/u, "the tail still reports the last words"); + assert.doesNotMatch(error.message, /CREDENTIAL|z{32}/u, "no fragment of the secret may survive the cut"); + return true; + }); + } finally { await rm(root, { recursive: true, force: true }); } +}); + test("redacts a 2000-byte exact secret before retaining a failed stderr tail", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-cli-long-secret-")); const engine = path.join(root, "long-secret-failure.mjs"); diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 147dc5c..de8319d 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -89,7 +89,23 @@ prelaunch failure ran nothing. `engineBrokerNativeClient.ts` redacts that tail exactly as the CLI child path redacts a failed engine child (`redactCredentialText` with the turn's own provider/MCP capabilities as exact secrets, the same `CLI_ENGINE_MAX_DIAGNOSTIC_BYTES` bound) and flattens it to -one line as `diagnostic.reason`. It is an optional, control-character-free +one line as `diagnostic.reason`. + +Two rules that live capture taught, both cheap and both load bearing. The +bytes are **decoded**, never stringified: `Uint8Array.prototype.toString("utf8")` +ignores its argument and renders bytes as comma-separated decimals, and a +worker's last words reached an operator as +`reason=108,111,110,101,46,32,87,104,101,110,...` — a string, control-character +free, inside the bound, and passing every check on the way out. So the frame is +normalized to a `Buffer` once on entry and the diagnostic goes through an +explicit `TextDecoder`, which also replaces rather than throws on the +multi-byte sequence a byte-counted window can cut in half. And the window +keeps **both ends** (`boundedDiagnosticWindow`): a worker that dies early +prints its error before it echoes its input, so a pure tail is the echo. The +marker is paid out of the same budget, and output that fits is returned +byte-identical. The launcher's own 512-byte window is still tail-only — see +`native/AGENTS.md`, it needs an artifact rebuild — so the head of a large blob +is still lost before Daimon sees it. It is an optional, control-character-free member of the sealed terminal response's closed diagnostic — admitted by `engineBrokerProtocol.ts` only for the statuses where a worker ran and spoke — so it replays with the sealed record and reaches the operator through diff --git a/src/runtime/engineBrokerNativeClient.test.ts b/src/runtime/engineBrokerNativeClient.test.ts index f960d63..4b0d8db 100644 --- a/src/runtime/engineBrokerNativeClient.test.ts +++ b/src/runtime/engineBrokerNativeClient.test.ts @@ -1,10 +1,10 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { CLI_ENGINE_MAX_DIAGNOSTIC_BYTES } from "../pi/cliChildOutput.js"; +import { boundedDiagnosticWindow, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES } from "../pi/cliChildOutput.js"; import { decodeNativeBrokerResult,encodeNativeBrokerTurn,ENGINE_BROKER_NATIVE_DIAGNOSTIC_BYTES,ENGINE_BROKER_NATIVE_RESULT_BYTES,NativeBrokerTurnFailure } from "./engineBrokerNativeClient.js"; const turnId="turn-1"; -function frame(values:Readonly<{status?:number;uid?:number;pid?:number;exit?:number;signal?:number;ticks?:bigint;stage?:number;failure?:number;profile?:number;diagnosticLength?:number;diagnostic?:string;text?:string}>={}):Buffer{ +function frame(values:Readonly<{status?:number;uid?:number;pid?:number;exit?:number;signal?:number;ticks?:bigint;stage?:number;failure?:number;profile?:number;diagnosticLength?:number;diagnostic?:string|Buffer;text?:string}>={}):Buffer{ const text=Buffer.from(values.text??""),diagnostic=Buffer.from(values.diagnostic??"");const out=Buffer.alloc(ENGINE_BROKER_NATIVE_RESULT_BYTES+text.length+diagnostic.length);out.writeUInt32LE(2,0);out.writeUInt32LE(values.status??0,4);out.writeUInt32LE(values.uid??2200,8);out.writeUInt32LE(text.length,12);out.writeInt32LE(values.pid??42,16);out.writeInt32LE(values.exit??0,20);out.writeInt32LE(values.signal??0,24);out.writeBigUInt64LE(values.ticks??123n,32);out.write(turnId,40);out.writeUInt32LE(values.stage??7,108);out.writeUInt32LE(values.failure??0,112);out.writeUInt32LE(values.profile??0,116);out.writeUInt32LE(values.diagnosticLength??diagnostic.length,120);text.copy(out,ENGINE_BROKER_NATIVE_RESULT_BYTES);diagnostic.copy(out,ENGINE_BROKER_NATIVE_RESULT_BYTES+text.length);return out; } @@ -41,3 +41,59 @@ test("a failed worker's own last words cross as a redacted, bounded reason",()=> }); assert.throws(()=>decodeNativeBrokerResult(frame({status:2,stage:6,failure:5,exit:1}),turnId,[]),(error:unknown)=>error instanceof NativeBrokerTurnFailure&&error.diagnostic.reason===undefined,"a worker that said nothing reports no reason rather than an empty one"); }); + +/** + * The assertion is the SENTENCE. A live turn reported its worker's last words + * as `reason=108,111,110,101,46,32,87,104,101,110,...` — the bytes of "lone. + * When ..." rendered as decimals, because a byte view that is not a Node + * `Buffer` answers `toString("utf8")` with a comma-separated list and every + * check the diagnostic passed on the way out (a string, no control bytes, + * under the bound) is satisfied by digits. Nothing weaker than the decoded + * text can catch that. + */ +const words = "lone. When assigned, read `room:assignment`, open my row in the desk index."; + +test("a worker's last words cross as decoded text, not as the decimals of their bytes", () => { + for (const [shape, view] of [["a Buffer", (bytes: Buffer): Uint8Array => bytes], ["a plain Uint8Array", (bytes: Buffer): Uint8Array => new Uint8Array(bytes)]] as const) { + assert.throws(() => decodeNativeBrokerResult(view(frame({ status: 2, stage: 6, failure: 5, exit: 1, diagnostic: words })), turnId, []), (error: unknown) => { + assert.ok(error instanceof NativeBrokerTurnFailure); + assert.equal(error.diagnostic.reason, words, `${shape}: the reason must be the worker's sentence, byte-identical`); + assert.doesNotMatch(error.diagnostic.reason ?? "", /^[0-9,]+$/u, `${shape}: a decimal byte list is what this regressed to before`); + return true; + }); + } +}); + +test("a window that cut a multi-byte sequence in half decodes with replacement instead of throwing", () => { + // The launcher's window is a byte count: these first two bytes are the tail + // of a three-byte sequence whose leading byte the window already dropped. + const cut = Buffer.concat([Buffer.from([0x9c, 0xa8]), Buffer.from(" grok: exiting 1")]); + assert.throws(() => decodeNativeBrokerResult(frame({ status: 2, stage: 6, failure: 5, exit: 1, diagnostic: cut }), turnId, []), (error: unknown) => { + assert.ok(error instanceof NativeBrokerTurnFailure); + const reason = error.diagnostic.reason ?? ""; + assert.match(reason, /grok: exiting 1$/u, "the legible remainder survives the cut sequence"); + assert.match(reason, /\uFFFD/u, "the cut sequence is replaced, not thrown on"); + return true; + }); +}); + +/** + * The window keeps both ends. A pure tail is what turned the one diagnostic + * this project has ever got out of a failed worker into 512 bytes of the + * worker's own prompt echoed back, with the error itself off the front. + */ +test("the bounded diagnostic window keeps the head, the tail, and a marker naming what it dropped", () => { + const blob = `START-OF-ERROR ${"m".repeat(40_000)} END-OF-ECHO`; + const window = boundedDiagnosticWindow(blob, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES); + assert.ok(window.startsWith("START-OF-ERROR "), `the head must survive: ${window.slice(0, 40)}`); + assert.ok(window.endsWith(" END-OF-ECHO"), "the tail must survive too"); + const marker = /\[… (\d+) bytes elided …\]/u.exec(window); + assert.ok(marker !== null, "the elision is named, not silent"); + assert.equal(Number(marker[1]) + Buffer.byteLength(window.replace(marker[0], ""), "utf8"), Buffer.byteLength(blob, "utf8"), "the marker's count is exactly what was dropped"); + assert.ok(Buffer.byteLength(window, "utf8") <= CLI_ENGINE_MAX_DIAGNOSTIC_BYTES, `the marker is paid for out of the same budget: ${Buffer.byteLength(window, "utf8")} bytes`); +}); + +test("output that fits the window is returned byte-identical, with no marker", () => { + for (const value of ["", "grok: exiting 1", `${"m".repeat(CLI_ENGINE_MAX_DIAGNOSTIC_BYTES - 4)}tail`]) + assert.equal(boundedDiagnosticWindow(value, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES), value, "a short diagnostic must not be reshaped at all"); +}); diff --git a/src/runtime/engineBrokerNativeClient.ts b/src/runtime/engineBrokerNativeClient.ts index 18fff46..cac8711 100644 --- a/src/runtime/engineBrokerNativeClient.ts +++ b/src/runtime/engineBrokerNativeClient.ts @@ -1,6 +1,6 @@ import { spawn } from "node:child_process"; import { redactCredentialText } from "../core/credentialRedaction.js"; -import { CLI_ENGINE_MAX_DIAGNOSTIC_BYTES } from "../pi/cliChildOutput.js"; +import { boundedDiagnosticWindow, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES } from "../pi/cliChildOutput.js"; import { terminateChild, trackCliChild } from "../pi/cliProcess.js"; export const ENGINE_BROKER_NATIVE_REQUEST_BYTES = 396; @@ -24,12 +24,23 @@ export async function runNativeBrokerTurn(executable:string,input:NativeBrokerTu } export function encodeNativeBrokerTurn(input:NativeBrokerTurn):Buffer{if(!Number.isInteger(input.slot)||input.slot<0)throw new TypeError("invalid engine broker turn");const p=Buffer.from(input.prompt),provider=Buffer.from(input.providerCapability),mcp=Buffer.from(input.mcpCapability);if(p.length<1||p.length>MAX_PROMPT||provider.length<1||provider.length>MAX_CAPABILITY||mcp.length<1||mcp.length>MAX_CAPABILITY||provider.equals(mcp))throw new TypeError("invalid engine broker turn");const c=Buffer.alloc(4+provider.length+mcp.length);c.writeUInt16LE(provider.length,0);provider.copy(c,2);c.writeUInt16LE(mcp.length,2+provider.length);mcp.copy(c,4+provider.length);const frame=Buffer.alloc(ENGINE_BROKER_NATIVE_REQUEST_BYTES+8+p.length+c.length);frame.writeUInt32LE(2,0);frame.writeUInt32LE(input.slot,4);field(frame,8,65,input.requestId);field(frame,73,65,input.turnId);field(frame,138,129,input.agentId);field(frame,267,129,input.wakeId);let o=ENGINE_BROKER_NATIVE_REQUEST_BYTES;frame.writeUInt32LE(p.length,o);o+=4;p.copy(frame,o);o+=p.length;frame.writeUInt32LE(c.length,o);o+=4;c.copy(frame,o);p.fill(0);provider.fill(0);mcp.fill(0);c.fill(0);return frame;} -export function decodeNativeBrokerResult(output:Buffer,turnId:string,secrets:readonly string[]=[]):NativeBrokerTurnResult{ +/** + * `output` is accepted as any byte view, and normalized to a `Buffer` once + * here: `Uint8Array.prototype.toString("utf8")` ignores its argument and + * renders the bytes as comma-separated decimals, which is how a live worker's + * last words reached an operator as `reason=108,111,110,101,...` instead of a + * sentence. Decoding is explicit from here on, never a stringification. + */ +export function decodeNativeBrokerResult(input:Uint8Array,turnId:string,secrets:readonly string[]=[]):NativeBrokerTurnResult{ + const output=Buffer.isBuffer(input)?input:Buffer.from(input.buffer,input.byteOffset,input.byteLength); if(output.lengthbytes.every((byte)=>byte===0)); if(output.length!==ENGINE_BROKER_NATIVE_RESULT_BYTES+length+diagnosticLength||output.readUInt32LE(0)!==2||status>=statuses.length||stage>=stages.length||failure>=failures.length||profile>1||!paddingZero||observed!==turnId||length>MAX_OUTPUT||diagnosticLength>ENGINE_BROKER_NATIVE_DIAGNOSTIC_BYTES)throw new Error("engine broker turn failed"); - const reason=workerReason(output.subarray(ENGINE_BROKER_NATIVE_RESULT_BYTES+length),secrets); + // Decoded from the caller's own view, not from the normalized frame: the + // decode is what must be correct for any byte view, and it is the step the + // live `reason=108,111,110,...` failure came from. + const reason=workerReason(input.subarray(ENGINE_BROKER_NATIVE_RESULT_BYTES+length),secrets); const diagnostic:NativeBrokerDiagnostic={status:statuses[status]!,stage:stages[stage]!,failureClass:failures[failure]!,profileApplied:profile===1,...(reason===undefined?{}:{reason}),exitCode,termSignal,workerPid:pid,workerUid:uid,startTicks:ticks.toString()}; const success=status===0&&stage===7&&failure===0&&profile===0&&pid>0&&uid>=2200&&ticks>0n&&exitCode===0&&termSignal===0&&diagnosticLength===0; const prelaunch=status===1&&stage>=1&&stage<=5&&failure>=1&&failure<=5&&profile===0&&pid===0&&uid===0&&ticks===0n&&diagnosticLength===0; @@ -44,16 +55,27 @@ export function decodeNativeBrokerResult(output:Buffer,turnId:string,secrets:rea * * A failed brokered turn otherwise reports nothing but `exit=1`: the launcher * merges the worker's stdout and stderr into one pipe and publishes no output - * for a failure, so this bounded tail is the only account of why it failed. + * for a failure, so this bounded window is the only account of why it failed. + * It keeps both ends of what it is given (`boundedDiagnosticWindow`), because + * a worker that dies early prints its error before it echoes anything. * It is worker-controlled text, so it is redacted exactly as the CLI child * path redacts a failed engine child (`redactCredentialText` with the turn's * own capabilities as exact secrets, the same diagnostic bound) and flattened * to one line, because it travels inside a failure message. */ -function workerReason(tail:Buffer,secrets:readonly string[]):string|undefined{ - if(tail.length===0)return undefined; - const flattened=tail.toString("utf8").replace(/[\u0000-\u001f\u007f]+/gu," ").replace(/\s+/gu," ").trim(); - const reason=redactCredentialText(flattened,secrets,CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); +function workerReason(tail:Uint8Array,secrets:readonly string[]):string|undefined{ + if(tail.byteLength===0)return undefined; + // Decoded explicitly, and with replacement rather than a throw: the window + // is a byte count, so it can cut a multi-byte sequence in half at either + // end, and a worker's last words must not be lost to its own encoding. + const flattened=UTF8.decode(tail).replace(/[\u0000-\u001f\u007f]+/gu," ").replace(/\s+/gu," ").trim(); + // Redact first, unbounded, then window: redaction can lengthen the text + // ([REDACTED] is longer than a short secret), so bounding before it could + // hand back more bytes than the boundary admits. + const redacted=redactCredentialText(flattened,secrets,Number.MAX_SAFE_INTEGER).trim(); + const reason=boundedDiagnosticWindow(redacted,CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); return reason.length===0?undefined:reason; } +/** Non-fatal by construction: a cut multi-byte sequence becomes U+FFFD, never an exception. */ +const UTF8=new TextDecoder("utf-8"); function field(target:Buffer,offset:number,length:number,value:string):void{if(!/^[A-Za-z0-9][A-Za-z0-9._:-]*$/u.test(value)||Buffer.byteLength(value)>=length)throw new TypeError("invalid engine broker turn");target.write(value,offset,"utf8");} diff --git a/src/runtime/native/AGENTS.md b/src/runtime/native/AGENTS.md index 5e8f4f9..16c3b2d 100644 --- a/src/runtime/native/AGENTS.md +++ b/src/runtime/native/AGENTS.md @@ -46,3 +46,19 @@ bytes of the worker's merged stdout/stderr and sends them after the fixed frame, while `output_length` stays 0 as before. Every other failure sends none, and `closed_result` refuses a frame that mixes the two. The bytes are the worker's own, so the broker redacts them before they cross any boundary. + +**Known gap: that window is the wrong end.** A worker that dies early prints +its error first and then echoes its own input, so a pure tail keeps the echo: +the one live capture this has ever produced was 512 bytes of the agent's own +prompt read back, with the error already off the front and erased here. The +broker side now keeps both ends of whatever it is handed +(`boundedDiagnosticWindow` in `../../pi/cliChildOutput.ts`, the same +head-plus-marker-plus-tail shape as an oversized tool result), but it cannot +recover a head this supervisor never sent. The fix belongs in +`engineBrokerLauncherServer.inc`, where the full `used` bytes are still in +hand at the point of the `memmove`: keep the first `DBL_MAX_DIAGNOSTIC / 2` +bytes, then a marker naming the elided count, then the last +`DBL_MAX_DIAGNOSTIC / 2`. It is a source change to a *pinned* artifact, so it +lands only together with `node --import tsx src/runtime/native/build.ts` and a +re-pin of `artifacts.sourceSha256`/`x64Sha256`/`arm64Sha256`; +`artifactsManifest.test.ts` fails by design until the binaries are rebuilt. diff --git a/src/runtime/toolResultSpill.ts b/src/runtime/toolResultSpill.ts index f557c38..5955577 100644 --- a/src/runtime/toolResultSpill.ts +++ b/src/runtime/toolResultSpill.ts @@ -116,7 +116,8 @@ const measure = (content: readonly unknown[], details: Record): Buffer.byteLength(safeStringify({ content, structuredContent: details }), "utf8"); /** Cut on a code-point boundary, from the front. */ -const headUtf8 = (value: string, maxBytes: number): string => { +/** Exported for the bounded diagnostic window (`cliChildOutput.ts`): one boundary-safe implementation, not two. */ +export const headUtf8 = (value: string, maxBytes: number): string => { const bytes = Buffer.from(value, "utf8"); if (bytes.byteLength <= maxBytes) return value; let end = Math.max(0, maxBytes); @@ -125,7 +126,8 @@ const headUtf8 = (value: string, maxBytes: number): string => { }; /** Cut on a code-point boundary, from the back. */ -const tailUtf8 = (value: string, maxBytes: number): string => { +/** Exported for the bounded diagnostic window (`cliChildOutput.ts`): one boundary-safe implementation, not two. */ +export const tailUtf8 = (value: string, maxBytes: number): string => { const bytes = Buffer.from(value, "utf8"); if (bytes.byteLength <= maxBytes) return value; let start = Math.max(0, bytes.byteLength - maxBytes); From 3f063c6f2ec913e55b7b1fa3da108d558ee4e335 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 01:48:03 +0200 Subject: [PATCH 42/69] fix: keep both ends of the launcher's diagnostic window and scrub credentials split by its cut --- src/contracts/runtimeContractManifest.ts | 6 +- src/runtime/AGENTS.md | 11 +++- src/runtime/engineBrokerNativeClient.test.ts | 33 ++++++++++ src/runtime/engineBrokerNativeClient.ts | 39 +++++++++++- src/runtime/native/AGENTS.md | 43 ++++++++----- .../artifacts/daimon-engine-broker-arm64 | Bin 67456 -> 67456 bytes ...daimon-engine-broker-arm64.provenance.json | 2 +- .../native/artifacts/daimon-engine-broker-x64 | Bin 55392 -> 55392 bytes .../daimon-engine-broker-x64.provenance.json | 2 +- src/runtime/native/engineBrokerLauncher.h | 5 ++ ...ngineBrokerLauncherIntegrationLauncher.inc | 43 +++++++++++++ .../native/engineBrokerLauncherServer.inc | 59 +++++++++++++++--- src/runtime/native/fixtureWorker.c | 2 +- 13 files changed, 210 insertions(+), 35 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 41baa05..548340c 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -133,9 +133,9 @@ export const GROK_ENGINE_BROKER = { projectionVersion: "noopolis.daimon.grok-broker-projection.v1", slotPreflightVersion: "noopolis.daimon.grok-slot-preflight.v2", artifacts: { - sourceSha256: "356a8e56e44dca9ab4784ac343587c0fc4d57e23f961124d8491cf6f504f8e98", - x64Sha256: "a21efd6a47059de7c5fe6add8b23799946728dfb44845ea9b6602402e5cfad02", - arm64Sha256: "a8bf311ca82ed004dd4efd69d1b7ae9edc1264876ca9bbb94c77226d40c75381" + sourceSha256: "d6bc575ab239f3cf3140b5ec296f72f9890617a6c38ca4328a7b77d6fd9f2c61", + x64Sha256: "3fd834c512a926002e215540568f75b8f1e9d9a5362ad8a9cb661ffa771884d5", + arm64Sha256: "ffa965b506160f432839e69ffe36518c1dc8c013d651bd4c8a200d4c44df2277" } } as const; export const AGY_SUBSCRIPTION_REALM = { diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index de8319d..e87dc52 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -103,9 +103,14 @@ multi-byte sequence a byte-counted window can cut in half. And the window keeps **both ends** (`boundedDiagnosticWindow`): a worker that dies early prints its error before it echoes its input, so a pure tail is the echo. The marker is paid out of the same budget, and output that fits is returned -byte-identical. The launcher's own 512-byte window is still tail-only — see -`native/AGENTS.md`, it needs an artifact rebuild — so the head of a large blob -is still lost before Daimon sees it. It is an optional, control-character-free +byte-identical. The launcher's own 512-byte window keeps both ends too +(`diagnostic_window`, `native/AGENTS.md`), so the head of a large blob now +survives the one place it used to be erased. Its elision is a cut, and a cut +can split a capability in half into a fragment exact redaction cannot match, so +`scrubCutFragments` matches that fragment here, where the turn's capabilities +are known — on both sides of every marker and at the window's outer ends. A +margin reserved in the launcher could not do this: there, what is kept is +exactly what is sent. It is an optional, control-character-free member of the sealed terminal response's closed diagnostic — admitted by `engineBrokerProtocol.ts` only for the statuses where a worker ran and spoke — so it replays with the sealed record and reaches the operator through diff --git a/src/runtime/engineBrokerNativeClient.test.ts b/src/runtime/engineBrokerNativeClient.test.ts index 4b0d8db..104150d 100644 --- a/src/runtime/engineBrokerNativeClient.test.ts +++ b/src/runtime/engineBrokerNativeClient.test.ts @@ -97,3 +97,36 @@ test("output that fits the window is returned byte-identical, with no marker", ( for (const value of ["", "grok: exiting 1", `${"m".repeat(CLI_ENGINE_MAX_DIAGNOSTIC_BYTES - 4)}tail`]) assert.equal(boundedDiagnosticWindow(value, CLI_ENGINE_MAX_DIAGNOSTIC_BYTES), value, "a short diagnostic must not be reshaped at all"); }); + +/** + * The cut the launcher makes is the one place exact redaction cannot reach on + * its own, and it is the cut this test straddles: the capability begins inside + * the retained head and ends inside the elided middle, so the redactor never + * sees it whole and, without the scrub, its first characters travel verbatim. + * The mirror case is the tail's leading edge, where the capability's last + * characters survive instead. + */ +test("a capability the launcher's window cut in half never crosses as a fragment", () => { + const provider = `provider-${"A".repeat(34)}`, mcp = `mcp-${"B".repeat(39)}`; + const elision = "[… 9000 bytes elided …]"; + const cut = `grok: refused ${provider.slice(0, 20)}${elision}${mcp.slice(mcp.length - 20)} exiting 1`; + assert.throws(() => decodeNativeBrokerResult(frame({ status: 2, stage: 6, failure: 5, exit: 1, diagnostic: cut }), turnId, [provider, mcp]), (error: unknown) => { + assert.ok(error instanceof NativeBrokerTurnFailure); + const reason = error.diagnostic.reason ?? ""; + assert.doesNotMatch(reason, /A{12}|B{12}|provider-A|BBB-?mcp/u, `no credential fragment may cross: ${reason}`); + assert.match(reason, /grok: refused \[REDACTED\]/u, "the head's cut fragment is marked where it was"); + assert.match(reason, /\[REDACTED\] exiting 1$/u, "and so is the tail's"); + assert.ok(reason.includes(elision), "the launcher's own elision marker is left alone"); + return true; + }); +}); + +test("ordinary words at a cut are not eaten by the fragment scrub", () => { + const provider = `provider-${"A".repeat(34)}`; + const words = "grok: profile refused, exiting 1"; + assert.throws(() => decodeNativeBrokerResult(frame({ status: 2, stage: 6, failure: 5, exit: 1, diagnostic: words }), turnId, [provider]), (error: unknown) => { + assert.ok(error instanceof NativeBrokerTurnFailure); + assert.equal(error.diagnostic.reason, words, "text that is not a credential fragment is untouched"); + return true; + }); +}); diff --git a/src/runtime/engineBrokerNativeClient.ts b/src/runtime/engineBrokerNativeClient.ts index cac8711..de6406a 100644 --- a/src/runtime/engineBrokerNativeClient.ts +++ b/src/runtime/engineBrokerNativeClient.ts @@ -72,10 +72,47 @@ function workerReason(tail:Uint8Array,secrets:readonly string[]):string|undefine // Redact first, unbounded, then window: redaction can lengthen the text // ([REDACTED] is longer than a short secret), so bounding before it could // hand back more bytes than the boundary admits. - const redacted=redactCredentialText(flattened,secrets,Number.MAX_SAFE_INTEGER).trim(); + const redacted=redactCredentialText(scrubCutFragments(flattened,secrets),secrets,Number.MAX_SAFE_INTEGER).trim(); const reason=boundedDiagnosticWindow(redacted,CLI_ENGINE_MAX_DIAGNOSTIC_BYTES).trim(); return reason.length===0?undefined:reason; } /** Non-fatal by construction: a cut multi-byte sequence becomes U+FFFD, never an exception. */ const UTF8=new TextDecoder("utf-8"); + +/** + * A credential the launcher's window cut in half, at either side of a cut. + * + * Exact redaction matches a secret whole, so a secret a cut split survives as + * a fragment it can never match: the piece before a cut can end with a + * secret's prefix, and the piece after it can begin with a secret's suffix. + * The trick that answers this where Daimon owns both ends — retain one whole + * secret more than is reported (`cliChildOutput.ts`) — cannot work at this + * boundary, because the launcher's window *is* what it sends: a margin + * reserved there would be reported along with everything else. So the fragment + * is matched here, where the turn's own capabilities are known, and every cut + * the window can make is covered: the two sides of each elision marker, and + * the outer ends, where the launcher's capture itself stopped reading. + * + * Only a fragment long enough to be a credential is scrubbed. Below + * {@link MIN_CREDENTIAL_FRAGMENT} characters a piece of a random token is + * indistinguishable from ordinary words and carries nothing usable, and + * scrubbing it would eat real text. + */ +const MIN_CREDENTIAL_FRAGMENT=12; +const ELISION_MARKER=/(\[… \d+ bytes elided …\])/u; +const scrubCutFragments=(value:string,secrets:readonly string[]):string=> + value.split(ELISION_MARKER).map((part)=>ELISION_MARKER.test(part)?part:scrubEnds(part,secrets)).join(""); +function scrubEnds(part:string,secrets:readonly string[]):string{ + let result=part; + for(const secret of secrets){ + if(secret.length<=MIN_CREDENTIAL_FRAGMENT)continue; + for(let length=Math.min(secret.length-1,result.length);length>=MIN_CREDENTIAL_FRAGMENT;length-=1){ + if(result.endsWith(secret.slice(0,length))){result=`${result.slice(0,result.length-length)}[REDACTED]`;break;} + } + for(let length=Math.min(secret.length-1,result.length);length>=MIN_CREDENTIAL_FRAGMENT;length-=1){ + if(result.startsWith(secret.slice(secret.length-length))){result=`[REDACTED]${result.slice(length)}`;break;} + } + } + return result; +} function field(target:Buffer,offset:number,length:number,value:string):void{if(!/^[A-Za-z0-9][A-Za-z0-9._:-]*$/u.test(value)||Buffer.byteLength(value)>=length)throw new TypeError("invalid engine broker turn");target.write(value,offset,"utf8");} diff --git a/src/runtime/native/AGENTS.md b/src/runtime/native/AGENTS.md index 16c3b2d..94627d9 100644 --- a/src/runtime/native/AGENTS.md +++ b/src/runtime/native/AGENTS.md @@ -47,18 +47,31 @@ frame, while `output_length` stays 0 as before. Every other failure sends none, and `closed_result` refuses a frame that mixes the two. The bytes are the worker's own, so the broker redacts them before they cross any boundary. -**Known gap: that window is the wrong end.** A worker that dies early prints -its error first and then echoes its own input, so a pure tail keeps the echo: -the one live capture this has ever produced was 512 bytes of the agent's own -prompt read back, with the error already off the front and erased here. The -broker side now keeps both ends of whatever it is handed -(`boundedDiagnosticWindow` in `../../pi/cliChildOutput.ts`, the same -head-plus-marker-plus-tail shape as an oversized tool result), but it cannot -recover a head this supervisor never sent. The fix belongs in -`engineBrokerLauncherServer.inc`, where the full `used` bytes are still in -hand at the point of the `memmove`: keep the first `DBL_MAX_DIAGNOSTIC / 2` -bytes, then a marker naming the elided count, then the last -`DBL_MAX_DIAGNOSTIC / 2`. It is a source change to a *pinned* artifact, so it -lands only together with `node --import tsx src/runtime/native/build.ts` and a -re-pin of `artifacts.sourceSha256`/`x64Sha256`/`arm64Sha256`; -`artifactsManifest.test.ts` fails by design until the binaries are rebuilt. +**The window keeps both ends.** A worker that dies early prints its error +first and then echoes its own input, so a pure tail kept the echo: the one live +capture this had ever produced was 512 bytes of the agent's own prompt read +back, with the error already off the front and erased here. `diagnostic_window` +keeps the first `DBL_MAX_DIAGNOSTIC / 2`, then `DBL_DIAGNOSTIC_ELISION` naming +the bytes dropped, then the last `DBL_MAX_DIAGNOSTIC / 2`, all inside the same +bound — the marker is sized against `used`, the largest count it can carry, so +the budget holds for every input, and a `snprintf` that will not fit falls back +to the tail. Output that already fits is left in place, byte-identical, with no +marker. The marker text is byte-identical to the TypeScript window's +(`boundedDiagnosticWindow`), so one grep finds an elision on either side of the +boundary. + +That elision is a *cut*, and a cut can split a turn capability in half, leaving +a fragment exact redaction can never match. The answer used where Daimon owns +both ends — retain one whole secret more than is reported — cannot work here, +because what this window keeps is exactly what it sends: a margin reserved here +would be sent too. So the fragment is scrubbed where the capabilities are +known, in `engineBrokerNativeClient.ts` (`scrubCutFragments`), on both sides of +every marker and at the window's outer ends. + +Changing any of the six pinned launcher sources means rebuilding: `node +--import tsx src/runtime/native/build.ts`, then re-pin +`artifacts.sourceSha256`/`x64Sha256`/`arm64Sha256` in the contract manifest and +re-emit it. `artifactsManifest.test.ts` fails by design until that is done. The +adversarial suite is `docker build -f Dockerfile.integration -t .` in this +folder and `docker run --rm --privileged `; `worker_flood_case` is the +head-and-tail cover and fails first if the window regresses to a tail. diff --git a/src/runtime/native/artifacts/daimon-engine-broker-arm64 b/src/runtime/native/artifacts/daimon-engine-broker-arm64 index 8f89d08f2297daaa569b69b164f914dd6d52455a..3ce7ade6e2817f7cc88e9b1fc5c33df78c69292f 100755 GIT binary patch delta 3187 zcmYjU3sjWH6`uM3W!dGS;PMhgb|HYQh>x%!?|%XDm6I6coV=Q55sgXl5d{>5W`B%L z+L9cZP7G0_S=-ozt!Xf8QupKqF|p>vL|jd3(wuZbTTk1h2ah6a5U1ZDHvP}Jd~@%e z|IVFn?%aRTcNz3uhOU8#S?y%vnbpp|et(*)`Ih!0`ngEgKFDEyUcsq?Cg(He;e=Car(edkf9VSvoN|&5grR7f9t4ld_zdS zQ?41F_P;eF35gpCBMw)h=64ZbisrY1cu7lI$T_ZIz6<%ej~j2e71QNUMnv6cgTj(P zO;W=`c0skcy+dk}mzwlIP)+n!Aen36w@ERbk#=d@_F+Ni`6o4gL7hG^uK;t>NJ672 zjMP~)JhpLQs;v-PI)tgevGZ%0LUlakOmq$YR=5l>f= zI=_IeQ*M^Hk?B?Exc+~c?woLtF3F~3(v2OY-EfXKO}({~;8uC~DPAQ;58QJ@bn{_x zRN976ezGW*!RAJc4xB^~x`HHSqVHnS4X{V`{xTpMazBlJZAW07I~Fc?$6!n8WJiEX zei0=K`t4{zHQ8QyQ*`S^#1TAB8Ks`ueE13U03u4d=DWDv)osxZm8aGQE`yy&@;IFM5SIR#?)<)W1MB81l#5XaCI$+cn+LUOsS z*kZQG{mdthJ-n59T8mga zKUi!@{VTj<@IIxcV2xzEqUg2#x?X!>Bk`Z(XE5V5W>jN_y!OGhn*a8Ui;cv8lGBp3 zdxMyv<%9}@h*6KVzk%L2OX|t?4(R8le?m{P%RD97ec95VWOqUzkbbEn`a$TY_@)v| z^sAFr$IrYWNbg-C(V{C+63Kp{y z))E$mu>o?(Rl;`mm9T%$p4u}Tx6}@4>^HJy{o(AN88JOM&}Tt%sEL_uCUz+$wkI2U z79sa{UD8bKgl=Mgw@mBFgz*l@p`eNFBxWY7hC(V`Ma-rbv9TH>3;oW>RM;ft4WyJ2 zt&|OgPwA0!(XYEI`%0P3W8`m?&Qx4xe!4V1^Do!1$HRtAk0}J5B9f(Jnrs4o6uVfZunSMA6C{O^9krF7 zQogn#Rk?7DKV9*(*(q~C_7yx{t5}*ixffX)ag*bEjAP{(naUEGGHLI?I1=OWw)cCe zRS!&jT_VZ#E6DRb-+nEF(mvkrOrpd5YvYiT%-ui_6(M>H)GAKNU-Y-r>0 z(EB2w+ZAR*hm4hesplCc^l0ffL&xei;t5>EO)ElQu)_AjO&9MVJ3)JbJ)hI+>KS?z z8y|~&{ShtcMvP2rE%6g$sBlnC_V)7n>MFXCAF8e}KXX<3G{2dDUtOhyuJQ$o7A*40 zPBAX>2ayTxrZ*A+^c%wQvVlw;sxmQ3x+b~-rJl`1i)M0>_Ynw(Y}dxb#3DcCJ&PWp z3EW&$5`PvxT}2lL54X!@1Am|B_|qcvfqzGW=)0y)~VF z#P_%EqbGUMwl?MPLH_o(?bOBRZ@0w1^tljs$=vX)fVAU8&hrPhuZ+KV`bRP(0T3n{hh35JewKcf;lCc#^^LpRju@fex z`QGkT2AUVqyk5RVNb~ycy#`tk+cl}*#?#${20Az5$Nw3+lf!61L~F~f*IHZt^k5jx GQvM6q%iP=m delta 3047 zcmYjT4|J2&6@Tyh(ll)a3T;VgDNTxPZ3=}-(|^$ROH-gC(^_*jQO_ZzJE9g*{hwG3_V@0e zci(;Y-Fs6{kIvJh>ph+Ntc#3&&$`&(-Kf8|ymy`9f1?4xgmt`@FIo>sxi;W?oaOaITba9qs{nh5D7 z{tE7%y^dN#S7xuK)E`Oj%`!EE^qDm^z}Q7>Os^rWpUyuNs41g?7gf-Zi_%niIn3QHX`caq0_f;CH!jD(x(PR8< z<;-b=@C%y(r$V|jB7FDxbz0cns12m>$yLQk^3+|tuFB~C@S1S6a?Ia z$9XJKaW@uQxq{p=khhRxmnBLPP?60 z`zOkxS1d8mQog@7BV`^mq?}m8{M}k(-ePdhxx{LQHyua?DWHhX)2i`-TH`c5EM$(H za}rfDd?H)KU&&tO=J^?UBRG8o+NafAoRMQu6i;iP$J2WLaS|Tn_rT&muxNsXY{A${ zE&QFug$@$F#J%(L?;$eQl0;PuKH6~hb?k`TCixtT7kmP|HpXXLWSlbDkz%ROw%EX5 zmU_v<`+o3W^MUyp6aF+Vm3cbHCofp+UJXgUn$tniJ!!Z_DyYL&#g=%hSPIq#P)WRs z8O+rz8EZZ0qO+Qv^jEVataYG+v1+zet6^W2P70Kw!Y!Z zwWqNHn8na9jU<_!S962jLQnMHe0$w=gEN^)<_rF2-QCu2&`PqkM7uStAWgCf;;2A{ zdZK&(i%JFWM01-|ZP=7Vj>S}JN$~ReJ16hKfwG2W(NL7%BYbE5eDmAzmvZPC)IPO5 z1!=s!q}vR?L$c!G>jVB}{d%20nQi244VgNVfvx468k#Ko4a`7O*m%ZzFZxs1$yf@z zU{~1rr`53}C6T9E8%?$RVnd#y4spHqOiG)K0nsn#FKG>TRE^(7mPW<+x((}cxkjY2 zM9RkHUaU*89(BDJpe9drY|^KZ;VX#qvrwLGCZ#cMYs{v8et+X#)WQRe#dYU~g>OBP z^_OY+@G6|Tw6LZmDm%wyQ!VdHnHw_jf5ckoNjZ||z&lv__yl`C4fEOdm9&_@WarCj z;7yY=yofS0#)TDpBo(|xVdivCv|Q>XU#1)5XG=YPOXTTh;*SoZ)0SZ_m>_%khLd~A zR-}EZ#lgKzP18v#8y!h|bGep%eXa~^B?%LKqAITD1W)sLlb!bP!i5bfzZ{f0EnLDM zTWD9Zuks@cZ(sO-(^o~sfO8sj|B$dfSK8GVb zvPByaV~0G+OBOAsm3;f6`B_(>^Qf4@$isK#@;v`^Q9kYG!;9=$$DkkXXbOz1TNwEK zhA_$;9_IHfo=3a*w#7f8asJuj$LTiyqvi(up6zTdrXBoP^NuP1h}6l}d|JIZu)VPe zzxR&%DHm>vRZKDyp4odT!e75*Ctc6Wmn^3ba<-(K*6_SLSJIx)wma>V)`$MQ)McXU z_`=6N3RSPJ(7Uxag!mer-*Q-pEtm)-B*ZKhrhKLYH zfu=qou1onJgm@5lfB%#ahk=m~h4>ig{78r=D0`_q;sB`k2>(TA8E@${(2x1D z&Y4P6h(Fm`L=W+UomKQQ|EzP4;yB7DcCF26d0Pk*w!npHk8ZDeE0)ud%KP5tySs|$ z0p8b@K?nKSuEomIcleBri|VH%xy{HOcK9jCtF!HSQQOP4M*f2KBGc?I^=dEoXm;Q8 z+OD0N{pUNh-QAk~mFKiqw`fvWLXJozeS0R?l*zmQ0w%|n)e0cM{S-R8TCqpLS0yKlX&6PpF<{Mn4^a;M- zWt_Ha0OcSzkEcgkI`CS4eg>AOLSMMr-CWz^=glK6%HXQd?vaxQWw6D!LPyI|2Up3R vaJ$O4TSqG=r$2U+{|iA9zWlp^o%H(l=XsGFNZD{hP@(FyG9iM5#2k4W9s4#ve+1WX&`&_Rk zJouG{It{p0k$)|mHu?d5przin8D z>E)G%{)kkPtp>gF!grLe3F)zd=*L<@hl+77C~zFf8yh{$3U}gX7DLKjUUV|U1a$woTa6Ff%p>@%$Ac=FD;I7eY&viq{=S}q`fKtz(QrR3K5ZVt(QEs-tJeM>b$LW&C` zxPnf_h0&boYE?<(5Sw}KWsa*c2O|?mmf4Rnk1cjs%NrXEkFngiCU@(XZ4)&4P{5ndM{V(SC8& zVYn=1PNX&YhH&d#zjZ*fmFcI-XQZ>ulRaa3i9c3oevz9q>uKkjON<5x=iGYz)7n+59Xf&;5-UJ9X>(>k>tNWp|&rr9?r zO?Ok#RRAg5Hg(pEN(%Vq7r;&Sj2Kg<#ws_OtJsb5CbqCy2bmaWA(a#CjybLBv2dk@>$}B8{fL zViabu$T!t3QLZ+U2y-8ypplF=rwI*>44~BOM0=qpc8|v4k-Bp8T6K=c1-l9AI@S8x~-ffK}qIQJQS@8&4SP;MQNoI;^rN z;4L2-Reiw(G%8sxu!hNyW0J9K{!HkUIl>{R?0|h$9N#=hss$o-*^*~m$0e%eSy)BLifp=2TaS zvf&i^wKi4f4%`qVN$oJINtG&|8W6W&RK?~rYq6AGBu6I>kvhSrx%e2LJo>kSOOhw# zy~S9;i66z+LX^!o*`5Pxx6hHp+~bgLgS3)0Ijj70j%crDd@cC*7n7$fJqGWA2}awY z#UxF*KY)AGPI4$!#_ zdXP#Gc{4da^v6YNseW;R98d1qsSzV9%H!h;q$PRAfXrK3>7%+17=|Un=-mq{lT^lz zj8^MwPi6n;gnqzoSN*A~{#H5X;99F*4oXns=V zBg!eT#r$`UY}h()7q1 z3$TJZu^ak3+o}d+{q_Q4lwyTT3rL1!88?5vmX@6xv>jq;ak3z4nQyrvC+ESdLA%?) zBbWFk6F-RXsZU1i{8fut@qVeLMcMNunUrdDWI+%(SvQrAXxyw_sHZ`>`vXMmNU&^N z$3PV!{hV^KZKR&k+P0CJp3eiL=6JZyX?snWUdeV?lJ6_GzhF6*id*@YUy*!2r^fSw zvXJylixaDm9#%~3{YJT6PpoNuC;kZyTW)lJRT!cytye=B=B60T!9c3MuMfzkYqAll zECmB|F{SxB)-IArN!kQHn*1*Kn;jYY*XIBk7i|AF)|$qOKVA$z5)^ zkM>bn#Fn`-AmJd98_^+oPR=q!E16fxqVy-aBwke;iR%Owif88L`D9CaY^*ou;VRa+ z3_oEp=$#R0d2gI2wdvi2E9Z%lK2G?)j`W+7EI8{(?v&@BrQc}2it#$ip6ohS$}8$r z(w(c4Ars*lL3F~&9oK)(!&%_kc8)dk8rDJ0P`1i<{^p*#1L4XP z*fp16#1d?-B_G+Q@sDkox>WF-Cd;Ny5z^0)>ZwEd$4JZ6qMkkG;$32rJr+}lT{`B7 zOGCPY5xYzZH`2p7Bx73di2GO#RP+y3?#>}cr{xMWPx}(5PvH4zUqR-Sj>7t4-_hr` zO&)P0iR0Y<{LXkD*1<`j-p$V-F&QE{cIGqFrQ++lnviNJVgop(p^$9eJyKYmL{9H^ z$MB%>s8^PKLMe{%wf#7|=-oq_N0B9a)(MS|`RE>V#|>UXkLSL1?$*cK?FITdujJ2r zNk7lN@WpvA>cP(P_3>WfiK!&U*L#0%P#66wO6!X#b>er>LFuz(XGsP-J{;%!p=3Mn za4w*<6~XB3l(yqc-{HIZtLSoBwyp_b4_2V~^^s+%hE2F3II&z{cfuBL{mSsdL zD=F>8yjVKx`vX`31d~~Z21QD*QkudfxFr18*1O2ILurEY8o7FCV26DUoVH*RQ{JaT zQ&Dh`pC^1pB$-hj7IM9q(tc!Nc^ZF;l$6H__BY5E<+Fvs8_CFu$$T~`s>m3q-$dzn z)j9G3bYjD;?YL}k8^INBBKIohL|)#Ecj0#GVXPj=vdXUf*Th|!uw>cwPh!8yTQ!@9Xip+UN!M*c8vSx9>!%Y2j47jVxm#+8Hfl3rC^=luc+ zUMJLxNiIV&1C^NBj$_AX7r6F%VW>#%ZB$-6Jw3RrcAN#W5#WxKtyPn{n?6OVaB$dV z!fzcU_5+kwka7M#B-S6xD`d2PBEO5g>VHc}`i{g@kLUN2nbi{x e6n+OOs!s@g@F9ql){hFcm1-(B-`)E8!v6tLVD3l& delta 4489 zcmZ8l4OmoF8ou`eGMI4(1x5Uu^lFMfFr|fxIHLnxaX|dZb$3zOG(;VZ3R>*324u$N zE|SzS)9eS^TIqRwiX2Ko7=|RtCpPg1r7RuGt&7y6ZJB24+4tN#qvU;_;l1Dcedqhm z`ObH~d%6DF5Pxk*t%2|VPC2Jt@062){FDgOKwUI>=J5%9RLMbIyp0}37&3sLMfOGx z8Jb?JD-WI1(VzpDM@~Rn7SMijm-Y?9i$jKb1KO^;w28zl4tb!aPIsl=rM(syb>%Kz zt@Au1?lubZwt8YrYeV_1#3?l;?A^w3vT{M~`i)YzPjnrJQ@L^)3D7M8x?QKYk@aKW z6t+u599Pc2?b$W%vcP{$$|iL4)nr5bv-|_3C%%=x*Hb%jZ#eHEiyn0GpLxzcxRU3W zl82|Q8+lI&^WG?x^BiYWmPiI!aY?xl~o}S zWwSIM)07pvWKviEo6?G~Tbsz3WZQteDdH%XmWjM7Qji>*=zu%%piN4v;YBwKWIk3C zE56iXi@12rCS`#<2r^aq-Wt&^x+4X0D0yq{^V%HXT>P+Au|}u5e-K?SGOj)xzS&Q% zut}yiOK1qYS7b-3`3Jjl!>(M$wCoeQ<;wliLT$EF?@b@kpn&CG%XIbfz%Z~6z;d6RV(N(#$l`Z+t%dI)q<<=F}l}E%;B?!o=tG=eREGblU zZM{XQw!Ai*Gzp$KYa{!M#kz{2s$hw#ER<%}&jH(T$3TsWg9|MKMRBl-^+qti!!J?V zy_!6fGKODF{**E-HU>_#wVJP4m3C_r^B`BuGLp)a__!y7N+mnC`!NV*o25xy;Uu}4 zoJbk$EOuC%IDz9br}t)xX}#UcV1XqU#cK}Bile!rer3h6T;Vji0=dSflczbZ%^Hd^ zNUGJ*(;QShsVhPQithfw$mWxo%Kw7dRfN8=Sjv@~r6e!M88~&=H#&U~w3-FMs(g&YQT_#F z(cK-!ad4cZ+AdOR_Gx)!d&;TbfPkHQ6jG~x%XUn~wnRZPUo%s+4h&OsOntwf(lEI) zrtopH!fJ|o4x(M@)#96d>g=z{cI!B?4XjP+l@*(0RA2FvR_hpHter>5Gaoabgj-)T>tbqLz}sqAG>-!lu<){6U=6>*FM=eq3E5!N;50(v z?T0;86W~3N)TvNu%a$u=?b)D>EY+4x8*bZU{SrLh_==_Dm*4S5@E5@&lIf1Zs*(1*shm3&R$ukHPbv<-1rH!W@x>v zN7$8L@Y~AumFk}j3ae_+kv2BsZ-|CU+Io%@r;Y6Y6eK&!W5qdA zowjU3=#P5oqq^#(#Az6`=$ytRHL#;2SJX*WsbCw)ed!~G5$A|CJw0*cT+SZtTMNVL&siTw zpO=+xvk#kk1gzJDBc-}OC!6HOF*2hNZI_HlFL(CU>7?sLUqYQh<9c8P8Z zVKfA}<)$g18>H)|YPx<9tYwrI8d!U{PKxZac^~<}o+GS3Lk4BOCA{P(^_jzjVm~>P zX_~rJZ@OCFx6iNnGx-(Ac4S%CXhV|rOVjR~WCv{f)qj#dfwed7--fgT6T48Kgt|@Km z`qSi_1?z>BlO#KP`q(iib!R8BGQFm$y8Q&Au|Z8B*$W?eH+!(K>=Zeg9nWtgSF>Y; zyc5K*P#$5&)cop6#(jCJQLyQW~6CV$&2O~zM8_o1c9uXIhj^3l2nC_VDUPauCHwr_KdwyO# zo9DYc(;mqR7Z$oa?wr>$roKC!f!b#wj%=!p2(<`DyFL`9KT=u=8EvyuV z*a1%6wTUd<_mCh@C(rM5$F(G}=FqD?@gAi(w><}NSkbH}zmr5J*Hj74NKbQ(HJlX0 z_+G4)JeFEN&vTv;2P#6MlD1I#Ln)=Z@au3=x{=g>u)t&d@KxSfolog?1hY6OHKB+~ zfY3z(HEz&d| z!1iAv&mNu_lMhWA)8LZvV>|yAX*xV#FgQv7#`u2iPD;;)kQt3*`c;>PhWL4MUsDVz zZX6JnQ%30!qBPFuCy~y^Q9^qqxz@Ns*ha|GrVPG>>}^^QU$TSJ6wNtS2Q0yU@9)Fq zg4+SEYX^z*J{n`&iFeyR<^xzckQcmz_+;|7H(|qsV<`B3g zeK>Y{LQ$N3?FOJAbHU~Ju`_{d?8Dihn+k3`sc%jnT3w4&vGaq=f?pLh?H)>-$a3Ep zlI$DK%jA!~xqK*j&9_I`b&1SqN#PA-Q_F1UvTrFJ&Bozklax_1h2SDJM(1+C72L%Y zfOFi%xxnRv8`5X&u7JP3+Kf1J_VWhebr%sjr}VF`(XFd%$c)Z3R9I diff --git a/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json b/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json index 2769a2c..74d77d1 100644 --- a/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json +++ b/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json @@ -1 +1 @@ -{"version":"daimon.engine-broker-native-build.v1","architecture":"x64","target":"linux/amd64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:356a8e56e44dca9ab4784ac343587c0fc4d57e23f961124d8491cf6f504f8e98","binary_sha256":"sha256:a21efd6a47059de7c5fe6add8b23799946728dfb44845ea9b6602402e5cfad02","install_path":"/opt/daimon/bin/daimon-engine-broker"} +{"version":"daimon.engine-broker-native-build.v1","architecture":"x64","target":"linux/amd64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:d6bc575ab239f3cf3140b5ec296f72f9890617a6c38ca4328a7b77d6fd9f2c61","binary_sha256":"sha256:3fd834c512a926002e215540568f75b8f1e9d9a5362ad8a9cb661ffa771884d5","install_path":"/opt/daimon/bin/daimon-engine-broker"} diff --git a/src/runtime/native/engineBrokerLauncher.h b/src/runtime/native/engineBrokerLauncher.h index 7d902a6..636e484 100644 --- a/src/runtime/native/engineBrokerLauncher.h +++ b/src/runtime/native/engineBrokerLauncher.h @@ -14,6 +14,11 @@ reason reaches the host instead of `exit=1`. It is a diagnostic, never the turn's output: `output_length` stays 0 on every failure. */ #define DBL_MAX_DIAGNOSTIC 512u +/* The marker that joins the two ends of an elided diagnostic, byte-identical + to the TypeScript window's (`boundedDiagnosticWindow` in + `src/pi/cliChildOutput.ts`), so one grep finds every elision on either side + of the boundary. Its own bytes are paid for out of DBL_MAX_DIAGNOSTIC. */ +#define DBL_DIAGNOSTIC_ELISION "[\xe2\x80\xa6 %llu bytes elided \xe2\x80\xa6]" #ifndef DBL_REGISTRY #define DBL_REGISTRY "/etc/daimon-engine-broker/registrations.bin" #endif diff --git a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc index 243d777..a1f8367 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc @@ -111,6 +111,48 @@ static void worker_failure_case(void) { close(p); close(c); } +/* A worker that dies after printing far more than the window: the error is at + the START of what it printed and the echo at the end, which is the shape a + pure tail got wrong. The distinctive head run STRADDLES the head's own cut — + its first bytes are inside the retained head and its later bytes are elided — + so a tail-only window loses it entirely and this case fails. */ +static void worker_flood_case(void) { + int s = connect_socket(), p = sealed("prompt"), + c = sealed_bundle("stderr-flood", "mcp.Flood-2"); + struct dbl_request q = request(); + struct dbl_result r; + size_t printed = strlen("HEAD-OF-ERROR grok: profile refused ") + 4096u + + strlen(" TAIL-OF-ECHO"); + unsigned long long elided = 0; + char *reason, *marker, *shown; + send_request(s, &q, p, c); + check(read_all(s, &r, sizeof(r)) && r.status == DBL_STATUS_WORKER_FAILED && + r.exit_code == 1 && r.output_length == 0 && + r.diagnostic_length > 0 && + r.diagnostic_length <= DBL_MAX_DIAGNOSTIC, + "worker flood diagnostic length"); + reason = calloc(1, r.diagnostic_length + 1); + check(read_all(s, reason, r.diagnostic_length), "worker flood diagnostic"); + check(!strncmp(reason, "HEAD-OF-ERROR grok: profile refused", + strlen("HEAD-OF-ERROR grok: profile refused")), + "worker flood head"); + shown = strstr(reason, " TAIL-OF-ECHO"); + check(shown && shown[strlen(" TAIL-OF-ECHO")] == 0, "worker flood tail"); + marker = strstr(reason, "[\xe2\x80\xa6 "); + check(marker && sscanf(marker, "[\xe2\x80\xa6 %llu bytes elided", &elided) == 1, + "worker flood marker"); + /* The count is exactly what was dropped: everything printed, less the two + ends that survived (the whole window less the marker itself). */ + size_t marker_length = (size_t)(strchr(marker, ']') - marker) + 1u; + check((size_t)elided + (size_t)r.diagnostic_length - marker_length == printed, + "worker flood elided count"); + char extra; + check(read(s, &extra, 1) == 0, "worker flood EOF"); + free(reason); + close(s); + close(p); + close(c); +} static void org_cases(void) { check(!setgid(DBL_BROKER_UID) && !setuid(DBL_BROKER_UID), "drop broker uid"); client_case(); @@ -119,6 +161,7 @@ static void org_cases(void) { output_boundary_case("exact-output", 0); output_boundary_case("overflow-output", 1); worker_failure_case(); + worker_flood_case(); int s = connect_socket(), p = sealed("prompt"), c = sealed_bundle("provider.Token-1", "mcp.Token-2"); struct dbl_request q = request(); diff --git a/src/runtime/native/engineBrokerLauncherServer.inc b/src/runtime/native/engineBrokerLauncherServer.inc index 1d4cd76..72e0293 100644 --- a/src/runtime/native/engineBrokerLauncherServer.inc +++ b/src/runtime/native/engineBrokerLauncherServer.inc @@ -148,6 +148,45 @@ static pid_t launch(const struct dbl_registration *r, int executable, launch_fail(status_fd, 6); } +/* Both ends of a failed worker's own output, inside DBL_MAX_DIAGNOSTIC. + A pure tail was the wrong end for the process this exists for: a worker that + dies early prints its error first and then echoes its input, so the tail is + the echo. One live turn reported 512 bytes of the agent's own prompt read + back, with the error already off the front and erased here — the head is + only recoverable where it still exists, which is here. + The marker is paid for out of the same budget and sized against `used`, the + largest count it can ever carry, so the result never exceeds the bound for + any input; output that already fits is left exactly as it is, in place and + byte-identical, with no marker at all. A snprintf that will not fit falls + back to the tail this replaced. */ +static size_t diagnostic_window(unsigned char *bytes, size_t used) { + char marker[64]; + int width, final; + size_t budget, head, tail; + if (used <= DBL_MAX_DIAGNOSTIC) + return used; + width = snprintf(marker, sizeof(marker), DBL_DIAGNOSTIC_ELISION, + (unsigned long long)used); + budget = (width > 0 && (size_t)width + 2u <= DBL_MAX_DIAGNOSTIC) + ? DBL_MAX_DIAGNOSTIC - (size_t)width + : 0u; + if (!budget) { + memmove(bytes, bytes + (used - DBL_MAX_DIAGNOSTIC), DBL_MAX_DIAGNOSTIC); + return DBL_MAX_DIAGNOSTIC; + } + head = budget / 2u; + tail = budget - head; + final = snprintf(marker, sizeof(marker), DBL_DIAGNOSTIC_ELISION, + (unsigned long long)(used - head - tail)); + if (final <= 0 || (size_t)final > (size_t)width) { + memmove(bytes, bytes + (used - DBL_MAX_DIAGNOSTIC), DBL_MAX_DIAGNOSTIC); + return DBL_MAX_DIAGNOSTIC; + } + /* The head stays where it is; the tail moves up behind the marker. */ + memmove(bytes + head + (size_t)final, bytes + (used - tail), tail); + memcpy(bytes + head, marker, (size_t)final); + return head + (size_t)final + tail; +} static void supervise(int client, pid_t pid, int output, struct dbl_result *out) { unsigned char bytes[DBL_MAX_OUTPUT + 1] = {0}; @@ -226,16 +265,16 @@ static void supervise(int client, pid_t pid, int output, } if (out->status != DBL_STATUS_OK) { /* A failed turn publishes no output, but a worker that exited on its own - account said why on the pipe it shares with stdout, and that tail is the - only reason the host can ever see: without it a failure reads `exit=1`. - Keep a bounded tail of it and erase the rest here; the broker redacts it - before it crosses any boundary. The other failures get none: an - output-limit tail is the very payload the bound refused to publish, a - cancelled turn has no reader left, and a prelaunch failure ran nothing. */ - size_t keep = out->status != DBL_STATUS_WORKER_FAILED ? 0 - : used > DBL_MAX_DIAGNOSTIC ? DBL_MAX_DIAGNOSTIC - : used; - memmove(bytes, bytes + (used - keep), keep); + account said why on the pipe it shares with stdout, and that window is + the only reason the host can ever see: without it a failure reads + `exit=1`. Keep a bounded window of it and erase the rest here; the + broker redacts it before it crosses any boundary. The other failures get + none: an output-limit window is the very payload the bound refused to + publish, a cancelled turn has no reader left, and a prelaunch failure ran + nothing. */ + size_t keep = out->status != DBL_STATUS_WORKER_FAILED + ? 0 + : diagnostic_window(bytes, used); erase(bytes + keep, sizeof(bytes) - keep); out->output_length = 0; out->diagnostic_length = (uint32_t)keep; diff --git a/src/runtime/native/fixtureWorker.c b/src/runtime/native/fixtureWorker.c index d2f3674..5da8b2e 100644 --- a/src/runtime/native/fixtureWorker.c +++ b/src/runtime/native/fixtureWorker.c @@ -6,4 +6,4 @@ #include #include #include "engineBrokerLauncher.h" -int main(int argc,char**argv){unsigned char bundle[8200]={0};ssize_t size=pread(4,bundle,sizeof(bundle),0);if(size<6)return 21;size_t offset=0;unsigned provider_length=bundle[0]|(bundle[1]<<8);offset=2;if(offset+provider_length+2>(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output");if(!strcmp(provider,"sleep"))sleep(30);if(!strcmp(provider,"stderr-failure")){fprintf(stderr,"grok: session store unwritable\n");return 1;}if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;id_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s tmpdir=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target,getenv("TMPDIR")?getenv("TMPDIR"):"unset");for(int i=0;i(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output");if(!strcmp(provider,"sleep"))sleep(30);if(!strcmp(provider,"stderr-failure")){fprintf(stderr,"grok: session store unwritable\n");return 1;}if(!strcmp(provider,"stderr-flood")){fprintf(stderr,"HEAD-OF-ERROR grok: profile refused ");for(size_t i=0;i<4096;i++)if(fputc(0x6d,stderr)==EOF)return 32;fprintf(stderr," TAIL-OF-ECHO");fflush(stderr);return 1;}if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;id_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s tmpdir=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target,getenv("TMPDIR")?getenv("TMPDIR"):"unset");for(int i=0;i Date: Fri, 18 Sep 2026 02:12:03 +0200 Subject: [PATCH 43/69] fix: give the brokered worker a blocking stdout pipe so a large write cannot kill it --- src/contracts/runtimeContractManifest.ts | 6 ++-- src/runtime/native/AGENTS.md | 27 ++++++++++++++++++ .../artifacts/daimon-engine-broker-arm64 | Bin 67456 -> 67456 bytes ...daimon-engine-broker-arm64.provenance.json | 2 +- .../native/artifacts/daimon-engine-broker-x64 | Bin 55392 -> 55392 bytes .../daimon-engine-broker-x64.provenance.json | 2 +- .../native/engineBrokerLauncherServer.inc | 9 +++++- 7 files changed, 40 insertions(+), 6 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 548340c..3cf1242 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -133,9 +133,9 @@ export const GROK_ENGINE_BROKER = { projectionVersion: "noopolis.daimon.grok-broker-projection.v1", slotPreflightVersion: "noopolis.daimon.grok-slot-preflight.v2", artifacts: { - sourceSha256: "d6bc575ab239f3cf3140b5ec296f72f9890617a6c38ca4328a7b77d6fd9f2c61", - x64Sha256: "3fd834c512a926002e215540568f75b8f1e9d9a5362ad8a9cb661ffa771884d5", - arm64Sha256: "ffa965b506160f432839e69ffe36518c1dc8c013d651bd4c8a200d4c44df2277" + sourceSha256: "2d36898f02793a89a4a58601fa77c0d8716cc028588daad6daf76ca44d28446f", + x64Sha256: "5f92d83fe8159d1ac371f62d5382c0b52ce7fd167e3855460de66187a4c505a8", + arm64Sha256: "8bb4f6b1053abe27001219ab640601289b39c8ff8c86831cd7d8782f0d9dab8c" } } as const; export const AGY_SUBSCRIPTION_REALM = { diff --git a/src/runtime/native/AGENTS.md b/src/runtime/native/AGENTS.md index 94627d9..e7930de 100644 --- a/src/runtime/native/AGENTS.md +++ b/src/runtime/native/AGENTS.md @@ -40,6 +40,33 @@ binary unrunnable: Grok 1.0.34 re-executes itself inside bubblewrap by path (`/usr/local/bin/grok`), so the image path's root ownership, not the launcher descriptor, is what protects the sandboxed process. +**The worker's end of that pipe is a blocking pipe.** `O_NONBLOCK` is a +property of the open file description, not of a descriptor, so creating the +merged stdout/stderr pipe with `pipe2(..., O_NONBLOCK)` handed non-blocking +writes to the worker along with `pipes[1]`: Grok 1.0.34 makes the first EAGAIN +from a headless stdout write fatal (`stdout write failed: Resource temporarily +unavailable (os error 11)`) and exits 1 before it issues a single model +request, so the turn burns a wake and buys nothing. It stayed invisible until +the MCP tools became reachable and the init frame that enumerates them grew to +roughly 9.5 KB — past a pipe buffer, which is not always the 64 KiB default +(8 KiB inside the Docker Desktop VM this suite runs in). So the pipe is created +`O_CLOEXEC` only and `O_NONBLOCK` is set afterwards on `pipes[0]` alone, the +read end this process polls; that one is load bearing, because the post-exit +drain loop has no `poll` and would otherwise park on a write end some surviving +grandchild still holds. + +A blocking child cannot wedge the launcher. `serve()` runs in its own forked +handler per connection, so one worker's backpressure never reaches another +turn; `supervise` drains the pipe on every pass of a 250 ms `poll`, and both +bounds act on a child that is asleep in `write()`: crossing `DBL_MAX_OUTPUT` +stops reading (`p[1].events = 0`) and `kill(-pid, SIGKILL)`s the worker's whole +process group in the same iteration, and a client disconnect does the same — +neither is refusable by a process sleeping on a pipe. `worker_spill_case` is +the cover: the fixture shrinks its own stdout pipe to the kernel minimum, +reports the capacity it actually got, and writes four times that in one +`write`, so it straddles the buffer on any host without assuming 64 KiB while +staying under `DBL_MAX_OUTPUT`. + The result frame's last word is `diagnostic_length`, not padding: on `DBL_STATUS_WORKER_FAILED` the supervisor keeps the last `DBL_MAX_DIAGNOSTIC` bytes of the worker's merged stdout/stderr and sends them after the fixed diff --git a/src/runtime/native/artifacts/daimon-engine-broker-arm64 b/src/runtime/native/artifacts/daimon-engine-broker-arm64 index 3ce7ade6e2817f7cc88e9b1fc5c33df78c69292f..b339cf2bf459884bb0824c6dc096754e5fb6642d 100755 GIT binary patch delta 1202 zcmYjQe`s4(6h8O8m-Z!Tw{~fg{;$KgfFKHN}Og8K=v9s6n+6DUHaqf4{ zchC9mci)rG3GzAN{p2n}(v!OeIkK3(_wZYdJ7OFJvQq~Y(8>Nzfa8Rt?{<$wvqlK0 z%@A)AV!Y(}ga5IeYdQl*6JF5RM^0kT3wHruW`J#9->W<%&eXO zrG1Sh{pMulq_GLdhZ_g z)PYm|)UA+;njM{z>*Ns~N}qKrU7YL(;0yr7F+=s9hRP9SM|ha90o)h6EsI}WXNj5rSlfYav;&W; z@(rm}fh8o&Wzhl+8*yNg7CrW7LThZlCirPwa&c{f%Gaj$5;&g}G^=Y< zORA#m4hhqBFzzqSprgGH4QLum&4&dD1bNe+B5p9DGbLiz zmI0nE{M-0{l=%N}wL@RtX924A?j~2LuFo!hf?tf*VydN&kOA7-AFOWwn#<3x zK^D_b)5Sw}ddFV%!C{G{T~D^y$Y1G`!|C#lo_WSbbpYx$#$GG~Xjx$F3`QB_s~FYm zjO8)rF%~f1!B|lakW4>!dW9!%Qg_(3)BDD({?b?a;L@V|<_-6Suk{zN=-q~;#V(^8 zQ}lA!LPGSTaI@}*ImTY6%i-OYXAM99vT%L=+VZd4z->Hy@vL}ZTzr9A#}qP3e;PBH x-?_@zA=rEdFq}b=ztiH_pn3I2#)JozDvWuo<9j2{Xn z9Fb%n$Rf>)Yy6W4(ZtbYi%Sz-vkx0G-I8ri3aGg0u7xZvn;*<@Yi088cd%w$a(Uk8 z_wRk)=e<);3hGJW@@N%-#8}l{j!Y$b`zI`* zXq6Cko77Zkj>TjP)M{JE3NJ?fl^MB*i@J-{lvU5`{3T(97wUNtIdAv4VKlYa4avGC z7Lzdc(Q*eJfWef;J3O@*8#Q5FgDIN%rlqc5fDQ96`;(alBR#wOwe4WczMAIC;+LOb zodLYs7rrD}XWv*a$vtiZ%AWKk{hS&G;LHHTqte1ad0{tJ_wg{~0(kt0dILMe4;EQW zdq+-+v?t)Ken>#cb?qKsIvN;nWz4B7(s?tkb^MFB$LYrbX*(>lS}ACZ-dx7*pRwAV z4QK}*SL4g$@hpo0#?okk&Mmm9N^^ml{qho95&bqsI!xeAqi6|5yHIqx0F(@hmQl0> zfTB5%TmW84T=iZvg7S1;pI8>Z0WhKhcwYgq-@!G|NMp_Z2?3OBCl}Ri(fE>hHGvUH z(5)_syMyK^C#b1uJg&;4UA)u7ff^>DrVi^CaM`P0qQx1so4`O=n6q)APIXO*(RXzQ^G?xTfcmXH4l|506F0l*JPy zn(GqAOJOif)uv5~t}#FSL-eMdxBYIyLK~jhzh+=)P2C>;|GK}qk}iT9tQA84XsRZY z)Y@Dle2(A51-Y3Jh5B0Bt-JnYu{7^M=k4)I+~FbptYs&eqmNqZNh#eLDks-yb0|c* z>8X&*dI@js{2@pZ{DshncxU2z$VRLcI3zzHjo~-wZ>=>6X^%wc=oTpq3HzTw7?#=n>`G|wONv}jI4d4ENQ|bLkwPT<8&fVGD z(=+!UtOuW^?aT?&=|R&$>g-LD4m#CqD?WFVu~t}p+7OPT$h$PxySI4ZN5+KzGDV0B Xn3YHe7f_O69}7^=i?in;QszLZAeYX6KAq1!Cf^8<5Knyn z|9@nhW9Jc|@qKI%jV3R;C$k57suaHZ|NrHoUtq%oUKj%%@p2ZB2Xc`bh|>$?Os)@f z;Q9+R9pt_~oyjKy(->i=(r5a6cJjs`876-D$@hcg_*wq{|IaVq0CW+9 t;enUmHuD5Kq%wZk?7L$%3#0zzpL-k_lO|j3m14D-^8bJR=E%M5(g53YuD$>O delta 377 zcmaE`f%(A(<_+t$8P9Isplzqjm_GTv-!I0h$*=rnEn*lM7{0ZrQ~)VY{uVzV<1Y14HY9Qi+!qAkofuAh9s87=MfEWcvVFM)t|E0pgMG85kHmdbg+uvN161 zQUG#%I^VxYzWD!t>jD1OOF+KiCBttno&R5)zwrOR2T-tMFGxo>tA8p;V~vWzJ_ZgB z29M4b6(68YFJ@eVXbqbDB0yQZ=KcTwk#UZlM}Qjp6o3E!-(90(@?zO!r9e;po>%|> zzr6bkEGF>64`}?$tw0{gE=Le&1(1`LrpGVe0<@5UVRC<<0~asU5q&n3Zw96@_D*&P z@@Mp&JRwM*iTB*(gF!MvNIIZ8C*Kc} Date: Fri, 18 Sep 2026 02:12:03 +0200 Subject: [PATCH 44/69] test: cover a worker write four times its own pipe buffer in the native suite --- ...ngineBrokerLauncherIntegrationLauncher.inc | 49 +++++++++++++++++++ src/runtime/native/fixtureWorker.c | 3 +- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc index a1f8367..2350808 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc @@ -153,6 +153,54 @@ static void worker_flood_case(void) { close(p); close(c); } +/* A worker's single write larger than the pipe buffer must complete, not kill + it. The launcher created its stdout/stderr pipe O_NONBLOCK as a whole, and + O_NONBLOCK is a property of the open file description, so the worker + inherited a non-blocking stdout: Grok 1.0.34 turns the first EAGAIN from a + headless stdout write into `stdout write failed: Resource temporarily + unavailable` and exits 1 before it ever reaches the model. Once its init + frame outgrew a pipe buffer that was every turn, at $0 apiece. + The fixture shrinks its own stdout pipe to the kernel minimum, reports the + capacity it actually got, and writes four times that in ONE write, so the + case straddles the buffer on any host instead of assuming 64 KiB — and + stays under DBL_MAX_OUTPUT, so it is the write that is bounded here, never + the turn's output. */ +static void worker_spill_case(void) { + int s = connect_socket(), p = sealed("prompt"), + c = sealed_bundle("spill-output", "mcp.Spill-2"); + struct dbl_request q = request(); + struct dbl_result r; + unsigned capacity = 0, count = 0; + size_t head, filler = 0; + char *out, *line, extra; + send_request(s, &q, p, c); + check(read_all(s, &r, sizeof(r)) && result_padding_zero(&r) && + r.status == DBL_STATUS_OK && r.stage == DBL_STAGE_OUTPUT && + r.failure_class == DBL_FAILURE_NONE && r.exit_code == 0 && + r.term_signal == 0 && r.diagnostic_length == 0 && + r.worker_pid > 0 && r.worker_uid == 2200 && + r.output_length > 0 && r.output_length <= DBL_MAX_OUTPUT, + "worker spill survived its own oversized write"); + out = calloc(1, (size_t)r.output_length + 1u); + check(out && read_all(s, out, r.output_length), "worker spill output"); + line = strchr(out, '\n'); + check(line && sscanf(out, "SPILL cap=%u count=%u", &capacity, &count) == 2 && + capacity > 0 && count == r.output_length && + count >= capacity * 4u, + "worker spill straddles the pipe buffer"); + head = (size_t)(line - out) + 1u; + while (head + filler + 10u < (size_t)r.output_length && + out[head + filler] == 's') + filler++; + check(head + filler + 10u == (size_t)r.output_length && + !memcmp(out + r.output_length - 10, "SPILL-TAIL", 10), + "worker spill bytes arrived intact"); + check(read(s, &extra, 1) == 0, "worker spill EOF"); + free(out); + close(s); + close(p); + close(c); +} static void org_cases(void) { check(!setgid(DBL_BROKER_UID) && !setuid(DBL_BROKER_UID), "drop broker uid"); client_case(); @@ -162,6 +210,7 @@ static void org_cases(void) { output_boundary_case("overflow-output", 1); worker_failure_case(); worker_flood_case(); + worker_spill_case(); int s = connect_socket(), p = sealed("prompt"), c = sealed_bundle("provider.Token-1", "mcp.Token-2"); struct dbl_request q = request(); diff --git a/src/runtime/native/fixtureWorker.c b/src/runtime/native/fixtureWorker.c index 5da8b2e..551bea5 100644 --- a/src/runtime/native/fixtureWorker.c +++ b/src/runtime/native/fixtureWorker.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include "engineBrokerLauncher.h" -int main(int argc,char**argv){unsigned char bundle[8200]={0};ssize_t size=pread(4,bundle,sizeof(bundle),0);if(size<6)return 21;size_t offset=0;unsigned provider_length=bundle[0]|(bundle[1]<<8);offset=2;if(offset+provider_length+2>(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output");if(!strcmp(provider,"sleep"))sleep(30);if(!strcmp(provider,"stderr-failure")){fprintf(stderr,"grok: session store unwritable\n");return 1;}if(!strcmp(provider,"stderr-flood")){fprintf(stderr,"HEAD-OF-ERROR grok: profile refused ");for(size_t i=0;i<4096;i++)if(fputc(0x6d,stderr)==EOF)return 32;fprintf(stderr," TAIL-OF-ECHO");fflush(stderr);return 1;}if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;id_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s tmpdir=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target,getenv("TMPDIR")?getenv("TMPDIR"):"unset");for(int i=0;i(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output"),spill=!strcmp(provider,"spill-output");if(!strcmp(provider,"sleep"))sleep(30);if(!strcmp(provider,"stderr-failure")){fprintf(stderr,"grok: session store unwritable\n");return 1;}if(!strcmp(provider,"stderr-flood")){fprintf(stderr,"HEAD-OF-ERROR grok: profile refused ");for(size_t i=0;i<4096;i++)if(fputc(0x6d,stderr)==EOF)return 32;fprintf(stderr," TAIL-OF-ECHO");fflush(stderr);return 1;}if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;iDBL_MAX_OUTPUT)return 34;char*blob=malloc(count);if(!blob)return 35;memset(blob,'s',count);int head=snprintf(blob,count,"SPILL cap=%d count=%zu\n",cap,count);if(head<=0||(size_t)head+10u>=count)return 36;blob[head]='s';memcpy(blob+count-10,"SPILL-TAIL",10);if(write(1,blob,count)!=(ssize_t)count)return 37;free(blob);return 0;}char prompt_bytes[128]={0};FILE*prompt_file=fopen("/proc/self/fd/3","r");if(!prompt_file)return 26;size_t prompt_read=fread(prompt_bytes,1,sizeof(prompt_bytes)-1,prompt_file);fclose(prompt_file);if(!prompt_read)return 27;char fds[128]={0};size_t fds_used=0;DIR*fd_dir=opendir("/proc/self/fd");if(!fd_dir)return 29;struct dirent*fd_entry;int fd_seen[64]={0};while((fd_entry=readdir(fd_dir))){if(fd_entry->d_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s tmpdir=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target,getenv("TMPDIR")?getenv("TMPDIR"):"unset");for(int i=0;i Date: Fri, 18 Sep 2026 02:48:08 +0200 Subject: [PATCH 45/69] test: seal the spend the proxy measured when a worker's output never reaches the host --- src/runtime/AGENTS.md | 16 +++++++ src/runtime/grokEngineBrokerUsage.test.ts | 55 ++++++++++++++++++++++- src/runtime/native/AGENTS.md | 20 +++++++++ 3 files changed, 90 insertions(+), 1 deletion(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index e87dc52..2f33635 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -117,6 +117,22 @@ so it replays with the sealed record and reaches the operator through `engineBrokerControlClient.ts`'s failure message. Nothing new is written to disk: the reason travels inside the response the broker already seals. +A turn whose worker said nothing still records what it spent. The launcher can +refuse to publish a worker's output (`DBL_MAX_OUTPUT`, `native/AGENTS.md`) and +the native transport can fail outright, and in both cases `result.text` never +exists, so there are no stream frames to read usage from. `streamOrMeterUsage` +then falls to the proxy's own per-request measurements — the meter admitted and +settled every forwarded request, so the broker knows the spend even when the +worker never speaks — and `finishBrokerTurnWithUsage` seals and appends it with +`outcome: "failed"`. That is the whole of the guarantee and it is pinned by +"a worker whose work succeeded but whose output crossed the launcher bound" +(`grokEngineBrokerUsage.test.ts`), which builds the launcher's own +output-limit frame at the ABI offsets and decodes it with the shipped client. +Deleting the meter fallback, or refusing that frame shape in +`decodeNativeBrokerResult`, both turn it red. The one window that stays open is +the documented one: a crash before the turn record's rename, which the next +boot seals `usage: null`. + Evaluator inference grants (`grokInferenceGrants.ts`) let Paideia judges and the DSPy optimizer — uid 2000, the trusted evaluator side — spend the broker's Grok credential without holding it. `request_inference_grant {model, diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index e316a57..8f0f50d 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -7,7 +7,7 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import test from "node:test"; -import type { NativeBrokerTurn, NativeBrokerTurnResult } from "./engineBrokerNativeClient.js"; +import { decodeNativeBrokerResult, ENGINE_BROKER_NATIVE_RESULT_BYTES, type NativeBrokerTurn, type NativeBrokerTurnResult } from "./engineBrokerNativeClient.js"; import type { EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; import { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; @@ -291,3 +291,56 @@ test("a response that called nothing records an empty list, and one that cannot }, undefined, () => 1, () => "bad gateway"); }); + +/** + * The exact 128-byte frame `supervise()` emits when a worker crosses + * `DBL_MAX_OUTPUT`: it stops reading, SIGKILLs the process group, and publishes + * `output_length = 0` with `DBL_STATUS_OUTPUT_FAILED`. Built here at the wire + * offsets the header's `_Static_assert`s pin, so the test drives the real + * decoder rather than a hand-made exception. + */ +function outputLimitFrame(turnId: string): Buffer { + const frame = Buffer.alloc(ENGINE_BROKER_NATIVE_RESULT_BYTES); + frame.writeUInt32LE(2, 0); frame.writeUInt32LE(3, 4); frame.writeUInt32LE(2_200, 8); frame.writeUInt32LE(0, 12); + frame.writeInt32LE(4_242, 16); frame.writeInt32LE(0, 20); frame.writeInt32LE(9, 24); + frame.writeBigUInt64LE(99n, 32); frame.write(turnId, 40, "utf8"); + frame.writeUInt32LE(7, 108); frame.writeUInt32LE(7, 112); frame.writeUInt32LE(0, 116); frame.writeUInt32LE(0, 120); + return frame; +} + +test("a worker whose work succeeded but whose output crossed the launcher bound still seals the spend the proxy measured", async () => { + await withBroker(async ({ turn, usageRows, requestRows, upstreamCalls }) => { + // The worker does its real work through the real proxy — two admitted, + // metered upstream requests — and only then loses its whole output: the + // launcher refused to publish it and the turn's text never exists. The + // frame is decoded by the shipped client, so the failure reaches the turn + // exactly as the native transport delivers it. + const worker: Worker = async (send) => { + assert.equal(await send(), 200); + assert.equal(await send(), 200); + throw decodeNativeBrokerResult(outputLimitFrame(turnIdFor("foreman", "wake-output-limit")), turnIdFor("foreman", "wake-output-limit"), []) as never; + }; + await assert.rejects(turn("wake-output-limit", worker), (error: unknown) => { + assert.ok(error instanceof EngineBrokerTurnFailure); + // No limit tripped and the credential is live: this is the worker's + // transport failing, not the turn being refused. + assert.equal(error.code, "engine_failed"); + assert.equal(error.diagnostic?.failureClass, "output_limit"); + // Mutation guard: the turn has no stream to read usage from, so this can + // only come from the proxy's own per-request measurements. Falling back + // to `null` here would report a fabricated zero for real spend. + assert.deepEqual(error.accounting, { outcome: "failed", usage: { input: 5_136, cacheRead: 256, cacheWrite: 0, output: 158, total: 5_550 }, model: "grok-4.6", requests: 2, limitReason: "none" }); + return true; + }); + assert.equal(upstreamCalls(), 2); + const [row, extra] = await usageRows(); + assert.equal(extra, undefined); + assert.deepEqual([row?.outcome, row?.reason, row?.total, row?.calls, row?.complete], ["failed", "unknown", 5_550, 2, false]); + assert.notEqual(row?.total, 0, "a zero row would be byte-identical to a measured zero"); + assert.deepEqual((await requestRows()).map((value) => value.request), [0, 1], "every request the proxy answered keeps its own row"); + // The sealed record is the durable truth: a replay returns that spend and never meters again. + await assert.rejects(turn("wake-output-limit", twoRequests), (error: unknown) => + error instanceof EngineBrokerTurnFailure && error.accounting?.usage?.total === 5_550); + assert.equal((await usageRows()).length, 1, "the replayed failure is not metered again"); + }); +}); diff --git a/src/runtime/native/AGENTS.md b/src/runtime/native/AGENTS.md index e7930de..736fec4 100644 --- a/src/runtime/native/AGENTS.md +++ b/src/runtime/native/AGENTS.md @@ -67,6 +67,26 @@ reports the capacity it actually got, and writes four times that in one `write`, so it straddles the buffer on any host without assuming 64 KiB while staying under `DBL_MAX_OUTPUT`. +**Crossing `DBL_MAX_OUTPUT` is a reported status, not a lost turn.** This is +worth stating because it has been guessed at twice: a trip sets +`output_limited`, stops reading, `SIGKILL`s the worker's process group, reaps +it, and then — `disconnected` is still 0, so the branch at the end of +`supervise` runs — writes the complete 128-byte result frame with +`DBL_STATUS_OUTPUT_FAILED`, `DBL_STAGE_OUTPUT`, `DBL_FAILURE_OUTPUT_LIMIT` and +`output_length = 0`. `closed_result` admits exactly that shape, the client +relays it, and `decodeNativeBrokerResult` raises a named +`NativeBrokerTurnFailure`. So a trip costs the turn its *text* and nothing +else: the broker still seals the turn and still meters the spend the proxy +measured. A lost terminal frame, an unnamed transport failure or an unmetered +turn therefore cannot be explained by this bound, and the only branch that +sends nothing at all is a client that already disconnected. + +The bound is the whole turn's stdout, not one frame. A live single-tool-call +brokered turn already emitted 26,486 bytes, 23,320 of them one tool-result +frame (`.runtime/grok-p1b/worker-a2-output.jsonl`), against a `--max-turns` of +48 and a 16 KiB tool-result spill bound, so 64 KiB is reachable by an ordinary +working turn rather than only by a runaway one. + The result frame's last word is `diagnostic_length`, not padding: on `DBL_STATUS_WORKER_FAILED` the supervisor keeps the last `DBL_MAX_DIAGNOSTIC` bytes of the worker's merged stdout/stderr and sends them after the fixed From 734056b1226c42ff08eb3a784750bcbbe7dc2a5b Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 02:58:21 +0200 Subject: [PATCH 46/69] fix: fail a brokered worker's model request fast instead of retrying a refusal blindly --- src/contracts/runtimeContractManifest.ts | 6 +++--- src/runtime/AGENTS.md | 16 ++++++++++++++++ .../receipt.extra-canary.json | 2 +- .../grok-slot-preflight/receipt.legacy-v1.json | 2 +- .../receipt.missing-canary.json | 2 +- .../receipt.readable-canary.json | 2 +- .../receipt.unknown-member.json | 2 +- .../grok-slot-preflight/receipt.valid.v2.json | 2 +- src/runtime/grokBrokerWorkerConfig.test.ts | 15 +++++++++++++++ src/runtime/grokBrokerWorkerConfig.ts | 17 ++++++++++++++++- 10 files changed, 56 insertions(+), 10 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 3cf1242..03af646 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -53,9 +53,9 @@ export const GROK_ENGINE_BROKER = { systemPromptSha256: "2c31c0085a54a4efbf9c0cf0b8124c56e47f38691b7f0c7fa233a74abaa8ddf8", // sha256 of `renderGrokBrokerWorkerConfig({ model, reasoningEffort })`, the only accepted config.toml bytes. configSha256: { - "grok-4.6": { low: "eed6a451150a72b2cb528b30c23b3d51c7d3bc38c67a8985d4dcdf956ff214d3", medium: "8850502dbebf8918c5161c63efcc4ccf18719488300f4cec1deceb2c112b451f", high: "3ce44ace503362326b47149b528b942ce638fe146313d62502f248acf9c7333d" }, - "grok-4.5": { low: "7aa13e90b9bc08d1a018f48b7a84de1dab41db586627ee2d5a25f69011ba7e25", medium: "218ba37e57a6f02fa36b265b4e154e68e30bd2d4794feb130cc226fdda7732a9", high: "0bb4ad8bfa5062169b28422d1d534b45420d4e46b1e546bda1c578eb34303646" }, - "grok-build": { low: "83ac7202442286a65c359cc596b0b8db7bc4529ee70e98224f6cd6f66deb6878", medium: "0146313f28739888eb4e861f1bfb285f7ee4e0a9164669256ebdf6492a2790ce", high: "bbe72aaf70c417dc7007823a7e9e1a7d1fa8d57e50bde6f24036083b32bcc859" } + "grok-4.6": { low: "ab58499ac32678097c146479896f2b8a8e2b0e39aea22dc0a60b6227e370538e", medium: "df1a5cc84346e7f6bf6090492fbd19faaefb42953e3bb2e8c6cbc0572242403f", high: "65b0212564fb74042b1503d293fb8d3620276033264c0efade2a539ca09218e3" }, + "grok-4.5": { low: "8247127c3625ff7c5d8d527a53596b89ec6557a821ac46cfd00bd122b90daff6", medium: "59288cee61297bb8c002097061a48f77b09d310754187a253ee089f7172a9155", high: "c63c3387ce92d94ec3f690abfe98942afcd7c9e17ff84816bbe751f340ab251f" }, + "grok-build": { low: "fb343f2809903f26d21681470943235031f946e99085542fd89555eb7782cbb5", medium: "8a587ef75c90eab70d19b24583e60051d6fba9d90c558839fdbb15588b4cc656", high: "a23724e00d670caee185ba7690d2daa868173e53905cf446f5329666f01ab4e3" } }, // Worker `GROK_HOME` layout the broker attests before every turn. The home and // its `sessions/` directory are root-owned, worker-group writable and sticky so diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 2f33635..682556b 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -173,6 +173,22 @@ must be run with `--model daimon-inference-grok`. The inference ledger directory must be provisioned setgid to the organization group (e.g. `2100:2000 2750`) for uid 2000 to read rows the broker creates `0640`. +Every model block the worker can reach carries `max_retries = 0`. Grok 1.0.34's +default retries a refused or failed request with backoff **past 45 s**, blindly: +one live turn emitted the same refusal fifteen times over five minutes, spent +$0 and died with no account of why. The session-title sink and the evaluator +client (`grokInferenceClientConfig.ts`) always pinned it; the worker's own +model — the single path that spends money — was left on the default, so the one +place a stall costs a wake was the only one that could idle for minutes after +its work was done, silently, because a retried request that never reaches +upstream writes no ledger row and prints no proxy line. Daimon owns the retry +decision here because the thing being retried is Daimon's own proxy: a +genuinely transient fault is already answered 503 and is the broker's to +retry, and everything else is a refusal that repeating cannot fix. The worker +fails fast instead and the turn reaches the host with a status. These bytes are +manifest-pinned per model and effort, so changing them rotates +`GROK_ENGINE_BROKER.worker.configSha256` and every deployment must re-vendor. + `grokBrokerProjection.ts` is the public, I/O-free projection of one brokered Grok agent's slot (`noopolis.daimon.grok-broker-projection.v1`): Daimon's own deny collectors plus the caller's evaluator paths, profile/config/prompt diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json index a1f00ca..b28307c 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.extra-canary.json @@ -4,7 +4,7 @@ "worker_uid": 2200, "generation": 3, "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", - "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", + "projection_sha256": "0614586f310b6d4fc50eba85d5a493e03a01d91af8c629989b4b1006dcc3f3a5", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", "sandbox_runtime": "bubblewrap", diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.legacy-v1.json b/src/runtime/fixtures/grok-slot-preflight/receipt.legacy-v1.json index 3ccf85c..fd1e2de 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.legacy-v1.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.legacy-v1.json @@ -2,7 +2,7 @@ "version": "noopolis.daimon.grok-slot-preflight.v1", "slot": 0, "worker_uid": 2200, - "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", + "projection_sha256": "0614586f310b6d4fc50eba85d5a493e03a01d91af8c629989b4b1006dcc3f3a5", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", "sandbox_runtime": "bubblewrap", diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json index 0b57369..03a3c9d 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.missing-canary.json @@ -4,7 +4,7 @@ "worker_uid": 2200, "generation": 3, "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", - "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", + "projection_sha256": "0614586f310b6d4fc50eba85d5a493e03a01d91af8c629989b4b1006dcc3f3a5", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", "sandbox_runtime": "bubblewrap", diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json b/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json index 19cb96f..c2f4908 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.readable-canary.json @@ -4,7 +4,7 @@ "worker_uid": 2200, "generation": 3, "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", - "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", + "projection_sha256": "0614586f310b6d4fc50eba85d5a493e03a01d91af8c629989b4b1006dcc3f3a5", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", "sandbox_runtime": "bubblewrap", diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json b/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json index 91774b2..d5978c9 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.unknown-member.json @@ -4,7 +4,7 @@ "worker_uid": 2200, "generation": 3, "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", - "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", + "projection_sha256": "0614586f310b6d4fc50eba85d5a493e03a01d91af8c629989b4b1006dcc3f3a5", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", "sandbox_runtime": "bubblewrap", diff --git a/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v2.json b/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v2.json index 94542eb..bf3e344 100644 --- a/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v2.json +++ b/src/runtime/fixtures/grok-slot-preflight/receipt.valid.v2.json @@ -4,7 +4,7 @@ "worker_uid": 2200, "generation": 3, "nonce": "5f0e2a1c9b8d7e6f5a4b3c2d1e0f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f", - "projection_sha256": "9e8bbd4e9337df40b90d5a382ece6c3745cb5e11038d10ea17b5032408bf30e4", + "projection_sha256": "0614586f310b6d4fc50eba85d5a493e03a01d91af8c629989b4b1006dcc3f3a5", "sandbox_profile_sha256": "2a1dcab471092eab77565838b17a5cf138489e3daa1349612ebcedcdb69e23c4", "seccomp_profile_sha256": "7777777777777777777777777777777777777777777777777777777777777777", "sandbox_runtime": "bubblewrap", diff --git a/src/runtime/grokBrokerWorkerConfig.test.ts b/src/runtime/grokBrokerWorkerConfig.test.ts index 091418a..0642b89 100644 --- a/src/runtime/grokBrokerWorkerConfig.test.ts +++ b/src/runtime/grokBrokerWorkerConfig.test.ts @@ -35,6 +35,21 @@ test("worker config disables every bundled 1.0.34 skill, workflows, and the per- assert.match(section(config, "[cli]"), /auto_update = false\nuse_leader = false/u); }); +test("every model the worker can reach fails fast rather than retrying a refusal blindly", () => { + // Grok 1.0.34's default retries a refused request with backoff past 45 s. + // The sink and the evaluator client always pinned this; the worker's own + // model — the one path that spends money — did not, so a refusal there could + // stall a turn for minutes after its work was done with nothing logged. + for (const model of GROK_BROKER_MODELS) { + for (const reasoningEffort of GROK_BROKER_REASONING_EFFORTS) { + const config = renderGrokBrokerWorkerConfig({ model, reasoningEffort }); + for (const block of ["[model.daimon-broker-grok]", "[model.daimon-session-title-disabled]"]) { + assert.match(section(config, block), /\nmax_retries = 0\n/u, `${block} ${model}/${reasoningEffort}`); + } + } + } +}); + test("the declared model and effort reach the worker's only model as its sole allowed effort", () => { const config = renderGrokBrokerWorkerConfig({ model: "grok-build", reasoningEffort: "medium" }); assert.match(section(config, "[model.daimon-broker-grok]"), /\nmodel = "grok-build"\n/u); diff --git a/src/runtime/grokBrokerWorkerConfig.ts b/src/runtime/grokBrokerWorkerConfig.ts index 5a2f88c..fde8952 100644 --- a/src/runtime/grokBrokerWorkerConfig.ts +++ b/src/runtime/grokBrokerWorkerConfig.ts @@ -79,6 +79,21 @@ export const renderGrokLeanBaseConfig = (): string => [ "[workflows]", "enabled = false", "" ].join("\n"); +/** + * `max_retries = 0` on the worker's own model, for the same reason its two + * siblings already carry it (the session-title sink above, + * `grokInferenceClientConfig.ts` for the evaluator): with Grok 1.0.34's + * default, a refused or failed request is retried with backoff **past 45 s** + * instead of failing in ~0.35 s, and the retries are blind — one live turn + * emitted the same refusal fifteen times over five minutes, spent $0, and died + * with no account of why. Only this model block was left on the default, so + * the one request path that spends money was also the only one that could + * stall a turn for minutes after its work was done. Daimon owns the retry + * decision here because the proxy is the thing being retried: a genuinely + * transient fault is already answered 503 and is the broker's to retry, and + * anything else is a refusal that repeating cannot fix. The worker instead + * fails fast and the turn reaches the host with a status. + */ /** * The only source of broker worker `config.toml` bytes. * @@ -107,7 +122,7 @@ export function renderGrokBrokerWorkerConfigWith(policy: GrokBrokerModelPolicy, "[models]", `default = "${GROK_BROKER_WORKER_MODEL_ID}"`, `default_reasoning_effort = "${declared.reasoningEffort}"`, `session_summary = "${GROK_SESSION_TITLE_SINK_MODEL_ID}"`, "", ...renderSessionTitleSink(proxyPort), `[model.${GROK_BROKER_WORKER_MODEL_ID}]`, `model = "${declared.model}"`, `base_url = "http://127.0.0.1:${proxyPort}/v1"`, `env_key = "${GROK_BROKER_PROVIDER_CAPABILITY_ENV}"`, - 'api_backend = "chat_completions"', "context_window = 131072", "supports_backend_search = false", "", + 'api_backend = "chat_completions"', "context_window = 131072", "supports_backend_search = false", "max_retries = 0", "", `[[model.${GROK_BROKER_WORKER_MODEL_ID}.reasoning_efforts]]`, `value = "${declared.reasoningEffort}"`, `label = "${label}"`, "default = true", "", "[mcp_servers.daimon]", `url = "${mcpUrl}"`, 'bearer_token_env_var = "DAIMON_MCP_CAPABILITY"', "" ].join("\n"); From d7e36db3323bbbf4838de9c3d97885e62b1f6ab2 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 02:58:30 +0200 Subject: [PATCH 47/69] fix: trip the launcher's total-output bound from the post-exit drain too --- src/contracts/runtimeContractManifest.ts | 6 ++--- src/runtime/native/AGENTS.md | 25 +++++++++++++++++- .../artifacts/daimon-engine-broker-arm64 | Bin 67456 -> 67456 bytes ...daimon-engine-broker-arm64.provenance.json | 2 +- .../native/artifacts/daimon-engine-broker-x64 | Bin 55392 -> 55392 bytes .../daimon-engine-broker-x64.provenance.json | 2 +- .../native/engineBrokerLauncherServer.inc | 25 +++++++++++++++--- 7 files changed, 50 insertions(+), 10 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 03af646..3df8e89 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -133,9 +133,9 @@ export const GROK_ENGINE_BROKER = { projectionVersion: "noopolis.daimon.grok-broker-projection.v1", slotPreflightVersion: "noopolis.daimon.grok-slot-preflight.v2", artifacts: { - sourceSha256: "2d36898f02793a89a4a58601fa77c0d8716cc028588daad6daf76ca44d28446f", - x64Sha256: "5f92d83fe8159d1ac371f62d5382c0b52ce7fd167e3855460de66187a4c505a8", - arm64Sha256: "8bb4f6b1053abe27001219ab640601289b39c8ff8c86831cd7d8782f0d9dab8c" + sourceSha256: "d8c9640a2d0084f584721d0d4afdc524af7434e9461c1fc2f8adcdff6454ba6d", + x64Sha256: "4059ec576065130e857cc937b03b97a0c45fffab7d790fb153fcb170bfd0f310", + arm64Sha256: "eb0e2975cf3204bb80d25edc1fec00dbd623a95466176f3f36e7e83178deb8d2" } } as const; export const AGY_SUBSCRIPTION_REALM = { diff --git a/src/runtime/native/AGENTS.md b/src/runtime/native/AGENTS.md index 736fec4..86d0c78 100644 --- a/src/runtime/native/AGENTS.md +++ b/src/runtime/native/AGENTS.md @@ -81,11 +81,34 @@ measured. A lost terminal frame, an unnamed transport failure or an unmetered turn therefore cannot be explained by this bound, and the only branch that sends nothing at all is a client that already disconnected. +**Both readers of that buffer trip the same bound**, through +`output_limit_crossed`. The poll loop always did; the post-exit drain did not, +so a worker that exited with more than `DBL_MAX_OUTPUT` still in the pipe left +`used` at the buffer's last byte with `output_limited` clear, and the turn was +published `DBL_STATUS_OK` with `output_length = DBL_MAX_OUTPUT + 1` — which +`closed_result` refuses, so the client replaced it with a fabricated +`prelaunch_failed`/`protocol` frame carrying no pid and no start ticks. That +frame says the worker never ran, about a turn that ran and whose work may have +succeeded, which is the one class of lie this boundary must never tell. +The window is real but narrow: `poll` is level-triggered, so the loop sees any +buffered byte, and the drain can only inherit data written in the gap between +`poll()` returning and `waitpid()` reaping. It is therefore **not +reproducible on demand in this suite** — the fix is by construction, and the +adversarial cases that do cross the bound (`output_boundary_case`, +`worker_flood_case`, `worker_spill_case`) only prove it did not regress. Do not +add a test that claims to cover it by feeding the bound through the poll loop: +that routes around the defect. + The bound is the whole turn's stdout, not one frame. A live single-tool-call brokered turn already emitted 26,486 bytes, 23,320 of them one tool-result frame (`.runtime/grok-p1b/worker-a2-output.jsonl`), against a `--max-turns` of 48 and a 16 KiB tool-result spill bound, so 64 KiB is reachable by an ordinary -working turn rather than only by a runaway one. +working turn rather than only by a runaway one. **Known limit, deliberately not +raised yet:** the same day changed the pipe's blocking mode, and raising the +bound alongside it would mix two variables in the next live run; the measured +ceiling is 26 KB against 64 KiB, so it is not the thing in the way. Revisit +once a trial scores, and decide the number on the spread of real turns rather +than on headroom-by-guess. The result frame's last word is `diagnostic_length`, not padding: on `DBL_STATUS_WORKER_FAILED` the supervisor keeps the last `DBL_MAX_DIAGNOSTIC` diff --git a/src/runtime/native/artifacts/daimon-engine-broker-arm64 b/src/runtime/native/artifacts/daimon-engine-broker-arm64 index b339cf2bf459884bb0824c6dc096754e5fb6642d..2dbe88d27e03d69ea06aa08648efa8f8bc54e551 100755 GIT binary patch delta 3240 zcmYLM4^)&@7Ju*i2ABc)=kQl#X9h%K(9m?`@5sj>f{>fxxY@dA8=_4&a+6RLF&Wa` za&b3a?-a-FxQ%UYJCmM82g_+|gEH%^wjo+4+iC~2%#)p!qbLOU_V)&?@0>focmI6% z-FNSO_dDZ$M(2J;*V%8_u!Dr24eb2e@t)07%n#`I(j&8I0be)!+xamps2Y7XA-YI7 zS!^MsU-Fh<3|j!13$hRoxy2mIo6`4C1D{N1G}iyjPtsHx%U`vu$FDKt5gO;;nUO)E zKae>_XYv>3bWs~$l+{NE{Uce85vloXAzY5C#5(kp78q0$d}cyAi6@`8&2ORw{%iAp zLzOs(zdp~eGZZiceU*?xZd&}mT-BFM+_hs3I`5dOSci6zyT+Tsyx(UkY}}}@;8qd{ z!ks!Mx@Pl2TXw{lIx{Dz&f@EBIn=|M&1kellKc@xwSGK<*rcyPHI2V+t6rN(oHyX^ z9mRO6T`_h9)l};uxG@njd^mwPn-tO^&D$K>@a0>=DRd+@i0EkwdEE4F_(YwaI7vy> zS&~cwb+8LYk)t$%1PEWVB->z$B>`hA39RA2U2>28jZx7R%qwg0C}c?{Y;W%;kK-W5 zQx}>@U<@{LZx{A8_VPbnXwm|5Z!eF}H?|RY7P+)V?i_qf3s|VTb|v1}FGTW)s#?c^ z`S5Jkk$5t@jKfpoNxDk3Ci>*&XZYd5w49}QbxqydC9py!Q6 zX^9KrUAv3ehGJyS6>AVQhaV_18lvQJ77`odZxtn{o4~mi5SsWA+7v!q zl$`n{{L2)%@<#Jz^hAyba}nFjEyc!k*$?B;?o!jR$1JO&xNG~|?%E5Fk-%wwFD%}N z#WGmP85CTv1#VkhY#@PN-d=3F_fuGCIivYO_^89)=fL;gA^BO>cJTdDAL3K2GESN6 z-fXE)v6g}VKfskxVuX1vy+r z%;GL$*(5y|BGEq>U3kL^8NJb4jCy)Ez=Jjfd` zi1t&cd!YGyv?+KO#*|sr#>`>lSWuummQ?@&Ny~Hms-J(XZRlKe=5m?PzN-J_^Fn#+BHl3`_OXg^H(iP1v zmMiSSE;W>&F!E%r!(-={O4I3oc$oH-p-}FR$n*JgTIq_|>2t`Ym@&Ps!`dL%h*CC3 z__VwoYZdFs)_;1b**!3+5_T<$j9o*F$NcGKnUo&qWsVel9QiuON~&FwdzdHSD|&Pq;6HH&&F=tNcjC zqKcO=3(S*Z@gR1`UG%9qfM-J3Uc~ozs9et|=?vX|NYITi^{6TzZ ze#Uc_KR{#np5?`fr=TM`3|$=lc7r_3&o56){0B5<1sbGJA?>xY7+-QjWqINrXvP|r zdB>Y7yr12|$VW1F^Sa7Kw1z)dc^^H-zpGqN*YPz~m2?l^UzJDg{KKl<@dwV&!tw47IL z%%xxZ8#j)Jx4B*s;tB@GroRiZ0|Ui;RET-F<8Gh^>;~dXLaqQe05!i5dx0*X2e=7% zTxHn@9)NWk5XJ0?U4a4&E((EOSZF9SQ@6e0!lq8qqCC2eHRTSBab;A!Baz&qX+ zq6et;3Na!16GHp~X?yy#5ZyrcheDhK&N(Z@CIsGl4uN3`#(j(f0A0XNVDBeFTm+r~ zjsp9EGjKrnr$Q6}`$vUX6~Q0bVzpoC7Gg2x7b%PQNgQAvsNe|VlQKL~rh%N0l>p^q z201}H_?a!4^AACTm_knie)ocoLnhxOWifI4l|NIt!^@yq)Z_c0(@h6(I zmD5N0Uz?ZEIQ~U*9^K5Nw>HsDynSmj-NyHA%}zWK5aJ=}3C}XnTI4Z-f4sGlKFZU# zRfd%ym*4PTZ!4g`;xBE>peOiy+g8(lp4yU0U*`6f7w92=zNHn`Yqq!2Rs76$W8&U_ z3vqYIBO$ARJCMo`_}KQ`=r#)SDI+|zb**N94r)!^%<{SE_+5WEclKtVoMptSW*yCHKjE}#5e;CbH F{tvGR=!5_O delta 3204 zcmYjU4NzRw6~6bqh20GhS(oKky1O7DEQutB zM-p`$Y~}X0WujBNj5@m8I7T;UwlxYyZ5$Jgo5UuqZ5Kj~9e+wDApr@w{VovOH*@y8 z=jWY!?m6e4H>*Es(4RE)pD-`&A#=}CcJ?icuVbG17Q;dMN+vDkcir^G@+1~hts%D% z10NFISA<|ffDXt-cC!y z7ng6N%7iD}S`;>zOBsTmAf$}ji~m!oxW@L7Tc_I$!EdYzo6c3(OnY2#MpZ*M5xqIq zNg^@0Q>Vp18ZUR}8>iG83u0;p-|8-)I%jUHl`bNslZxsZN+j+yyfdn1^4Hz#yp(t^ z!?l5sly-&m`D?WcpA+JJTd{u7t62MDYPzckZta9zNEE^wL*z7_JlJa@UQ*R~(w9ae zS4^b1MTU_;UTrawh>ve6$v69NAd&WN5^?j#OE%>FX-W*liafgl3Mt8j?RDSsShd9Z z!MS!4`G(Jyr1{^!B)q~vZkSY6mk*YuARoMnLb}U;B>rmbel6mpdh_k@_>2%~(ryOm zFc1s5$-@!p_Fr5P-j*y9>AWJ`pAu5+kGCqGu1A6c_s z^~|=T5_6gBlwFq@s2%(ij5f;2c<2(2>rFgRm>TP=Nqc+)XeX%w&nfhGLE?4u>D?zU@j!E99YRa zU#puj8?r*Zd3y9i9<0tbnwea`%a2!Ya39A7lNV03x+7!oAaB4Z+E1Z+^x<#Nrr>=T zM-Ek+cEpjPm`Yt|_#L-3Sss$sXkghg6wSAmzjRxr^C|cP=t^$?8AvNM!H%64`j2!p`kc=kgOrb~o1s zs<=_hrssH`_L$it_ebP4d`PQVo3?Nc*%T=Y>lUmna*Zfub1Ytv_hOxZ^=$W>LF&+> zvnpX16Ug*M#CSMd?a8Hdn77oXO=VCbP(BkDXy0$;k8?fXv$Q{ z&N*4C>qRN^Lk9kNtVKLoiOhLX1J@v5UT3HK`1-o7igA*^R`;td8F1%Iz^8l(WloH2 z8~E5F@GgZpF%2zJFZo*xb9|E2o55rI-(K@^CkaF^;ii?zxiBYB#f{k;E#`x(>k|^$ z>`cN-cWdcallg_!R{AoJtFNQm_}cm!^P`vG3HQasS$z$i;vd(qst;inIA%rhD0atP z3?<+Ifvdv(dwliA>P(E1{<+&vY2ZPkB{AvwP541sUA`GnicfdwhkWgtyJ;#9uBo)0 zgpL?6baD9W4YG|-ugS6<4g?zI?3s^#_Eqt4IlUK8KFw#Je;FID9* z!xp{CJi7LmbR%zBcekbNqMW~}n7=8Z95m@T|9D*`J@;+e!Ibr03W%pkbWJ0<-VDCf8YfnzQoWt@Ujp+ z7$&|~g;<9BJq*<3JqO||LY9SvxEH7cp8y7c0pJ1PA*nwogq#b*z!cp3jYC3I0gnMY zfWFs+cn)~t4I$FytN^Z1iJ#naScuIK90mRic<)<690cmaLR;EK~Y08j_^17n{EF%BFDP5~!?i8x^3j1XnO6H`KLH1g06 zSM{=2geb=BA|B&+k^##=U68GiGCWeIfqo5H6;M77kQej@x9rSa{v|YsY3@nE?-QUY z$mE;EvpD&8$}?2%@EmA@x9lucieKeV?aZOi@xwdwmBcXrY-b7To!wbP5AyY$?eqX2 z=}e=C_}R{Uo8gZ_+#@~VSq<8ZJbJmUtAXz4Kj>p?D$@?cjPy@!u=<2|U%mz>T$ z5tr=3pXMSxG(9~jf|SduvyYagCsmb;f*`%`lCv8`sb(z>&ixJXQeB5DvUkuW=L38n zphEhT$d9F(2nz)IR9}T`gHP`02R-Y1#*~?z6(eebD_+vQ|6|OP7GALdzEik}Z>1dH z7Ks;q1>&pelWInW2|{i)6A-gh_3TibG%}#H@EqLaC%Dqcm0Z%LkrTdjS@ln!tS0}c zzjl%_wI(R7{|F(d;s>aVT78AvSpK={v#zNWpnQHVxtdG*b4hl&52=&go)cA4&HV^N zdaqL<*>UR5A&7+kDP7s&@SG=bVU#M*AAXnWZmgL`vl7scKLo22W2F(o7Log)8Kqc;2ArV9B#mP7x~N z)twOtfd6re$k?dYsj0jIH4_k*;#n0ZEE}eHs#e7dl9#s}L3A$vtpaa1l%+NEd652c zxj-Yov}MAaGpPE9A6m-G-LIcN&e&Z4ivolUCNQnE2LeqH&@k(B{NZsf@~CW6G0t|C zbU1^05K1net4g+MzBpMSUCPX$^vr44-o<_LVYO?e`?Tca8tP`tCHvGvy?OrhNDU1Z zKha<*UQPZ`6_2T6w@RX9JrQM9eNnQ8+QQRM8ipxe^@k60kzZ8OHP4@JJNstuvsW0? z$OX)si}blk)%jeq_lBu!KoX&;>wSl3?Xxy4GpiCFDr0XJOgMwNUD@S)B(IGh1~nX0 z7KMSB)VvxDjST7uKcO@=O!5{&hjTP?Q6CUYQvE1wa?+ zH@c|5L-#1Mcstb_1Sdl-6J0FDmMf-LeNE6^RH}7mJ9=t2*E{u{ta#ie&!p) zu;V%d!wdXN6So;I45mW%0`K^S{`2nf<+qv{t$h%;5^>qvu*L*EE`+K;+xUPX5}m;k z!d`nD2YEUOL;o=Zj^!Y~(HuMy0wYbMwl*s_+D>P=#*B&y@dkjkbJL3Has{n$O`Y~3 zxQb`5=$?6X*>TlxEnSRa>_025vNpVc(e>|;OW?Uom7d%O?m~LVHB$UD?R6zgehhp- z4?%&manG$4c*o>i;*eCno5MQbztU89Tve(mza%7o0gT0_;GS#dpTJp4_R2v-$+3ex z4gq?(elJ!#j1#Y%ig=*wf^2Se=SCi#RNl!sX?WKN&tgW%&_U>M8=ZARzM-6HDyN!` zKj2!b35^e_^n_Git~1>gfb9&z#aY+acGw zfga$$5OD2lg3E4FeO?1pJ!t~I{1DTp(99h54{t)1-w)hRJ&o4oOi9ce;P=_I+{_0+ z-}$9!D3^5V>*(>E<>O8-0^D?z?>T(%RnluL_I{vk4qOPmj#jEgaqr^nJS>fH$u{1e zqp0+D+M|vUU#C}9_vnd_nI(AmIkP9c1gp8fTDmZI%h07?m}%;d(TKd+*LSX=(lqm(N&L|jnjDXx6@(T zMAv9+NYY*Nb){9GcEs?STElQi!a?o0(^% zp7tkkF}-Q-TVfsEIX6M9q(|nCyZIh-3wcNLPldllzJ#0ER?VYNr!n~>#Wjge0`K=p09HGh`4m42U}EoRaw^Gb@_ z&1K>BT;^u*V~&r~orld$`J@3_!(bIldnm3-hW_w%4=`*0FES%te6uJJ#^;H2>5J)! zdFA3PIuT%~?x#Hk zia3WR&Cea<#NzJVebJDy>t{d3>1CXFVZZ;BuAQGFw$g+1mxviObHO~zl~3r#1ycb9 z7d$og>qi8kE2lgGUZt2JWzrJsmq4VL9J`b4DWg(hQtW?B5yy3rUDR8+L>v`3SC}bU zjvffyQ9O5uI6vThU~}GW@1_YtO|T*w-;u*G5&Dg$6r)a(gymRNT&Q|Z+q$gP5JWEo zeG@I-Izv>cf9sl29W(gF?bo0B7h||Rr?yLpnb?w3w$YS^jpCL-TZ3iD_NM8!Dm!89 z8;ocsWh{kn4yuDXKre$1Y=Iqo zzJz+lGC>`n{BxEHx`?0OV9X0z3%Y5DP$dlDr4c~M-x&*n20=dqO??xFLF+*;gF4<~ z%#N=0>|*RbP)93c<^22+V@;f5pPU1=_cKBd@c;$|uynHE{U%~Y%p%setN?Ekut=%N4s delta 3867 zcmZ8j4OCQB9)EY3iNRRjxPW4!4DR?rSvp58+yLj%3_j=ysBliTZcDmLQKQl>D0SQh zc4j=jr7`WMnPqKd>298l8czuaVIUISEYYHMD+OD(-sA_UMX2fB-+gaL>zy<6fA{zQ z{^NieBNk;k@b!T&Pm&oMIqDYeu8rB4j5fNi-EIg9UfsZt8;ih)0JtT`UO_)mtkz1wAu@x}ST&?z8Og<6fs9If}CwNi}tTv%K83B$7v?n)f zvj0mxKQZw7RmSwXjU+0y@^n$fz!o|&Z>*v(H7DftI@aEFtr9s%C>r1fGU~bIX0E9> zJC-$?na(r+6RL?M(`iFqif;}S^dOx>l6p`=@pzKC@@he-M39>|TBk5^&lB(tF>d3h&!>DCr-mm#mCR8cRsxuD>TD4{tgs=4`r!iE7Z zrKsn%%O@BE)kVNjmT7l~%ro4o*J0)3Ym-%9hclw}Ld=cES|xR#G&dst02dyZ3xZ4x)IyWPdPK*y0^rdHfbqt+=y|ApQagXqwqkUmR`85g(zjeyla~%N z7L16&<5olxTI0xW69lWEJpxe}M_F_)xA)Qg`RTqrFv;=ffJ-PxqIyZasx3Wk zp#B(Y)zQQlx*VvMaSCUMGvYkpJOX=>cH}E)567(4K%#vLkeEap&l?dJfe$A+++!JJ zT5hy~(RKscsrl%NyT(+-FV_VRL(r|VmLJ&|BnJ!>9^?4;Kf!Iloxow(evKii-N45g zxK($dGcm5f$uWB7uF2kq!A30W?#GUUx~+{LQCe9Hm4UYLNkb&sycjr(gFGFCq1_w< zCvp&zqJyb1Fu^ctZL@Hrj6UI>FmB-&hQmQ%?cB6vxNN4o-M7sc1x?Ank9g)@tvw-! zEU!L-VjQ?2K4H-|V{?XfJ1XG0QWjV22X`AC?HNm6qgkHxTXujC=p`s{c0b;C;~kT8 zrx%du?c}fq_-$J4N%2)1@^8lEhrxJ*zl+}YM$G>MYcVU}h#*Ri_3$`c&?^iDaBlPc z7jVo(JkSk5wy>dVBacq>c5zNN-W8Q+u{vg`2Rht_d9zTgE4vKkYQypSTuU~fKg3k} zV=A9*GTfDd?To<1J|5X%jN3&Q79>yI0~v5SIL8J#?rN8Tb#lxOOpE)RHWoOn7arE9 z>{I$~K}JF!779B2Puf>-uUvM{7*Xety4x{x4s>4mOh>X?_=wAmyB!-mKofD-Vq~&| zk=kF%jR75${n>&TTn902Ijx^B|KcQ6`ToHE)aq#O{97~rI>`64VY!75fHwRe!%(5x zr5&W1@~X_~Re%SN@>362wyZ{-5j1iJ&^9|Zgw{*<$Ym*KA3_dC#4$L6d3%nc(&wqY zZ~{3&^9nsTJ@}wef|s8*dcsRk&;9k#ZG|t5YWla4rnZ9?6wND$$LR-WIr`Cnin)j7|+i2(M-O5z= zeK_9KzToF|7Hws(ZhB^rVX+hox#O}u##?OgX6w8HEF?c;HTYs;T6QmeN69m#RnZ~k zH{=bvXu)fwmkuvTCwpjWapw5V#wirINx$)09hb2)+ZuTEYv{t_v80Sv6;HIy`9|N0 zfdf}~p)H%~v&9p}H&tGX7ft;ZVdLaU!HKb+9)jvu^y}h0@->}OQb6kI@{)>jX6*KW z{|9acA7gx!9zSB7pDT3Gau=-P&>n@Yk~?kbH4iXb`Y|$7yZIp`L0D0T)Nbt!I;qr4 zo}f>a{(ADV!-lCdh~F3`rXA{mFRpSFAYw0}XG_PVZ;3fwblIqC3+^N1{L((8_JtB@ zq)QhTPIwe~M>{X+GB#}%Ohvb1#S1&7hwfiEpIo7~MJvglY5k%SQ|=*pc+qX-%F~M< zoj86OF5mgybT}19$5)CgE#rWQqvzY4?7d1_vUpPRWn>2t`!$h^v}^H7a&LHcnTwdx zJ`AraUl30=hP#%$STyTwwjcx}HA%Qej=)6h8%!!knMy>Qjb-JFW&dewx1|Gu`uH8C zn|9nm*3l1ltQ|)H;x^PCeUmY4o>Om&87>@>w??UV=SI>OzPi&CPpea1Z#T`h`g+ze z_646(%Gh)qz7bFbbnrRGc7awukCNiRZUS8bItW?~8rjC!C!jr`8fZW051>si!47V) z2&fCRAC%u{gP;%b^~;O}Kucd?Y)ic06Poa{3xNGq#v-8gpr3#af@+`@+hG{AAJm4% z{jHg?dqDeJ8T0b>2aGjwiX-J)&>rmA4fsN+A7HEp)P{4*HcAj&pbpSd&}`5OP-Rpy zGQ>*-fDNEEpiN;<^99o_jl3xNNw%j9n7&Z7nt(4ADFDyS_Xlur&FBy3oCN0{biM34Y#7*f zVAsnfqWc2f*q%IQ$qVRpOxg`P8=U3xxW zvYl>=JWsOe+>Uustatus = DBL_STATUS_OUTPUT_FAILED; + out->stage = DBL_STAGE_OUTPUT; + out->failure_class = DBL_FAILURE_OUTPUT_LIMIT; + return 1; +} static void supervise(int client, pid_t pid, int output, struct dbl_result *out) { unsigned char bytes[DBL_MAX_OUTPUT + 1] = {0}; @@ -212,12 +230,9 @@ static void supervise(int client, pid_t pid, int output, ssize_t got = read(output, bytes + used, sizeof(bytes) - used); if (got > 0) used += (size_t)got; - if (used > DBL_MAX_OUTPUT) { + if (output_limit_crossed(used, out)) { output_limited = 1; p[1].events = 0; - out->status = DBL_STATUS_OUTPUT_FAILED; - out->stage = DBL_STAGE_OUTPUT; - out->failure_class = DBL_FAILURE_OUTPUT_LIMIT; } } if (disconnected || output_limited) @@ -236,6 +251,8 @@ static void supervise(int client, pid_t pid, int output, if (got <= 0) break; used += (size_t)got; + if (output_limit_crossed(used, out)) + output_limited = 1; } if (WIFEXITED(status)) out->exit_code = WEXITSTATUS(status); From 128dd560fc55c70311ea4811a140d575c8a312a0 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 04:19:47 +0200 Subject: [PATCH 48/69] test: take the facade before any mount listens so a contended fixed port fails red instead of parking the runner --- src/runtime/engineBrokerMcpFacade.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/runtime/engineBrokerMcpFacade.test.ts b/src/runtime/engineBrokerMcpFacade.test.ts index 057e798..80bdfc0 100644 --- a/src/runtime/engineBrokerMcpFacade.test.ts +++ b/src/runtime/engineBrokerMcpFacade.test.ts @@ -46,8 +46,9 @@ const startFacade = async (): Promise => { }; test("MCP facade routes only valid active capabilities to the registered mount", async () => { + const facade=await sharedFacade(); let calls=0;const target=createServer((_request,response)=>{calls++;response.writeHead(200,{"content-type":"application/json"});response.end('{"ok":true}');});await new Promise((resolve)=>target.listen(0,"127.0.0.1",resolve));const address=target.address();if(address===null||typeof address==="string")throw new Error(); - const facade=await sharedFacade();const token=facade.register("agent","turn-capabilities",`http://127.0.0.1:${address.port}/mcp`);const call=(value:string)=>fetch(FACADE_URL,{method:"POST",headers:{authorization:`Bearer ${value}`,"content-type":"application/json"},body:"{}"}); + const token=facade.register("agent","turn-capabilities",`http://127.0.0.1:${address.port}/mcp`);const call=(value:string)=>fetch(FACADE_URL,{method:"POST",headers:{authorization:`Bearer ${value}`,"content-type":"application/json"},body:"{}"}); try{assert.equal((await call("wrong-token-abcdefghijklmnopqrstuvwxyz0123456789")).status,403);assert.equal((await call(token)).status,200);assert.equal(calls,1);facade.revoke("turn-capabilities");assert.equal((await call(token)).status,403);assert.equal(calls,1);}finally{facade.revoke("turn-capabilities");await new Promise((resolve)=>target.close(()=>resolve()));} }); @@ -231,6 +232,9 @@ test("the facade forwards a closed header allowlist and never the worker's beare }); test("the facade withholds a mount response header that is not on the allowlist", async () => { + // The facade comes first: nothing must be listening while the fixed port is + // still in doubt, or a refused start leaks this mount and parks the runner. + const facade = await sharedFacade(); const target = createServer((_request, response) => { response.writeHead(200, { "content-type": "application/json", @@ -244,7 +248,6 @@ test("the facade withholds a mount response header that is not on the allowlist" await new Promise((resolve) => target.listen(0, "127.0.0.1", resolve)); const address = target.address(); if (address === null || typeof address === "string") throw new Error("target address unavailable"); - const facade = await sharedFacade(); const capability = facade.register("alpha", "turn-response-headers", `http://127.0.0.1:${address.port}/mcp`); try { const answered = await fetch(FACADE_URL, { From 2ef45890c50df36925719a80545a2e2a4a260f06 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 04:20:49 +0200 Subject: [PATCH 49/69] fix: end a per-wake MCP mount's leftover connections so a finished turn cannot park on its own teardown --- src/pi/AGENTS.md | 10 ++++ src/pi/cliSession.ts | 12 ++++- src/pi/cliSessionMcpMountClose.test.ts | 64 ++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 src/pi/cliSessionMcpMountClose.test.ts diff --git a/src/pi/AGENTS.md b/src/pi/AGENTS.md index eb26811..4685c74 100644 --- a/src/pi/AGENTS.md +++ b/src/pi/AGENTS.md @@ -9,3 +9,13 @@ removing redundant nested denies that cannot be mounted by its Linux sandbox. An intervening readable path or workspace root makes a deeper deny necessary. Verify changes with generated production arguments and real local sandbox commands; a model call is neither required nor permitted for this check. + +The per-wake MCP mount is torn down on the wake's own completion path, so that +teardown must be bounded. `Server.close()` waits for every open connection, and +a connection the MCP transport has no record of — a socket opened before +`initialize`, or an idle keep-alive socket a client's pool still holds, which is +what relaying a turn through the broker MCP facade leaves behind — is not the +transport's to end. Close the transport first, then end the remaining +connections; never wait for the client to release them. A finished turn that +parks here publishes nothing and dies to an outer deadline, which loses exactly +the terminal evidence the turn existed to produce. diff --git a/src/pi/cliSession.ts b/src/pi/cliSession.ts index ec4de9e..f161009 100644 --- a/src/pi/cliSession.ts +++ b/src/pi/cliSession.ts @@ -189,7 +189,17 @@ const startMcp = async ( await startupSettled; await transport.close().catch(() => undefined); await mcpServer.close().catch(() => undefined); - if (httpServer.listening) await new Promise((resolve) => httpServer.close(() => resolve())); + // `close` only stops accepting and then waits for every open connection, + // including the ones the transport has no record of and so cannot end (a + // socket opened before `initialize`, or a client pool's idle keep-alive + // socket, which relaying a turn through the broker MCP facade leaves + // behind). That wait is unbounded and sits on the wake's own completion + // path: measured, one such connection parked a finished broker turn with + // its result in hand and published nothing. By here this one wake's engine + // has returned, failed or been cancelled, so anything still connected is a + // leftover — see `AGENTS.md`, and the facade, which bounds itself the same + // way. + if (httpServer.listening) await new Promise((resolve) => { httpServer.close(() => resolve()); httpServer.closeAllConnections(); }); lifecycle = "closed"; })(); const mount = { get endpoint(): string { return endpoint; }, close }; diff --git a/src/pi/cliSessionMcpMountClose.test.ts b/src/pi/cliSessionMcpMountClose.test.ts new file mode 100644 index 0000000..69b74ed --- /dev/null +++ b/src/pi/cliSessionMcpMountClose.test.ts @@ -0,0 +1,64 @@ +import assert from "node:assert/strict"; +import { connect, type Socket } from "node:net"; +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { setTimeout as delay } from "node:timers/promises"; + +import { createCliSessionFactory } from "./cliSession.js"; + +type PublishedTurn = { readonly message: { readonly content: readonly unknown[] } }; + +/** + * A finished turn must not wait on a connection to its own per-wake MCP mount. + * + * The mount's HTTP server is torn down on the wake's completion path, and + * `Server.close()` only stops accepting: it waits for every open connection. + * The MCP transport ends the sessions it knows about, but a connection it has + * no record of — a socket opened before `initialize`, or an idle HTTP + * keep-alive socket a client's connection pool is still holding — is not its + * to end. The broker MCP facade is exactly such a client: relaying a turn's + * session leaves pooled connections to the mount behind it. + * + * Measured before this was bounded: the broker turn returned its result and + * the wake then never completed and never published — the terminal evidence + * that is the whole point of letting a finished turn finish. + */ +const TURN_COMPLETION_BOUND_MS = 5_000; + +test("a finished Grok broker turn publishes without waiting on an open connection to its own MCP mount", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-mount-close-")); + const peers: Socket[] = []; + const hangUp = (): void => { for (const peer of peers) peer.destroy(); }; + try { + const { session } = await createCliSessionFactory({ + engine: "grok", + command: "/nonexistent-engine", + grokBrokerTurn: async (_prompt, mcpEndpoint) => { + // A peer holding a connection the transport never saw an `initialize` + // on: none of this connection is the mount's own session state. + const peer = connect({ host: "127.0.0.1", port: Number(new URL(mcpEndpoint).port) }); + peers.push(peer); + peer.on("error", () => undefined); + await new Promise((resolve, reject) => { peer.once("connect", resolve); peer.once("error", reject); }); + return "Filed and delivered."; + } + })({ cwd: root }); + const published: PublishedTurn[] = []; + session.subscribe((event) => { published.push(event as unknown as PublishedTurn); }); + try { + const settled = session.prompt("wake").then(() => "completed" as const); + const outcome = await Promise.race([settled, delay(TURN_COMPLETION_BOUND_MS).then(() => "parked" as const)]); + // Hang the peer up before asserting, so a regression reports the parked + // wake instead of parking the suite's own teardown behind it. + hangUp(); + assert.equal(outcome, "completed", `the finished wake did not complete within ${TURN_COMPLETION_BOUND_MS}ms`); + assert.equal(published.length, 1); + assert.deepEqual(published[0]?.message.content, [{ type: "text", text: "Filed and delivered." }]); + } finally { hangUp(); await session.disposeAsync?.(); } + } finally { + hangUp(); + await rm(root, { recursive: true, force: true }); + } +}); From 25f84ef02c65c2f8b327626a2082542e3705eb17 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 04:24:06 +0200 Subject: [PATCH 50/69] fix: end the broker provider proxy's leftover sockets on shutdown instead of waiting for a worker to release them --- src/runtime/grokBrokerProxy.test.ts | 17 +++++++++++++++++ src/runtime/grokBrokerProxy.ts | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/runtime/grokBrokerProxy.test.ts b/src/runtime/grokBrokerProxy.test.ts index 526bd78..93d4a9f 100644 --- a/src/runtime/grokBrokerProxy.test.ts +++ b/src/runtime/grokBrokerProxy.test.ts @@ -1,9 +1,12 @@ import assert from "node:assert/strict"; import { request as httpRequest } from "node:http"; +import { connect } from "node:net"; import test from "node:test"; +import { setTimeout as delay } from "node:timers/promises"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; import { GROK_SESSION_TITLE_SINK_KEY } from "./grokBrokerWorkerConfig.js"; import { estimateGrokRequestUsage, GrokBrokerTurnMeter } from "./grokBrokerTurnMeter.js"; +import { DEFAULT_GROK_BROKER_MODEL_POLICY } from "./grokBrokerModelPolicy.js"; import { ENGINE_BROKER_AUTH_STALE } from "./engineBrokerProtocol.js"; import { GROK_INFERENCE_AUTH_STALE_BODY } from "./grokInferenceProxy.js"; @@ -227,3 +230,17 @@ test("a response whose usage cannot be decoded is still delivered, and charged t assert.equal(snapshot.timings[0]?.toolCalls, undefined, "an undecodable response records no tool-call attempt either"); } finally { await proxy.close(); } }); + +test("proxy shutdown ends a worker's leftover keep-alive socket instead of waiting for it", async () => { + const proxy = await startGrokBrokerProxy({ accessToken: async () => "token", markRejected: async () => undefined }, async () => ({ status: 200, headers: { "content-type": "text/event-stream" }, body: Buffer.from("data: done\n\n") }), DEFAULT_GROK_BROKER_MODEL_POLICY, 0); + // A worker's HTTP client keeps its connection to the proxy pooled; nothing + // in the proxy's own state accounts for it, so a shutdown that waits for the + // client to release it has no bound. + const socket = connect({ host: "127.0.0.1", port: proxy.port }); + socket.on("error", () => undefined); + await new Promise((resolve, reject) => { socket.once("connect", resolve); socket.once("error", reject); }); + try { + const outcome = await Promise.race([proxy.close().then(() => "closed" as const), delay(5_000).then(() => "parked" as const)]); + assert.equal(outcome, "closed", "proxy shutdown parked on a socket the proxy does not track"); + } finally { socket.destroy(); } +}); diff --git a/src/runtime/grokBrokerProxy.ts b/src/runtime/grokBrokerProxy.ts index 9857cff..7ee024e 100644 --- a/src/runtime/grokBrokerProxy.ts +++ b/src/runtime/grokBrokerProxy.ts @@ -35,7 +35,7 @@ export async function startGrokBrokerProxy(authority: GrokBrokerCredentialAuthor const guards=new MapPromise>();const turns=new Map();const server = createServer((request, response) => { void serve(request, response, authority, upstream, capabilities,guards,turns,declared,grants); }); await new Promise((resolve, reject) => { server.once("error", reject); server.listen(listenPort, "127.0.0.1", () => { server.off("error", reject); resolve(); }); }); const address = server.address() as AddressInfo; - return { port: address.port, capabilities,registerIsolationGuard(turnId,guard){guards.set(turnId,guard);},revokeIsolationGuard(turnId){guards.delete(turnId);},registerTurn(turnId,turn){turns.set(turnId,{policy:parseGrokBrokerModelPolicy(turn.policy),meter:turn.meter});},revokeTurn(turnId){turns.delete(turnId);}, close: () => new Promise((resolve, reject) => server.close((error) => error === undefined ? resolve() : reject(error))) }; + return { port: address.port, capabilities,registerIsolationGuard(turnId,guard){guards.set(turnId,guard);},revokeIsolationGuard(turnId){guards.delete(turnId);},registerTurn(turnId,turn){turns.set(turnId,{policy:parseGrokBrokerModelPolicy(turn.policy),meter:turn.meter});},revokeTurn(turnId){turns.delete(turnId);}, close: () => new Promise((resolve, reject) => { server.close((error) => error === undefined ? resolve() : reject(error)); /* `close` waits for every open connection, and a worker keeps its client pool's socket to this proxy open with nothing here accounting for it — so that wait has no bound. Ending them is this listener's to do, exactly as the MCP facade does. */ server.closeAllConnections(); }) }; } /** From 6c33505409048bddd57797543cbd448e52658f2a Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 11:15:39 +0200 Subject: [PATCH 51/69] test: split the MCP tunnel drain test out of the facade suite --- src/runtime/engineBrokerMcpTunnel.test.ts | 59 +++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 src/runtime/engineBrokerMcpTunnel.test.ts diff --git a/src/runtime/engineBrokerMcpTunnel.test.ts b/src/runtime/engineBrokerMcpTunnel.test.ts new file mode 100644 index 0000000..b22bf36 --- /dev/null +++ b/src/runtime/engineBrokerMcpTunnel.test.ts @@ -0,0 +1,59 @@ +import assert from "node:assert/strict"; +import { createServer } from "node:http"; +import { connect, type Socket } from "node:net"; +import test from "node:test"; + +import { awaitMcpTunnelDrain } from "./engineBrokerMcpFacade.js"; + +const withDeadline = async (work: Promise, ms: number, message: string): Promise => { + let timer: NodeJS.Timeout | undefined; + try { return await Promise.race([work, new Promise((_resolve, reject) => { timer = setTimeout(() => reject(new Error(message)), ms); })]); } + finally { if (timer) clearTimeout(timer); } +}; + +/** + * The relay parks on this await whenever a tunnel is backpressured, and a + * parked await is invisible from outside: no status, no refusal, no line. So + * the assertion is that it settles at all — on a client that hung up + * mid-write, on the turn's abort, and on a genuine drain — with a deadline + * standing in for the hang. + */ +test("a backpressured MCP tunnel always settles: on a hang-up, on the turn's abort, and on a real drain", async () => { + const parked = new Map>(); + // One controller per phase: the turn whose abort is under test must not be + // the turn that is still relaying. + const controllers = new Map(); + const server = createServer((request, response) => { + const phase = request.url ?? ""; + const controller = new AbortController(); controllers.set(phase, controller); + response.writeHead(200, { "content-type": "text/event-stream" }); + // A paused client cannot absorb this, so `write` reports backpressure and + // the relay would park exactly here. + response.write("data: open\n\n"); + if (phase === "/drain") assert.equal(response.write(Buffer.alloc(16 * 1024 * 1024, 0x61)), false, "a paused client must backpressure the tunnel"); + parked.set(phase, awaitMcpTunnelDrain(response, controller.signal).then(() => "drained", (error: Error) => error.message)); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); if (address === null || typeof address === "string") throw new Error(); + const open = (phase: string): Promise => new Promise((resolve) => { + const socket = connect(address.port, "127.0.0.1", () => socket.write(`GET ${phase} HTTP/1.1\r\nhost: facade\r\n\r\n`)); + socket.once("data", () => { socket.pause(); resolve(socket); }); + }); + const settled = (phase: string): Promise => withDeadline(parked.get(phase)!, 3_000, `the ${phase} await never settled: the relay is parked`); + const sockets: Socket[] = []; + try { + sockets.push(await open("/hangup")); + sockets[0]!.destroy(); + assert.equal(await settled("/hangup"), "MCP tunnel closed", "a client that hung up mid-write wakes the await"); + sockets.push(await open("/abort")); + controllers.get("/abort")!.abort(); + assert.equal(await settled("/abort"), "MCP tunnel aborted", "the turn's own abort wakes the await"); + const draining = await open("/drain"); + sockets.push(draining); + draining.resume(); + assert.equal(await settled("/drain"), "drained", "a client that resumes reading resolves the await"); + } finally { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve) => server.close(() => resolve())); + } +}); From c05306eacee8cb51170349478ff86d9ed9e1ff36 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 11:15:42 +0200 Subject: [PATCH 52/69] feat: seal the brokered worker's in-flight MCP tool calls so a hung call is visible as one --- src/runtime/AGENTS.md | 26 ++++ src/runtime/engineBrokerControlClient.ts | 12 +- src/runtime/engineBrokerMcpCallLog.test.ts | 73 +++++++++++ src/runtime/engineBrokerMcpCallLog.ts | 137 +++++++++++++++++++++ src/runtime/engineBrokerMcpFacade.test.ts | 134 ++++++++++++-------- src/runtime/engineBrokerMcpFacade.ts | 33 ++++- src/runtime/engineBrokerProtocol.test.ts | 25 ++++ src/runtime/engineBrokerProtocol.ts | 38 +++++- src/runtime/engineBrokerService.test.ts | 11 ++ src/runtime/engineBrokerService.ts | 2 +- src/runtime/grokEngineBrokerTurn.ts | 16 ++- src/runtime/grokEngineBrokerUsage.test.ts | 38 +++++- 12 files changed, 480 insertions(+), 65 deletions(-) create mode 100644 src/runtime/engineBrokerMcpCallLog.test.ts create mode 100644 src/runtime/engineBrokerMcpCallLog.ts diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 682556b..5a158bb 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -288,6 +288,32 @@ outcome but a real drain rejects, so the relay tears the tunnel down instead of writing into a socket that is gone. Never widen it into a transparent proxy: the whole point of the boundary is that the allowlist is closed. +The facade is also the only place an MCP tool call is observable *while it is +still running*. Daimon writes a tool receipt on completion, so a call that +started and never returned is byte-identical, in every artifact, to a call that +was never made — and that was the last unlit path under a live hang where the +worker stopped acting after its eighth provider response, the per-request +ledger published `open: 0`, and the trial deadline killed it seven minutes +later. `engineBrokerMcpCallLog.ts` records each relayed `tools/call` POST and +whether the facade ever answered it, and the observation rides the *sealed +terminal response* of a failed turn (`mcpCalls`, optional and v2-only) — +the seam the worker's redacted last words and the sealed usage already take, +because the slot's control root is tmpfs that dies with the container. It +replays with the record and reaches the operator through +`engineBrokerControlClient.ts` as `mcp=/ answered` plus +`mcp_outstanding=@ms`. Its rules are the per-request ledger's: names +and timings only (never arguments, never a result, never a session id or +bearer; a name that is not a plain short identifier is ``, and the +list is bounded with a `` last entry); absence stays absence (a turn +the facade never registered observes as *nothing*, a turn that called nothing +observes `started: 0`, and a POST body the facade could not read counts in +`undecoded` rather than inventing a name); and it can never fail, delay or +refuse a turn. "Answered" means one thing and it is load bearing: the relay +reached its own `end()`. A tunnel torn down when the worker dies did not +answer, so the call it was blocked on stays outstanding with the elapsed time +it had reached — otherwise the turn's death would erase the evidence the +instrument exists to keep. + Worker `GROK_HOME` layout the deployment must provision (attested before every turn by `grokWorkerHomeAttestation.ts`, recorded in `GROK_ENGINE_BROKER.worker.home`): `$GROK_HOME` and `$GROK_HOME/sessions` `root: 1771`; `config.toml`, diff --git a/src/runtime/engineBrokerControlClient.ts b/src/runtime/engineBrokerControlClient.ts index a02d758..6744615 100644 --- a/src/runtime/engineBrokerControlClient.ts +++ b/src/runtime/engineBrokerControlClient.ts @@ -2,10 +2,20 @@ import { createHash, randomUUID } from "node:crypto"; import { createConnection } from "node:net"; import { ENGINE_BROKER_VERSION, encodeEngineBrokerFrame,EngineBrokerFrameDecoder,parseEngineBrokerResponse } from "./engineBrokerProtocol.js"; import type { EngineBrokerInferenceFailureCode, EngineBrokerInferenceRequest, EngineBrokerInferenceResponse } from "./engineBrokerInferenceProtocol.js"; +import type { EngineBrokerMcpCallObservation } from "./engineBrokerMcpCallLog.js"; import type { EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; import type { GrokBrokerModel, GrokBrokerReasoningEffort } from "./grokBrokerModelPolicy.js"; import type { GrokInferencePurpose } from "./inferenceUsageLedger.js"; +/** + * What the broker saw of the worker's MCP tool calls on a failed turn + * (`engineBrokerMcpCallLog.ts`). Absent for a turn with no observation at all; + * `outstanding` names every call that started and was never answered, with how + * long it had been waiting — the one thing a completion-only tool receipt can + * never say. + */ +const renderMcpCalls=(calls:EngineBrokerMcpCallObservation|undefined):string=>calls===undefined?"":`; mcp=${calls.answered}/${calls.started} answered${calls.undecoded===0?"":`; mcp_undecoded=${calls.undecoded}`}${calls.outstanding.length===0?"":`; mcp_outstanding=${calls.outstanding.map((call)=>`${call.name}@${call.outstandingMs}ms`).join(",")}`}`; + export type EngineBrokerInferenceGrant = Omit, "version" | "kind" | "requestId">; /** A refused grant request; `code` is closed (`auth_stale` is the stale shared realm, `grant_limit` the live-grant cap). */ export class EngineBrokerInferenceGrantRefused extends Error { @@ -45,6 +55,6 @@ export class EngineBrokerControlClient implements EngineBrokerTurnClient { const turnId=createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"),requestId=randomUUID();const request={version:ENGINE_BROKER_VERSION,kind:"start_turn",requestId,turnId,agentId,wakeId,prompt,mcpEndpoint,...(options.limits===undefined?{}:{limits:options.limits})} as const;const socket=createConnection({path:this.socketPath});const decoder=new EngineBrokerFrameDecoder(); return new Promise((resolve,reject)=>{let accepted=false,settled=false;const fail=()=>{if(settled)return;settled=true;cleanup();reject(new Error("engine broker unavailable"));};const cleanup=()=>{signal?.removeEventListener("abort",abort);socket.destroy();};const abort=()=>fail();signal?.addEventListener("abort",abort,{once:true});if(signal?.aborted)return abort();socket.once("connect",()=>socket.write(encodeEngineBrokerFrame(request)));socket.on("data",(chunk)=>{try{for(const value of decoder.push(chunk)){const response=parseEngineBrokerResponse(value);if((response.kind!=="accepted"&&response.kind!=="completed"&&response.kind!=="failed")||response.requestId!==requestId||response.turnId!==turnId)throw new Error();if(response.kind==="accepted"){if(accepted)throw new Error();accepted=true;continue;}if(!accepted||settled)throw new Error();settled=true;cleanup(); if(options.model!==undefined&&response.model!==options.model){reject(new Error(`engine broker turn used model ${response.model}, not the declared ${options.model}`));return;} - if(response.kind==="completed")resolve(response.text);else reject(new Error(`engine broker turn failed (${response.code}${response.limitReason==="none"?"":`; limit=${response.limitReason}`}${response.diagnostic ? `; ${response.diagnostic.stage}/${response.diagnostic.failureClass}; exit=${response.diagnostic.exitCode}; signal=${response.diagnostic.termSignal}${response.diagnostic.reason===undefined?"":`; reason=${response.diagnostic.reason}`}` : ""})`));}}catch{fail();}});socket.once("error",fail);socket.once("close",()=>{if(!settled)fail();});}); + if(response.kind==="completed")resolve(response.text);else reject(new Error(`engine broker turn failed (${response.code}${response.limitReason==="none"?"":`; limit=${response.limitReason}`}${response.diagnostic ? `; ${response.diagnostic.stage}/${response.diagnostic.failureClass}; exit=${response.diagnostic.exitCode}; signal=${response.diagnostic.termSignal}${response.diagnostic.reason===undefined?"":`; reason=${response.diagnostic.reason}`}` : ""}${renderMcpCalls(response.mcpCalls)})`));}}catch{fail();}});socket.once("error",fail);socket.once("close",()=>{if(!settled)fail();});}); } } diff --git a/src/runtime/engineBrokerMcpCallLog.test.ts b/src/runtime/engineBrokerMcpCallLog.test.ts new file mode 100644 index 0000000..8550608 --- /dev/null +++ b/src/runtime/engineBrokerMcpCallLog.test.ts @@ -0,0 +1,73 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { EngineBrokerMcpCallLog, ENGINE_BROKER_MCP_CALL_INVALID, ENGINE_BROKER_MCP_CALL_TRUNCATED, ENGINE_BROKER_MCP_OUTSTANDING_MAX } from "./engineBrokerMcpCallLog.js"; + +const body = (value: unknown): Buffer => Buffer.from(JSON.stringify(value), "utf8"); +const call = (name: unknown, id = 1): Buffer => body({ jsonrpc: "2.0", id, method: "tools/call", params: { name, arguments: { text: "argument-bytes" } } }); + +/** A controllable clock: an outstanding call's whole value is how long it has been outstanding. */ +const clock = (): { log: EngineBrokerMcpCallLog; advance: (ms: number) => void } => { + let at = 1_000; + return { log: new EngineBrokerMcpCallLog(() => at), advance: (ms: number): void => { at += ms; } }; +}; + +test("an unanswered tool call is outstanding with its name and its elapsed time; an answered one is neither", () => { + const { log, advance } = clock(); + log.open("turn"); + const answered = log.begin("turn", call("daimon__moltnet_send")); + advance(20); answered.answer(); answered.close(); + const hung = log.begin("turn", call("daimon__moltnet_read", 2)); + advance(420_000); + const observed = log.observe("turn"); + assert.deepEqual(observed?.outstanding, [{ name: "daimon__moltnet_read", outstandingMs: 420_000 }]); + assert.equal(observed?.started, 2); assert.equal(observed?.answered, 1); assert.equal(observed?.undecoded, 0); + // A relay torn down by the turn's death answered nothing, and the elapsed + // time freezes where it stopped rather than growing with the report. + hung.close(); advance(5_000); + assert.deepEqual(log.observe("turn")?.outstanding, [{ name: "daimon__moltnet_read", outstandingMs: 420_000 }]); + assert.ok(!JSON.stringify(log.observe("turn")).includes("argument-bytes"), "names and timings only"); +}); + +test("absence stays absence: an unopened turn observes undefined, a turn that called nothing observes zero", () => { + const { log } = clock(); + assert.equal(log.observe("turn"), undefined); + log.open("turn"); + assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 0, outstanding: [] }); + // Everything that is not a tool call records nothing at all, so `started` + // stays a count of tool calls and not of traffic. + for (const value of [body({ jsonrpc: "2.0", id: 1, method: "tools/list" }), body({ jsonrpc: "2.0", method: "notifications/initialized" }), Buffer.alloc(0)]) log.begin("turn", value).answer(); + log.begin("turn", undefined).answer(); + assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 0, outstanding: [] }); + log.close("turn"); + assert.equal(log.observe("turn"), undefined, "a revoked turn keeps nothing"); +}); + +test("a body the facade could not read counts as undecoded, never as a call with a name", () => { + const { log } = clock(); + log.open("turn"); + log.begin("turn", Buffer.from("{not json", "utf8")); + log.undecodable("turn"); + assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 2, outstanding: [] }); + log.undecodable("absent"); +}); + +test("a tool name that is not a plain identifier is recorded as invalid, and a batch names each of its calls", () => { + const { log } = clock(); + log.open("turn"); + log.begin("turn", call("moltnet read\nBearer sk-live-000")); + log.begin("turn", body([])); + log.begin("turn", Buffer.from(`[${call("memory_recall", 2).toString("utf8")},${call("world_probe", 3).toString("utf8")}]`, "utf8")); + assert.deepEqual(log.observe("turn")?.outstanding.map((entry) => entry.name), [ENGINE_BROKER_MCP_CALL_INVALID, "memory_recall", "world_probe"]); +}); + +test("the outstanding list is bounded, and the earliest calls are the ones kept", () => { + const { log, advance } = clock(); + log.open("turn"); + for (let index = 0; index < ENGINE_BROKER_MCP_OUTSTANDING_MAX + 4; index += 1) { log.begin("turn", call(`tool_${index}`, index)); advance(1); } + const observed = log.observe("turn"); + assert.equal(observed?.started, ENGINE_BROKER_MCP_OUTSTANDING_MAX + 4); + assert.equal(observed?.outstanding.length, ENGINE_BROKER_MCP_OUTSTANDING_MAX); + assert.equal(observed?.outstanding[0]?.name, "tool_0"); + assert.equal(observed?.outstanding.at(-1)?.name, ENGINE_BROKER_MCP_CALL_TRUNCATED); +}); diff --git a/src/runtime/engineBrokerMcpCallLog.ts b/src/runtime/engineBrokerMcpCallLog.ts new file mode 100644 index 0000000..fc49dbf --- /dev/null +++ b/src/runtime/engineBrokerMcpCallLog.ts @@ -0,0 +1,137 @@ +/** + * The in-flight MCP tool calls of one brokered turn. + * + * Daimon writes a tool receipt only when a call *completes*, so a call that + * started and never returned is byte-identical, in every artifact, to a call + * that was never made. A live Grok turn stopped acting after its eighth + * provider response and was killed by the trial deadline seven minutes later + * with the proxy's per-request ledger reporting `open: 0` — every provider + * request closed — which leaves exactly one unlit path: a tool call the worker + * issued and the facade never answered. + * + * This is that light, and it follows the per-request ledger's rules rather + * than inventing its own: + * + * - **names and timings only.** The tool name off the JSON-RPC envelope and + * two clocks. Never arguments, never a result, never a session id, never a + * capability or bearer. A name that is not a plain short identifier is + * recorded as {@link ENGINE_BROKER_MCP_CALL_INVALID} rather than passed + * through, and the list is bounded at + * {@link ENGINE_BROKER_MCP_OUTSTANDING_MAX} with + * {@link ENGINE_BROKER_MCP_CALL_TRUNCATED} as its last entry. + * - **absence stays absence.** A turn the log never opened observes as + * `undefined`; a turn that made no call observes `started: 0`, which is not + * the same statement as an answered call. A POST whose body the facade could + * not read counts in `undecoded` and never as a call with a name, because a + * fabricated name is byte-identical to a measured one — and because "zero + * calls started" is exactly the reading this instrument exists to make + * trustworthy. + * - **it cannot fail a turn.** Every operation here is arithmetic over a map, + * the one parse is wrapped, and the facade treats a missing handle as a + * no-op. + * + * "Answered" means the facade wrote a complete response back to the worker — + * the relay reached its own `end()`. A relay that was torn down (the worker + * died, the tunnel broke, the turn aborted) did *not* answer, so its calls stay + * outstanding with the elapsed time they had reached. That is the whole point: + * the turn's death must not retroactively mark the call it was blocked on as + * finished. + */ +export const ENGINE_BROKER_MCP_OUTSTANDING_MAX = 16; +export const ENGINE_BROKER_MCP_CALL_INVALID = ""; +export const ENGINE_BROKER_MCP_CALL_TRUNCATED = ""; +/** A plain short identifier, or one of the two sentinels above. */ +export const ENGINE_BROKER_MCP_CALL_NAME = /^(?:||[A-Za-z0-9_.-]{1,64})$/u; +const TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/u; + +export type EngineBrokerOutstandingMcpCall = Readonly<{ name: string; outstandingMs: number }>; +/** + * What the facade saw of one turn's tool calls: how many started, how many the + * facade answered, how many POST bodies it could not read, and the ones still + * unanswered with the time each has been outstanding. + */ +export type EngineBrokerMcpCallObservation = Readonly<{ started: number; answered: number; undecoded: number; outstanding: readonly EngineBrokerOutstandingMcpCall[] }>; + +/** One relayed POST's calls. `answer` marks a complete relay; `close` ends it unanswered. Both are idempotent. */ +export interface EngineBrokerMcpCallHandle { answer(): void; close(): void } +const INERT: EngineBrokerMcpCallHandle = { answer: () => undefined, close: () => undefined }; + +type CallRecord = { readonly name: string; readonly startedAt: number; endedAt?: number }; +type TurnLog = { started: number; answered: number; undecoded: number; readonly live: Set; readonly ended: CallRecord[] }; + +export class EngineBrokerMcpCallLog { + private readonly turns = new Map(); + constructor(private readonly now: () => number = Date.now) {} + + /** A turn the facade registered. Re-opening an id resets it: a turn id is unique per turn. */ + open(turnId: string): void { this.turns.set(turnId, { started: 0, answered: 0, undecoded: 0, live: new Set(), ended: [] }); } + close(turnId: string): void { this.turns.delete(turnId); } + + /** A POST body the facade is about to relay. Anything that is not a `tools/call` records nothing. */ + begin(turnId: string, body: Uint8Array | undefined): EngineBrokerMcpCallHandle { + const log = this.turns.get(turnId); + if (log === undefined || body === undefined || body.byteLength === 0) return INERT; + const names = toolCallNames(body); + if (names === undefined) { log.undecoded += 1; return INERT; } + if (names.length === 0) return INERT; + const startedAt = this.now(); + const records = names.map((name): CallRecord => ({ name, startedAt })); + log.started += records.length; + for (const record of records) log.live.add(record); + let settled = false; + return { + answer: (): void => { + if (settled) return; settled = true; + log.answered += records.length; + for (const record of records) log.live.delete(record); + }, + close: (): void => { + if (settled) return; settled = true; + const endedAt = this.now(); + for (const record of records) { + log.live.delete(record); record.endedAt = endedAt; + // Retain only as many as can be reported; the earliest are the ones + // a hang is about, so a later flood cannot displace them. + if (log.ended.length < ENGINE_BROKER_MCP_OUTSTANDING_MAX) log.ended.push(record); + } + } + }; + } + + /** A POST whose body the facade refused to read (over its own bound), which is a call it cannot name. */ + undecodable(turnId: string): void { const log = this.turns.get(turnId); if (log !== undefined) log.undecoded += 1; } + + observe(turnId: string): EngineBrokerMcpCallObservation | undefined { + const log = this.turns.get(turnId); + if (log === undefined) return undefined; + const at = this.now(); + const pending = [...log.live, ...log.ended].sort((left, right) => left.startedAt - right.startedAt); + const outstanding = pending.map((record): EngineBrokerOutstandingMcpCall => ({ name: record.name, outstandingMs: Math.max(0, (record.endedAt ?? at) - record.startedAt) })); + return { + started: log.started, answered: log.answered, undecoded: log.undecoded, + outstanding: outstanding.length > ENGINE_BROKER_MCP_OUTSTANDING_MAX + ? [...outstanding.slice(0, ENGINE_BROKER_MCP_OUTSTANDING_MAX - 1), { name: ENGINE_BROKER_MCP_CALL_TRUNCATED, outstandingMs: 0 }] + : outstanding + }; + } +} + +/** + * The tool names one JSON-RPC POST asks for: `undefined` when the body did not + * decode at all (which is a fact of its own, not zero calls), `[]` when it + * decoded and asked for no tool. A batch names each of its calls. + */ +function toolCallNames(body: Uint8Array): readonly string[] | undefined { + let parsed: unknown; + try { parsed = JSON.parse(Buffer.from(body.buffer, body.byteOffset, body.byteLength).toString("utf8")); } catch { return undefined; } + const entries: readonly unknown[] = Array.isArray(parsed) ? parsed : [parsed]; + const names: string[] = []; + for (const entry of entries) { + if (!isRecord(entry) || entry.method !== "tools/call") continue; + const name = isRecord(entry.params) ? entry.params.name : undefined; + names.push(typeof name === "string" && TOOL_NAME.test(name) ? name : ENGINE_BROKER_MCP_CALL_INVALID); + } + return names; +} + +const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value); diff --git a/src/runtime/engineBrokerMcpFacade.test.ts b/src/runtime/engineBrokerMcpFacade.test.ts index 80bdfc0..54261ed 100644 --- a/src/runtime/engineBrokerMcpFacade.test.ts +++ b/src/runtime/engineBrokerMcpFacade.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; -import { createServer, type Server as HttpServer, type IncomingMessage } from "node:http"; -import { connect, type Socket } from "node:net"; +import { createServer, type Server as HttpServer, type IncomingMessage, type ServerResponse } from "node:http"; + import { randomUUID } from "node:crypto"; import test from "node:test"; @@ -52,6 +52,89 @@ test("MCP facade routes only valid active capabilities to the registered mount", try{assert.equal((await call("wrong-token-abcdefghijklmnopqrstuvwxyz0123456789")).status,403);assert.equal((await call(token)).status,200);assert.equal(calls,1);facade.revoke("turn-capabilities");assert.equal((await call(token)).status,403);assert.equal(calls,1);}finally{facade.revoke("turn-capabilities");await new Promise((resolve)=>target.close(()=>resolve()));} }); +/** + * Daimon writes a tool receipt only on completion, so a call that started and + * never returned reads exactly like a call that was never made — the one path + * a seven-minute live hang left unlit. The facade is where that difference is + * visible, and it has to survive the tear-down that ends the turn: a tunnel + * destroyed when the worker dies must not mark the call it was blocked on as + * answered, or the instrument erases the very evidence it exists to keep. + */ +test("MCP facade reports a tool call that started and never returned as outstanding, by name", async () => { + const facade = await sharedFacade(); + const held: ServerResponse[] = []; + // Two ways for a mount not to answer: never reply at all (`moltnet_read`), + // or open the stream and never deliver the result (`memory_recall`). + const target = createServer((request, response) => { + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.on("end", () => { + const asked = Buffer.concat(chunks).toString("utf8"); + if (asked.includes("moltnet_read")) { held.push(response); return; } + if (asked.includes("memory_recall")) { response.writeHead(200, { "content-type": "text/event-stream" }); response.write(": open\n\n"); held.push(response); return; } + response.writeHead(200, { "content-type": "application/json" }); response.end('{"jsonrpc":"2.0","id":1,"result":{}}'); + }); + }); + await new Promise((resolve) => target.listen(0, "127.0.0.1", resolve)); + const address = target.address(); if (address === null || typeof address === "string") throw new Error(); + const turnId = "turn-outstanding", token = facade.register("agent", turnId, `http://127.0.0.1:${address.port}/mcp`); + const post = (name: string, signal?: AbortSignal): Promise => fetch(FACADE_URL, { method: "POST", signal, headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name, arguments: { text: "argument-bytes-that-must-never-be-recorded" } } }) }); + const pending = new AbortController(); + /** + * A live call's elapsed time grows with every read, so two identical reads + * mean every relay has settled — the only moment at which "answered" is + * final. Polling for a name instead would read the log mid-teardown. + */ + const settled = async (): Promise> => { + for (let attempt = 0; attempt < 100; attempt += 1) { + const before = JSON.stringify(facade.observe(turnId)); + await new Promise((resolve) => setTimeout(resolve, 60)); + if (JSON.stringify(facade.observe(turnId)) === before) return facade.observe(turnId); + } + throw new Error("the facade's observation never settled"); + }; + const names = (observed: ReturnType): readonly string[] => (observed?.outstanding ?? []).map((call) => call.name); + try { + const answered = await post("daimon__moltnet_send"); + assert.equal(answered.status, 200); await answered.text(); + const hanging = post("daimon__moltnet_read", pending.signal).catch(() => undefined); + for (let attempt = 0; attempt < 200 && held.length === 0; attempt += 1) await new Promise((resolve) => setTimeout(resolve, 25)); + assert.equal(held.length, 1, "the mount never received the hung tool call"); + + const observed = facade.observe(turnId); + assert.deepEqual(names(observed), ["daimon__moltnet_read"], "the unanswered call must be outstanding, by name"); + assert.equal(observed?.started, 2); assert.equal(observed?.answered, 1, "the completed call must not be outstanding"); assert.equal(observed?.undecoded, 0); + assert.ok((observed?.outstanding[0]?.outstandingMs ?? -1) >= 0, "an outstanding call reports how long it has been waiting"); + assert.ok(!JSON.stringify(observed).includes("argument-bytes"), "names and timings only: no arguments"); + + // The turn's own death tears the tunnel down. The call was still never + // answered, and must still say so. + pending.abort(); await hanging; + const afterTeardown = await settled(); + assert.deepEqual(names(afterTeardown), ["daimon__moltnet_read"], "a torn-down relay is not an answer"); + assert.equal(afterTeardown?.answered, 1); + + // A stream the facade opened and never finished relaying is not an answer + // either. Awaiting the headers and one chunk puts the facade inside its own + // streaming relay before the client walks away, which is the branch that + // decides whether a half-written tunnel counts as an answer. + const halted = new AbortController(); + const half = await post("memory_recall", halted.signal); + assert.equal(half.status, 200); await half.body!.getReader().read(); halted.abort(); + const afterHalfRelay = await settled(); + assert.deepEqual(names(afterHalfRelay), ["daimon__moltnet_read", "memory_recall"], "a half-relayed stream is not an answer"); + assert.equal(afterHalfRelay?.answered, 1); + + facade.revoke(turnId); + assert.equal(facade.observe(turnId), undefined, "a turn the facade never registered observes as absence, not as zero"); + } finally { + pending.abort(); facade.revoke(turnId); + for (const response of held) response.destroy(); + target.closeAllConnections(); + await new Promise((resolve) => target.close(() => resolve())); + } +}); + /** * The brokered worker's real route: a real Daimon MCP mount behind a real * Streamable HTTP transport, reached by a real MCP client through the facade. @@ -315,50 +398,3 @@ test("revoking a turn tears down its open server-to-client stream, and closing n await second.close().catch(() => undefined); await rig.close().catch(() => undefined); }); - -/** - * The relay parks on this await whenever a tunnel is backpressured, and a - * parked await is invisible from outside: no status, no refusal, no line. So - * the assertion is that it settles at all — on a client that hung up - * mid-write, on the turn's abort, and on a genuine drain — with a deadline - * standing in for the hang. - */ -test("a backpressured MCP tunnel always settles: on a hang-up, on the turn's abort, and on a real drain", async () => { - const parked = new Map>(); - // One controller per phase: the turn whose abort is under test must not be - // the turn that is still relaying. - const controllers = new Map(); - const server = createServer((request, response) => { - const phase = request.url ?? ""; - const controller = new AbortController(); controllers.set(phase, controller); - response.writeHead(200, { "content-type": "text/event-stream" }); - // A paused client cannot absorb this, so `write` reports backpressure and - // the relay would park exactly here. - response.write("data: open\n\n"); - if (phase === "/drain") assert.equal(response.write(Buffer.alloc(16 * 1024 * 1024, 0x61)), false, "a paused client must backpressure the tunnel"); - parked.set(phase, awaitMcpTunnelDrain(response, controller.signal).then(() => "drained", (error: Error) => error.message)); - }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - const address = server.address(); if (address === null || typeof address === "string") throw new Error(); - const open = (phase: string): Promise => new Promise((resolve) => { - const socket = connect(address.port, "127.0.0.1", () => socket.write(`GET ${phase} HTTP/1.1\r\nhost: facade\r\n\r\n`)); - socket.once("data", () => { socket.pause(); resolve(socket); }); - }); - const settled = (phase: string): Promise => withDeadline(parked.get(phase)!, 3_000, `the ${phase} await never settled: the relay is parked`); - const sockets: Socket[] = []; - try { - sockets.push(await open("/hangup")); - sockets[0]!.destroy(); - assert.equal(await settled("/hangup"), "MCP tunnel closed", "a client that hung up mid-write wakes the await"); - sockets.push(await open("/abort")); - controllers.get("/abort")!.abort(); - assert.equal(await settled("/abort"), "MCP tunnel aborted", "the turn's own abort wakes the await"); - const draining = await open("/drain"); - sockets.push(draining); - draining.resume(); - assert.equal(await settled("/drain"), "drained", "a client that resumes reading resolves the await"); - } finally { - for (const socket of sockets) socket.destroy(); - await new Promise((resolve) => server.close(() => resolve())); - } -}); diff --git a/src/runtime/engineBrokerMcpFacade.ts b/src/runtime/engineBrokerMcpFacade.ts index 03eee1f..25a741f 100644 --- a/src/runtime/engineBrokerMcpFacade.ts +++ b/src/runtime/engineBrokerMcpFacade.ts @@ -1,6 +1,7 @@ import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import { Readable } from "node:stream"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; +import { EngineBrokerMcpCallLog, type EngineBrokerMcpCallObservation } from "./engineBrokerMcpCallLog.js"; /** * The brokered worker's only route to its own per-wake Daimon MCP mount. The @@ -53,6 +54,12 @@ export async function startEngineBrokerMcpFacade() { const targets = new Map(); /** In-flight upstream calls per turn, so a revoke or a close tears down any open SSE tunnel. */ const inflight = new Map>(); + /** + * What the facade saw of each turn's tool calls. A tool receipt is written + * only on completion, so without this a call that started and never returned + * and a call never made are the same absence (`engineBrokerMcpCallLog.ts`). + */ + const calls = new EngineBrokerMcpCallLog(); const server = createServer((request, response) => { void route(request, response).catch((error: unknown) => { @@ -76,8 +83,13 @@ export async function startEngineBrokerMcpFacade() { // Only POST carries a JSON-RPC body; drain anything else so the socket // never stalls waiting for a body the facade will not forward. - const body = method === "POST" ? await bounded(request) : (request.resume(), undefined); + let body: Buffer | undefined; + // A body refused for size is a call the log can never name, and counting + // it keeps "no tool call started" an honest reading rather than a gap. + try { body = method === "POST" ? await bounded(request) : (request.resume(), undefined); } + catch (error) { calls.undecodable(scope.turnId); throw error; } const payload = body === undefined ? undefined : (body.buffer.slice(body.byteOffset, body.byteOffset + body.byteLength) as ArrayBuffer); + const call = calls.begin(scope.turnId, body); const controller = new AbortController(); const open = inflight.get(scope.turnId) ?? new Set(); @@ -86,8 +98,12 @@ export async function startEngineBrokerMcpFacade() { const abort = (): void => controller.abort(); response.on("close", abort); try { - await forward(target, method, headersFor(method, request), payload, controller.signal, response); + // Answered only on a relay that reached its own end: a tunnel torn down + // by the worker's death must not mark the call it was blocked on as + // finished. + if (await forward(target, method, headersFor(method, request), payload, controller.signal, response)) call.answer(); } finally { + call.close(); response.off("close", abort); open.delete(controller); if (open.size === 0) inflight.delete(scope.turnId); @@ -110,13 +126,20 @@ export async function startEngineBrokerMcpFacade() { if (url.protocol !== "http:" || url.hostname !== "127.0.0.1" || url.pathname !== "/mcp") throw new TypeError("invalid scoped MCP mount"); if (targets.has(turnId)) throw new Error("MCP turn already registered"); targets.set(turnId, url.href); + calls.open(turnId); return capabilities.issue(agentId, turnId, 15 * 60_000, 128); }, revoke(turnId: string): void { targets.delete(turnId); capabilities.revoke(turnId); endTurnStreams(turnId); + calls.close(turnId); }, + /** + * What the facade saw of this turn's tool calls, or `undefined` for a turn + * it never registered. Read on the failure path, before `revoke`. + */ + observe: (turnId: string): EngineBrokerMcpCallObservation | undefined => calls.observe(turnId), close: async (): Promise => { for (const turnId of [...inflight.keys()]) endTurnStreams(turnId); await new Promise((resolve, reject) => { @@ -157,7 +180,7 @@ async function forward( body: ArrayBuffer | undefined, signal: AbortSignal, response: ServerResponse -): Promise { +): Promise { const upstream = await fetch(target, { method, headers, body, signal, redirect: "manual" }); // MCP never redirects, and following one would let the mount aim the facade // at a host the capability was never scoped to. `manual` also reports an @@ -171,7 +194,7 @@ async function forward( } outbound["content-type"] ??= "application/json"; response.writeHead(upstream.status, outbound); - if (upstream.body === null) { response.end(); return; } + if (upstream.body === null) { response.end(); return true; } const stream = Readable.fromWeb(upstream.body as Parameters[0]); try { for await (const chunk of stream) { @@ -179,10 +202,12 @@ async function forward( if (!response.write(chunk as Uint8Array)) await awaitMcpTunnelDrain(response, signal); } response.end(); + return true; } catch { // The client hung up or the mount's stream broke: tear the tunnel down // rather than leaving a half-written response open. response.destroy(); + return false; } finally { stream.destroy(); } diff --git a/src/runtime/engineBrokerProtocol.test.ts b/src/runtime/engineBrokerProtocol.test.ts index 2ce17e4..6f57c10 100644 --- a/src/runtime/engineBrokerProtocol.test.ts +++ b/src/runtime/engineBrokerProtocol.test.ts @@ -77,3 +77,28 @@ test("a failed worker's redacted reason is an optional bounded member of its dia {...failed,diagnostic:{...prelaunch,reason:"no worker ran"}} ])assert.throws(()=>parseEngineBrokerResponse(bad),/invalid broker frame/u); }); + +/** + * The in-flight tool-call observation (`engineBrokerMcpCallLog.ts`) rides the + * sealed failed frame, so the seam that carries the worker's last words and + * its accounting carries this too — nothing new on a tmpfs that dies with the + * container. It is names and timings, bounded, and internally consistent: a + * frame that claims more answered than started, or more outstanding than + * started minus answered, is a fabrication and is refused rather than clamped. + */ +test("a failed frame carries the broker's in-flight MCP tool-call observation, bounded and consistent", () => { + const value = { version: start.version, kind: "failed", requestId: "request-1", turnId: "turn-1", code: "limit_exceeded", mcpCalls: { started: 3, answered: 2, undecoded: 0, outstanding: [{ name: "daimon__moltnet_read", outstandingMs: 419_000 }] }, outcome: "failed", usage: null, model: "grok-4.6", requests: 8, limitReason: "timeout" } as const; + assert.deepEqual(parseEngineBrokerResponse(value), value); + for (const mcpCalls of [ + { ...value.mcpCalls, answered: 4 }, + { ...value.mcpCalls, started: 2 }, + { ...value.mcpCalls, outstanding: [{ name: "daimon__moltnet_read", outstandingMs: 1 }, { name: "memory_recall", outstandingMs: 1 }] }, + { ...value.mcpCalls, outstanding: [{ name: "daimon__moltnet_read", outstandingMs: -1 }] }, + { ...value.mcpCalls, outstanding: [{ name: "moltnet read; Bearer sk-live", outstandingMs: 1 }] }, + { ...value.mcpCalls, outstanding: [{ name: "daimon__moltnet_read", outstandingMs: 1, arguments: { text: "secret" } }] }, + { ...value.mcpCalls, outstanding: Array.from({ length: 17 }, () => ({ name: "tool", outstandingMs: 1 })) }, + { started: 3, answered: 2, outstanding: [] } + ]) assert.throws(() => parseEngineBrokerResponse({ ...value, mcpCalls }), /invalid broker frame/u, JSON.stringify(mcpCalls)); + // A v1 record predates the instrument; a v1 frame that carries it is forged. + assert.throws(() => parseEngineBrokerV1TerminalResponse({ version: "noopolis.daimon.engine-broker.v1", kind: "failed", requestId: "request-1", turnId: "turn-1", code: "engine_failed", mcpCalls: value.mcpCalls }), /invalid broker frame/u); +}); diff --git a/src/runtime/engineBrokerProtocol.ts b/src/runtime/engineBrokerProtocol.ts index 224ca44..643e41c 100644 --- a/src/runtime/engineBrokerProtocol.ts +++ b/src/runtime/engineBrokerProtocol.ts @@ -1,4 +1,5 @@ import { isEngineBrokerInferenceRequestKind, isEngineBrokerInferenceResponseKind, parseEngineBrokerInferenceRequest, parseEngineBrokerInferenceResponse, type EngineBrokerInferenceRequest, type EngineBrokerInferenceResponse } from "./engineBrokerInferenceProtocol.js"; +import { ENGINE_BROKER_MCP_CALL_NAME, ENGINE_BROKER_MCP_OUTSTANDING_MAX, type EngineBrokerMcpCallObservation } from "./engineBrokerMcpCallLog.js"; import { parseEngineBrokerTurnAccounting, parseEngineBrokerTurnLimitOverrides, type EngineBrokerTurnAccounting, type EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; /** @@ -34,7 +35,7 @@ export type EngineBrokerResponse = | Readonly<{ version: typeof VERSION; kind: "ready"; requestId: string; brokerUid: 2100; providerProxyPort: 43123; mcpFacadePort: 43124; registrations: number; credentialStale: false; realmLease: true; workerIsolation: true }> | Readonly<{ version: typeof VERSION; kind: "accepted"; requestId: string; turnId: string }> | (Readonly<{ version: typeof VERSION; kind: "completed"; requestId: string; turnId: string; text: string; workerPid: number; workerUid: number; workerStartTime: string }> & EngineBrokerTurnAccounting) - | (Readonly<{ version: typeof VERSION; kind: "failed"; requestId: string; turnId: string; code: EngineBrokerFailureCode; diagnostic?: EngineBrokerFailureDiagnostic }> & EngineBrokerTurnAccounting) + | (Readonly<{ version: typeof VERSION; kind: "failed"; requestId: string; turnId: string; code: EngineBrokerFailureCode; diagnostic?: EngineBrokerFailureDiagnostic; mcpCalls?: EngineBrokerMcpCallObservation }> & EngineBrokerTurnAccounting) | EngineBrokerInferenceResponse; /** * The one name for a fenced credential realm. The turn's failure code, the @@ -115,8 +116,12 @@ function parseTerminal(input: JsonRecord, expected: typeof VERSION | typeof V1): const base = { kind: "completed", requestId: id(input.requestId), turnId: id(input.turnId), text: text(input.text, 262_144), workerPid: input.workerPid as number, workerUid: input.workerUid as number, workerStartTime: id(input.workerStartTime) } as const; return expected === VERSION ? { version: VERSION, ...base, ...parseEngineBrokerTurnAccounting(input, "completed") } : { version: V1, ...base }; } - const fields = ["version", "kind", "requestId", "turnId", "code", ...accounting]; - exact(input, input.diagnostic === undefined ? fields : [...fields, "diagnostic"]); + // `mcpCalls` is the broker's own observation of the worker's tool calls + // (`engineBrokerMcpCallLog.ts`), additive in v2 and never part of a v1 + // record, which predates the instrument entirely. + if (expected === V1 && input.mcpCalls !== undefined) throw new TypeError("invalid broker frame"); + const fields = ["version", "kind", "requestId", "turnId", "code", ...accounting, ...(input.diagnostic === undefined ? [] : ["diagnostic"]), ...(input.mcpCalls === undefined ? [] : ["mcpCalls"])]; + exact(input, fields); const codes: readonly string[] = expected === VERSION ? ENGINE_BROKER_FAILURE_CODES : ENGINE_BROKER_FAILURE_CODES.filter((code) => code !== "limit_exceeded"); if (!codes.includes(input.code as string)) throw new TypeError("invalid broker frame"); let diagnostic:EngineBrokerFailureDiagnostic|undefined; @@ -125,7 +130,32 @@ function parseTerminal(input: JsonRecord, expected: typeof VERSION | typeof V1): if (expected === V1) return { version: V1, ...base } as V1Failed; const accountingValue = parseEngineBrokerTurnAccounting(input, "failed"); if ((input.code === "limit_exceeded") !== (accountingValue.limitReason !== "none")) throw new TypeError("invalid broker frame"); - return { version: VERSION, ...base, ...accountingValue }; + const mcpCalls = input.mcpCalls === undefined ? undefined : parseMcpCallObservation(input.mcpCalls); + return { version: VERSION, ...base, ...(mcpCalls === undefined ? {} : { mcpCalls }), ...accountingValue }; +} + +/** + * Names and timings, bounded, and internally consistent: a report can never + * claim more answered calls than started ones, nor more outstanding ones than + * started minus answered. Every count is its own measurement, so a missing + * field is refused rather than defaulted — a zero the broker did not measure + * would read exactly like one it did. + */ +function parseMcpCallObservation(value: unknown): EngineBrokerMcpCallObservation { + const input = record(value); + exact(input, ["started", "answered", "undecoded", "outstanding"]); + const started = input.started, answered = input.answered, undecoded = input.undecoded; + if (![started, answered, undecoded].every((count) => Number.isSafeInteger(count) && (count as number) >= 0)) throw new TypeError("invalid broker frame"); + if (!Array.isArray(input.outstanding) || input.outstanding.length > ENGINE_BROKER_MCP_OUTSTANDING_MAX) throw new TypeError("invalid broker frame"); + const outstanding = input.outstanding.map((entry) => { + const call = record(entry); + exact(call, ["name", "outstandingMs"]); + if (typeof call.name !== "string" || !ENGINE_BROKER_MCP_CALL_NAME.test(call.name)) throw new TypeError("invalid broker frame"); + if (!Number.isSafeInteger(call.outstandingMs) || (call.outstandingMs as number) < 0) throw new TypeError("invalid broker frame"); + return { name: call.name, outstandingMs: call.outstandingMs as number }; + }); + if ((answered as number) > (started as number) || outstanding.length > (started as number) - (answered as number)) throw new TypeError("invalid broker frame"); + return { started: started as number, answered: answered as number, undecoded: undecoded as number, outstanding }; } function closedDiagnostic(value:JsonRecord):boolean{ diff --git a/src/runtime/engineBrokerService.test.ts b/src/runtime/engineBrokerService.test.ts index c48a192..1dac5d5 100644 --- a/src/runtime/engineBrokerService.test.ts +++ b/src/runtime/engineBrokerService.test.ts @@ -62,3 +62,14 @@ test("a failed worker's own reason reaches the client instead of a bare exit cod await assert.rejects(client.turn("agent-a","wake-a","hello","http://127.0.0.1:44001/mcp"),/engine broker turn failed \(engine_failed; wait\/exec; exit=1; signal=0; reason=grok: session store unwritable\)/u); },async()=>{throw new EngineBrokerTurnFailure("engine_failed",{status:"worker_failed",stage:"wait",failureClass:"exec",profileApplied:false,exitCode:1,termSignal:0,workerPid:31,workerUid:2200,startTicks:"9",reason:"grok: session store unwritable"},{outcome:"failed",usage:null,model:"grok-4.6",requests:4,limitReason:"none"});}); }); + +/** + * The end of the seam: an outstanding tool call has to be readable by whoever + * reads the failure, not just sealed. The live hang would have read + * `mcp=1/2 answered; mcp_outstanding=daimon__moltnet_read@419000ms`. + */ +test("an outstanding MCP tool call reaches the client by name, with how long it waited", async () => { + await withService(async (client) => { + await assert.rejects(client.turn("agent-a","wake-a","hello","http://127.0.0.1:44001/mcp"),/engine broker turn failed \(limit_exceeded; limit=timeout; mcp=1\/2 answered; mcp_undecoded=1; mcp_outstanding=daimon__moltnet_read@419000ms\)/u); + },async()=>{throw new EngineBrokerTurnFailure("limit_exceeded",undefined,{outcome:"failed",usage:null,model:"grok-4.6",requests:8,limitReason:"timeout"},{started:2,answered:1,undecoded:1,outstanding:[{name:"daimon__moltnet_read",outstandingMs:419_000}]});}); +}); diff --git a/src/runtime/engineBrokerService.ts b/src/runtime/engineBrokerService.ts index aef6e33..79f55f0 100644 --- a/src/runtime/engineBrokerService.ts +++ b/src/runtime/engineBrokerService.ts @@ -54,7 +54,7 @@ function failed(socket:Socket,request:Extract & EngineBrokerTurnAccounting; export class EngineBrokerTurnFailure extends Error { - constructor(readonly code: Exclude, readonly diagnostic?: NativeBrokerDiagnostic, readonly accounting?: EngineBrokerTurnAccounting) { super("engine broker turn failed"); } + constructor(readonly code: Exclude, readonly diagnostic?: NativeBrokerDiagnostic, readonly accounting?: EngineBrokerTurnAccounting, readonly mcpCalls?: EngineBrokerMcpCallObservation) { super("engine broker turn failed"); } } /** Everything one broker turn touches, injected so the accounting and limit paths run under test without a native launcher. */ export type GrokEngineBrokerTurnDependencies = Readonly<{ turns: EngineBrokerTurnRegistry; proxy: Readonly<{ capabilities: Readonly<{ issue(agentId: string, turnId: string): string; revoke(turnId: string): void }>; registerIsolationGuard(turnId: string, guard: () => Promise): void; revokeIsolationGuard(turnId: string): void; registerTurn(turnId: string, turn: GrokBrokerProxyTurn): void; revokeTurn(turnId: string): void }>; - mcp: Readonly<{ register(agentId: string, turnId: string, endpoint: string): string; revoke(turnId: string): void }>; + mcp: Readonly<{ register(agentId: string, turnId: string, endpoint: string): string; revoke(turnId: string): void; observe?(turnId: string): EngineBrokerMcpCallObservation | undefined }>; credentialStale(): boolean; prepareIsolation(registration: EngineBrokerServiceRegistration): Promise<() => Promise>; runNative(input: NativeBrokerTurn, signal: AbortSignal): Promise>; @@ -91,10 +92,15 @@ export async function runGrokEngineBrokerTurn(deps: GrokEngineBrokerTurnDependen const diagnostic = error instanceof NativeBrokerTurnFailure ? error.diagnostic : nativeDiagnostic && !attested ? { ...nativeDiagnostic, status: "worker_failed" as const, stage: "attestation" as const, failureClass: error instanceof GrokWorkerAttestationFailure ? error.failureClass : "profile_invalid" as const, profileApplied: false } : undefined; const stream = output === undefined ? undefined : decodeGrokStreamUsage(output); const accounting = { outcome: "failed", usage: streamOrMeterUsage(stream, snapshot), model: declared, requests: requestCount(stream, snapshot), limitReason: snapshot.limitReason } as const; - const failed: EngineBrokerTerminalResponse = { version: request.version, kind: "failed", requestId: request.requestId, turnId, code, ...(diagnostic ? { diagnostic } : {}), ...accounting }; + // Read before the `finally` revokes this turn's MCP registration, and + // swallowed like every other instrument here: it must never be the reason + // a turn reports something other than why it failed. + let mcpCalls: EngineBrokerMcpCallObservation | undefined; + try { mcpCalls = deps.mcp.observe?.(turnId); } catch { mcpCalls = undefined; } + const failed: EngineBrokerTerminalResponse = { version: request.version, kind: "failed", requestId: request.requestId, turnId, code, ...(diagnostic ? { diagnostic } : {}), ...(mcpCalls === undefined ? {} : { mcpCalls }), ...accounting }; const reason = snapshot.limitReason !== "none" ? limitReasonFor[snapshot.limitReason] : rejected ? "turn_rejected" : "unknown"; await finishBrokerTurnWithUsage(deps.turns, request, failed, metering, { notionalUsd: 0, complete: false, reason, estimatedRequests: snapshot.estimatedRequests, requests: requestRows(stream, snapshot), ...(stream?.sessionId === undefined ? {} : { session: stream.sessionId }) }); - throw new EngineBrokerTurnFailure(code, diagnostic, accounting); + throw new EngineBrokerTurnFailure(code, diagnostic, accounting, mcpCalls); } finally { clearTimeout(timer); signal?.removeEventListener("abort", onAbort); meter.abortInFlight(); deps.proxy.revokeTurn(turnId); deps.proxy.revokeIsolationGuard(turnId); deps.proxy.capabilities.revoke(turnId); deps.mcp.revoke(turnId); @@ -105,7 +111,7 @@ function replay(response: EngineBrokerTerminalResponse): GrokEngineBrokerTurnRes const accounting = { outcome: response.outcome, usage: response.usage, model: response.model, requests: response.requests, limitReason: response.limitReason }; if (response.kind === "completed") return { text: response.text, workerPid: response.workerPid, workerUid: response.workerUid, workerStartTime: response.workerStartTime, ...accounting, outcome: "completed" }; const code = response.code === "turn_conflict" || response.code === "unavailable" ? "engine_failed" : response.code; - throw new EngineBrokerTurnFailure(code, response.diagnostic as NativeBrokerDiagnostic | undefined, accounting); + throw new EngineBrokerTurnFailure(code, response.diagnostic as NativeBrokerDiagnostic | undefined, accounting, response.mcpCalls); } /** The proxy saw every forwarded request; the stream is the fallback when no request crossed this proxy. */ diff --git a/src/runtime/grokEngineBrokerUsage.test.ts b/src/runtime/grokEngineBrokerUsage.test.ts index 8f0f50d..b8d6701 100644 --- a/src/runtime/grokEngineBrokerUsage.test.ts +++ b/src/runtime/grokEngineBrokerUsage.test.ts @@ -11,6 +11,7 @@ import { decodeNativeBrokerResult, ENGINE_BROKER_NATIVE_RESULT_BYTES, type Nativ import type { EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; import { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; import { startGrokBrokerProxy } from "./grokBrokerProxy.js"; +import type { EngineBrokerMcpCallObservation } from "./engineBrokerMcpCallLog.js"; import { EngineBrokerTurnFailure, runGrokEngineBrokerTurn, type GrokEngineBrokerTurnDependencies } from "./grokEngineBrokerTurn.js"; import { TURN_REQUEST_LEDGER_VERSION } from "./turnRequestLedger.js"; import { dedupeTurnUsageRows, TURN_USAGE_LEDGER_VERSION } from "./turnUsageLedger.js"; @@ -69,7 +70,7 @@ const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId const registration: EngineBrokerServiceRegistration = { agentId: "foreman", slot: 0, workerUid: 2_200, workspace: "/workspace", profilePath: "/workers/0/.grok/sandbox.toml", eventsPath: "/workers/0/.grok/sessions/sandbox-events.jsonl", profileSha256: "a".repeat(64), usageLedgerPath: ledger, limits, model: { model: "grok-4.6", reasoningEffort: "low" } }; const deps: GrokEngineBrokerTurnDependencies = { turns: syncDirectory === undefined ? new EngineBrokerTurnRegistry(turnStore) : new EngineBrokerTurnRegistry(turnStore, undefined, syncDirectory), proxy, credentialStale: () => false, - mcp: { register: () => "mcp-capability-0123456789abcdef", revoke: () => undefined }, + mcp: { register: () => "mcp-capability-0123456789abcdef", revoke: () => undefined, observe: () => mcpObservation }, prepareIsolation: async () => async () => undefined, runNative: async (input: NativeBrokerTurn, signal: AbortSignal) => nativeResult(await worker(() => post(proxy.port, input.providerCapability), signal)) }; @@ -83,6 +84,9 @@ const withBroker = async (body: (context: Readonly<{ root: string; turn: (wakeId } finally { await proxy.close(); await rm(root, { recursive: true, force: true }); } }; +/** What the facade would have seen of this turn's tool calls; the turn only reads it. */ +let mcpObservation: EngineBrokerMcpCallObservation | undefined; + const twoRequests: Worker = async (send) => { assert.equal(await send(), 200); assert.equal(await send(), 200); return stream(); }; test("a completed turn seals its accounting, writes one usage row and per-request rows, and a replay never re-meters", async () => { @@ -344,3 +348,35 @@ test("a worker whose work succeeded but whose output crossed the launcher bound assert.equal((await usageRows()).length, 1, "the replayed failure is not metered again"); }); }); + +/** + * The hang this instrument was built for: the worker stops acting with every + * provider request closed, the deadline kills it, and the only remaining + * question is whether it was waiting on a tool call. The answer has to reach + * the host, and the slot's control root is tmpfs that dies with the container — + * so it rides the seam the worker's last words and the sealed usage already + * ride: the sealed terminal response, which a replay hands back unchanged. + */ +test("a failed turn carries the facade's in-flight tool-call observation, and its replay still does", async () => { + await withBroker(async ({ turn }) => { + mcpObservation = { started: 3, answered: 2, undecoded: 0, outstanding: [{ name: "daimon__moltnet_read", outstandingMs: 419_000 }] }; + const worker: Worker = async (send) => { assert.equal(await send(), 200); assert.equal(await send(), 200); throw new Error("engine broker turn failed"); }; + const carried = (error: unknown): boolean => { + assert.ok(error instanceof EngineBrokerTurnFailure); + assert.deepEqual(error.mcpCalls, { started: 3, answered: 2, undecoded: 0, outstanding: [{ name: "daimon__moltnet_read", outstandingMs: 419_000 }] }); + return true; + }; + await assert.rejects(turn("wake-mcp-outstanding", worker), carried); + // The replay reads the durable record back through the frame parser, so + // this is the sealed bytes answering, not the live facade. + mcpObservation = undefined; + await assert.rejects(turn("wake-mcp-outstanding", worker), carried); + }); +}); + +test("a turn whose facade observed nothing seals no observation at all", async () => { + await withBroker(async ({ turn }) => { + mcpObservation = undefined; + await assert.rejects(turn("wake-mcp-absent", async (send) => { await send(); throw new Error("engine broker turn failed"); }), (error: unknown) => error instanceof EngineBrokerTurnFailure && error.mcpCalls === undefined); + }); +}); From 1ec76f2286e844ff620d4bbaa7604363a74197ff Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 11:59:22 +0200 Subject: [PATCH 53/69] feat: seal a brokered turn's terminal evidence into the broker's ledger directory --- src/runtime/AGENTS.md | 34 ++++++ src/runtime/engineBrokerSealLedger.test.ts | 122 +++++++++++++++++++++ src/runtime/engineBrokerSealLedger.ts | 109 ++++++++++++++++++ src/runtime/engineBrokerServiceConfig.ts | 13 ++- src/runtime/grokEngineBrokerLedger.ts | 47 ++++++-- src/runtime/grokEngineBrokerMetering.ts | 7 +- src/runtime/grokEngineBrokerTurn.ts | 4 +- src/runtime/index.ts | 3 +- 8 files changed, 322 insertions(+), 17 deletions(-) create mode 100644 src/runtime/engineBrokerSealLedger.test.ts create mode 100644 src/runtime/engineBrokerSealLedger.ts diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 5a158bb..f4243e1 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -314,6 +314,40 @@ answer, so the call it was blocked on stays outstanding with the elapsed time it had reached — otherwise the turn's death would erase the evidence the instrument exists to keep. +That seam is enough for a turn that *fails with a reply* and not for the turn +the instrument was built for. A worker that crashes still produces a terminal +response; a worker that HANGS is cancelled by its client's deadline, and a +cancelled turn has no client left to answer, so the sealed response — with +`mcpCalls` and the worker's redacted last words riding on it — is sealed into a +turn record in the broker's own `0700` turn store and dies with the slot's +tmpfs. Six live runs reproduced that exactly. What *does* survive a slot is the +broker's ledger directory, which Paideia already recovers `usage.jsonl` and +`requests.jsonl` from on the failure path, so `engineBrokerSealLedger.ts` writes +a third stream beside them: one `noopolis.daimon.turn-seal.v1` row per sealed +terminal turn (`turns.jsonl`, `engineBrokerSealLedgerPathFor`), rendered from +the sealed response and nothing else. Its members are the accounting, the +failure `code`, the diagnostic's closed `status`/`stage`/`failure_class` with +the reason `engineBrokerNativeClient.ts` already redacted and bounded, and the +facade's `mcp` observation — names, counts and elapsed milliseconds. Never a +prompt, body, reply, bearer, capability or session id; the terminal response +carries none of those in the first place, and the projection is an allow-list +rather than a spread, so a future additive member of the response cannot become +a ledger field by accident. + +Two invariants make it worth having. The row is rendered for *every* terminal +turn including one whose `usage` is `null` — a turn cancelled before any spend +could be attributed writes no usage row at all, and is precisely the turn whose +outstanding call has no other route out. And absence stays absence three ways: +no `mcp` member when the facade never observed the turn, `started: 0` when it +observed a turn that called nothing, and no row when nothing sealed. Reading +any of those three as another is the failure this stream exists to prevent. The +line is sealed into the turn record's ledger bytes with the other two and +appended last, so a replay completes an interrupted append the same way and +readers dedupe on `turn`; `seal` is optional in `parseBrokerTurnLedgerLines`, so +a record written before the stream existed still replays. It is advisory +throughout: `recordLedgerLines` swallows every I/O fault, and nothing here can +refuse, delay or fail a turn. + Worker `GROK_HOME` layout the deployment must provision (attested before every turn by `grokWorkerHomeAttestation.ts`, recorded in `GROK_ENGINE_BROKER.worker.home`): `$GROK_HOME` and `$GROK_HOME/sessions` `root: 1771`; `config.toml`, diff --git a/src/runtime/engineBrokerSealLedger.test.ts b/src/runtime/engineBrokerSealLedger.test.ts new file mode 100644 index 0000000..9b5ecc2 --- /dev/null +++ b/src/runtime/engineBrokerSealLedger.test.ts @@ -0,0 +1,122 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import type { EngineBrokerMcpCallObservation } from "./engineBrokerMcpCallLog.js"; +import { engineBrokerSealLedgerPathFor, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +import { TURN_SEAL_LEDGER_VERSION } from "./engineBrokerSealLedger.js"; +import { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; +import { EngineBrokerTurnFailure, runGrokEngineBrokerTurn, type GrokEngineBrokerTurnDependencies } from "./grokEngineBrokerTurn.js"; + +/** + * The hung turn's evidence, across the one boundary it has to cross. + * + * A cancelled or timed-out turn never answers its client, so the sealed + * terminal response — the only carrier of `mcpCalls` and the worker's redacted + * last words — dies with the slot. These tests pin the other route: the same + * sealed response, projected into the broker's own ledger directory, which + * Paideia already recovers `usage.jsonl` and `requests.jsonl` from. + * + * The turn runs through its real registry, meter, seal and ledger path; only + * the launcher, the proxy registrations and the MCP facade's observation are + * stubbed, exactly as the live hang presented them. + */ +const registration = (usageLedgerPath: string): EngineBrokerServiceRegistration => ({ + agentId: "foreman", slot: 0, workerUid: 2_200, workspace: "/workspace", + profilePath: "/workers/0/.grok/sandbox.toml", eventsPath: "/workers/0/.grok/sessions/sandbox-events.jsonl", + profileSha256: "a".repeat(64), usageLedgerPath, limits: { maxRequests: 32, maxTokens: 300_000, timeoutMs: 240_000 }, + model: { model: "grok-4.6", reasoningEffort: "low" } +}); + +const proxy: GrokEngineBrokerTurnDependencies["proxy"] = { + capabilities: { issue: () => "provider-capability-0123456789ab", revoke: () => undefined }, + registerIsolationGuard: () => undefined, revokeIsolationGuard: () => undefined, + registerTurn: () => undefined, revokeTurn: () => undefined +}; + +/** A worker that does real work and then stops acting, until the deadline aborts it. */ +const hangs = (signal: AbortSignal): Promise => new Promise((_resolve, reject) => { + const fail = (): void => reject(new Error("engine broker turn failed")); + if (signal.aborted) fail(); else signal.addEventListener("abort", fail, { once: true }); +}); + +async function cancelledTurn(observe: () => EngineBrokerMcpCallObservation | undefined): Promise | undefined; usage: string; failure: EngineBrokerTurnFailure }>> { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-seal-")); + try { + const usageLedgerPath = path.join(root, "usage.jsonl"); + const deps: GrokEngineBrokerTurnDependencies = { + turns: new EngineBrokerTurnRegistry(path.join(root, "turns")), proxy, credentialStale: () => false, + mcp: { register: () => "mcp-capability-0123456789abcdef", revoke: () => undefined, observe }, + prepareIsolation: async () => async () => undefined, + runNative: async (_input, signal) => hangs(signal) + }; + const controller = new AbortController(); + const running = runGrokEngineBrokerTurn(deps, registration(usageLedgerPath), "wake-hung", "prompt", "http://127.0.0.1:43124/mcp", controller.signal); + setTimeout(() => controller.abort(), 5); + const failure = await running.then(() => { throw new Error("the hung turn resolved"); }, (error: unknown) => error as EngineBrokerTurnFailure); + const text = await readFile(engineBrokerSealLedgerPathFor(usageLedgerPath), "utf8").catch(() => ""); + const lines = text.split("\n").filter((line) => line.length > 0); + assert.ok(lines.length <= 1, "one sealed turn writes at most one seal row"); + return { seal: lines[0] === undefined ? undefined : JSON.parse(lines[0]) as Record, usage: await readFile(usageLedgerPath, "utf8").catch(() => ""), failure }; + } finally { await rm(root, { recursive: true, force: true }); } +} + +/** + * The finding this whole instrument exists for, on the host side of the seam. + * + * Live: ten provider requests, all closed, one tool receipt, then 430 s of + * silence and an abort at 489 s. The facade's log says the worker was blocked + * on `use_tool` the whole time, the sealed response carries it — and the + * sealed response never travels, because a cancelled turn has no client left to + * answer. The row in `turns.jsonl` is that evidence on a durable file the + * evaluator already reads. + * + * Mutation: drop the `seal` append from `appendBrokerTurnLedger`, or the `mcp` + * member from `renderBrokerTurnSealLine`, and this goes red. So does reverting + * `renderBrokerTurnLedger` to return `EMPTY_BROKER_TURN_LEDGER` for a turn with + * no attributable usage — which is exactly this turn. + */ +test("a cancelled turn's outstanding MCP call reaches the host on the broker's own ledger directory", async () => { + const { seal, usage, failure } = await cancelledTurn(() => ({ started: 1, answered: 0, undecoded: 0, outstanding: [{ name: "use_tool", outstandingMs: 430_112 }] })); + assert.equal(failure.code, "cancelled"); + // The usage ledger stays silent for a turn with nothing to attribute, so the + // seal row is the only host-visible account this turn has. + assert.equal(usage, ""); + assert.ok(seal, "a cancelled turn seals a row"); + assert.equal(seal.v, TURN_SEAL_LEDGER_VERSION); + assert.equal(seal.agent, "foreman"); + assert.equal(seal.wake, "wake-hung"); + assert.equal(seal.outcome, "failed"); + assert.equal(seal.code, "cancelled"); + assert.equal(seal.limit_reason, "none"); + assert.equal(seal.model, "grok-4.6"); + assert.deepEqual(seal.mcp, { started: 1, answered: 0, undecoded: 0, outstanding: [{ name: "use_tool", outstanding_ms: 430_112 }] }); +}); + +/** + * Absence stays absence, and the two absences are not the same statement. + * + * A turn that called no tool measured `started: 0`; a turn the facade never + * registered measured nothing at all and writes no `mcp` member; a turn that + * never sealed writes no row. Reading them as one another is precisely the + * mistake this session kept making. + * + * Mutation: render `mcp` unconditionally (as `{}` or as zeros) when the + * observation is `undefined`, and the second assertion goes red. + */ +test("a cancelled turn that called no tool is distinguishable from one the facade never observed, and both from no row at all", async () => { + const called = await cancelledTurn(() => ({ started: 0, answered: 0, undecoded: 0, outstanding: [] })); + assert.deepEqual(called.seal?.mcp, { started: 0, answered: 0, undecoded: 0, outstanding: [] }); + + const unobserved = await cancelledTurn(() => undefined); + assert.ok(unobserved.seal, "an unobserved turn still seals its row"); + assert.equal(Object.hasOwn(unobserved.seal, "mcp"), false); + + // And the third state: no seal row at all, which is what every one of the six + // live runs published before this stream existed. + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-seal-none-")); + try { await assert.rejects(readFile(engineBrokerSealLedgerPathFor(path.join(root, "usage.jsonl")), "utf8")); } + finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/src/runtime/engineBrokerSealLedger.ts b/src/runtime/engineBrokerSealLedger.ts new file mode 100644 index 0000000..52d38c1 --- /dev/null +++ b/src/runtime/engineBrokerSealLedger.ts @@ -0,0 +1,109 @@ +import { ENGINE_BROKER_MCP_CALL_NAME, ENGINE_BROKER_MCP_OUTSTANDING_MAX } from "./engineBrokerMcpCallLog.js"; +import type { EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; +import { TURN_USAGE_LEDGER, TURN_USAGE_MAX_IDENTIFIER_CHARS } from "./turnUsageLedger.js"; + +/** + * The operator-visible half of a sealed terminal turn, as a durable row. + * + * The seal itself already carries everything an operator needs to read a turn + * that stopped acting — the failure code, the worker's own redacted last words, + * and `engineBrokerMcpCallLog.ts`'s observation of the tool call the worker was + * blocked on. All of it travels in the control-protocol terminal response, and + * that response is the one artifact a *hung* turn never produces: the client is + * gone before the broker answers, the turn registry record lives in the + * broker's own `0700` turn store, and a training slot's control root is tmpfs + * that dies with the container. Six live runs reproduced the same signature and + * none of them could read the instrument built for it. + * + * What does survive a slot is the broker's ledger directory: Paideia already + * recovers `usage.jsonl` and `requests.jsonl` from it on the failure path. So + * this is a third stream beside those two, written by the broker — still the + * single sealed usage writer — from the sealed response and nothing else, at + * the moment that response is sealed. + * + * Its rules are the two ledgers' rules: + * + * - **Numbers, names, timings and closed vocabularies only.** The failure code, + * the accounting, the diagnostic's closed `status`/`stage`/`failure_class` + * and its already-redacted, already-bounded, control-character-free `reason` + * (`engineBrokerNativeClient.ts` produced it; nothing here re-derives it), + * plus tool-call names and elapsed milliseconds. Never a prompt, a body, a + * reply, a bearer, a capability or a session id — none of which the terminal + * response carries in the first place. + * - **Absence stays absence.** `mcp` is written only when the facade actually + * observed the turn, `diagnostic` only when the sealed response carried one, + * and `code` only for a failure. A turn that called no tool publishes + * `started: 0`, which is a measurement; a turn the facade never registered + * publishes no `mcp` member at all, which is not. + * - **It can never fail a turn.** The row is rendered from an + * already-validated frame and appended through `recordLedgerLines`, which + * swallows every I/O fault. + * + * A separate stream and a separate `v`, for `turnRequestLedger.ts`'s reason: + * Spawnfile's reader pins `noopolis.daimon.turn-usage.v1` and drops any other + * `v` outright, and Paideia's request reader refuses a row it cannot type. A + * row in a new file is invisible to both. + */ +export const TURN_SEAL_LEDGER_VERSION = "noopolis.daimon.turn-seal.v1" as const; + +/** Default location: beside `usage.jsonl` and `requests.jsonl`, no new mount. */ +export const TURN_SEAL_LEDGER = { + version: TURN_SEAL_LEDGER_VERSION, + directoryPath: TURN_USAGE_LEDGER.directoryPath, + filePath: `${TURN_USAGE_LEDGER.directoryPath}/turns.jsonl`, + rotatedFilePath: `${TURN_USAGE_LEDGER.directoryPath}/turns.jsonl.1`, + fileMode: TURN_USAGE_LEDGER.fileMode +} as const; + +/** A rendered seal line is bounded by its own contents: a 768-byte reason plus 16 bounded names. */ +export const TURN_SEAL_MAX_LINE_BYTES = 8_192; + +export type BrokerTurnSealEntry = Readonly<{ agent: string; wake: string; at: string }>; + +const bounded = (value: string): string => [...value].slice(0, TURN_USAGE_MAX_IDENTIFIER_CHARS).join(""); + +/** + * One newline-terminated row for one sealed terminal turn. + * + * Every member is copied from the terminal response the protocol already + * validated, so this projection cannot widen what the response admits; it is an + * allow-list rather than a spread, so a future additive member of the response + * does not silently become a ledger field. + */ +export const renderBrokerTurnSealLine = (terminal: EngineBrokerTerminalResponse, entry: BrokerTurnSealEntry): string => `${JSON.stringify({ + v: TURN_SEAL_LEDGER_VERSION, + agent: bounded(entry.agent), + wake: bounded(entry.wake), + engine: "grok", + at: entry.at, + turn: terminal.turnId, + outcome: terminal.outcome, + requests: terminal.requests, + model: terminal.model, + limit_reason: terminal.limitReason, + ...(terminal.kind === "failed" ? { code: terminal.code } : {}), + ...(terminal.kind === "failed" && terminal.diagnostic !== undefined + ? { + diagnostic: { + status: terminal.diagnostic.status, + stage: terminal.diagnostic.stage, + failure_class: terminal.diagnostic.failureClass, + exit_code: terminal.diagnostic.exitCode, + term_signal: terminal.diagnostic.termSignal, + ...(terminal.diagnostic.reason === undefined ? {} : { reason: terminal.diagnostic.reason }) + } + } + : {}), + ...(terminal.kind === "failed" && terminal.mcpCalls !== undefined + ? { + mcp: { + started: terminal.mcpCalls.started, + answered: terminal.mcpCalls.answered, + undecoded: terminal.mcpCalls.undecoded, + outstanding: terminal.mcpCalls.outstanding + .slice(0, ENGINE_BROKER_MCP_OUTSTANDING_MAX) + .map((call) => ({ name: ENGINE_BROKER_MCP_CALL_NAME.test(call.name) ? call.name : "", outstanding_ms: call.outstandingMs })) + } + } + : {}) +})}\n`; diff --git a/src/runtime/engineBrokerServiceConfig.ts b/src/runtime/engineBrokerServiceConfig.ts index af2a159..eb6d274 100644 --- a/src/runtime/engineBrokerServiceConfig.ts +++ b/src/runtime/engineBrokerServiceConfig.ts @@ -12,7 +12,7 @@ export const ENGINE_BROKER_SERVICE_V2 = "noopolis.daimon.engine-broker-service.v /** One root-provisioned broker slot. Every field is fixed at provisioning time; a wake can only lower `limits`. */ export type EngineBrokerServiceRegistration = Readonly<{ agentId: string; slot: number; workerUid: number; workspace: string; profilePath: string; eventsPath: string; profileSha256: string; - /** Per-slot usage ledger the broker appends turn rows to; per-request rows go to `requests.jsonl` beside it. */ + /** Per-slot usage ledger the broker appends turn rows to; per-request rows go to `requests.jsonl` and per-turn seals to `turns.jsonl` beside it. */ usageLedgerPath: string; limits: EngineBrokerTurnLimits; model: GrokBrokerModelPolicy; @@ -41,6 +41,13 @@ const absolute = (item: unknown): item is string => typeof item === "string" && /** The per-request stream written beside a registration's usage ledger. */ export const engineBrokerRequestLedgerPathFor = (usageLedgerPath: string): string => path.posix.join(path.posix.dirname(usageLedgerPath), "requests.jsonl"); +/** + * The per-turn seal stream written beside the other two + * (`engineBrokerSealLedger.ts`): the operator-visible half of a sealed terminal + * response, for the turn whose response never reaches a client. + */ +export const engineBrokerSealLedgerPathFor = (usageLedgerPath: string): string => path.posix.join(path.posix.dirname(usageLedgerPath), "turns.jsonl"); + /** * Strict `service.json` parser. * @@ -67,7 +74,7 @@ export function parseEngineBrokerServiceConfig(value: unknown): EngineBrokerServ const base = { agentId, slot: slot as number, workerUid: workerUid as number, workspace, profilePath, eventsPath, profileSha256 }; if (!v2) return { ...base, usageLedgerPath: TURN_USAGE_LEDGER.filePath, limits: DEFAULT_GROK_BROKER_TURN_LIMITS, model: DEFAULT_GROK_BROKER_MODEL_POLICY }; const usageLedgerPath = entry.usageLedgerPath; - if (!ledgerPath(usageLedgerPath) || usageLedgerPath === engineBrokerRequestLedgerPathFor(usageLedgerPath)) throw invalid(); + if (!ledgerPath(usageLedgerPath) || usageLedgerPath === engineBrokerRequestLedgerPathFor(usageLedgerPath) || usageLedgerPath === engineBrokerSealLedgerPathFor(usageLedgerPath)) throw invalid(); let limits: EngineBrokerTurnLimits; try { limits = parseEngineBrokerTurnLimits(entry.limits); } catch { throw invalid(); } return { ...base, usageLedgerPath, limits, model: parseServiceModel(entry.model) }; @@ -76,7 +83,7 @@ export function parseEngineBrokerServiceConfig(value: unknown): EngineBrokerServ if (!Object.hasOwn(value, "inferenceLedgerPath")) return base; const inferenceLedgerPath = value.inferenceLedgerPath; if (!ledgerPath(inferenceLedgerPath)) throw invalid(); - const subject = new Set([TURN_USAGE_LEDGER.filePath, TURN_REQUEST_LEDGER.filePath, ...registrations.flatMap((entry) => [entry.usageLedgerPath, engineBrokerRequestLedgerPathFor(entry.usageLedgerPath)])]); + const subject = new Set([TURN_USAGE_LEDGER.filePath, TURN_REQUEST_LEDGER.filePath, ...registrations.flatMap((entry) => [entry.usageLedgerPath, engineBrokerRequestLedgerPathFor(entry.usageLedgerPath), engineBrokerSealLedgerPathFor(entry.usageLedgerPath)])]); if (subject.has(inferenceLedgerPath)) throw invalid(); return { ...base, inferenceLedgerPath }; } diff --git a/src/runtime/grokEngineBrokerLedger.ts b/src/runtime/grokEngineBrokerLedger.ts index ebf6073..3d21e47 100644 --- a/src/runtime/grokEngineBrokerLedger.ts +++ b/src/runtime/grokEngineBrokerLedger.ts @@ -1,5 +1,6 @@ import { readFile } from "node:fs/promises"; +import { renderBrokerTurnSealLine, TURN_SEAL_LEDGER_VERSION, TURN_SEAL_MAX_LINE_BYTES } from "./engineBrokerSealLedger.js"; import { recordLedgerLines, renderGrokTurnRequestLines, TURN_REQUEST_LEDGER_VERSION, type GrokTurnRequest } from "./turnRequestLedger.js"; import { renderTurnUsageLine, TURN_USAGE_LEDGER_VERSION, type TurnUsageFailureReason } from "./turnUsageLedger.js"; import type { EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; @@ -16,15 +17,23 @@ import type { EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; * normal append finds the row and writes nothing, and readers dedupe on `turn` * should two replays race. */ -export type BrokerTurnLedgerLines = Readonly<{ usage: string | null; requests: string }>; +export type BrokerTurnLedgerLines = Readonly<{ usage: string | null; requests: string; seal?: string }>; +/** A v1 record, or a replay with no sealed bytes at all: nothing to append, including no seal. */ export const EMPTY_BROKER_TURN_LEDGER: BrokerTurnLedgerLines = Object.freeze({ usage: null, requests: "" }); export type BrokerTurnLedgerDetail = Readonly<{ agentId: string; wakeId: string; notionalUsd: number; complete: boolean; reason?: TurnUsageFailureReason; requests: readonly GrokTurnRequest[]; session?: string; estimatedRequests: number }>; export function renderBrokerTurnLedger(terminal: EngineBrokerTerminalResponse, detail: BrokerTurnLedgerDetail): BrokerTurnLedgerLines { - if (terminal.usage === null) return EMPTY_BROKER_TURN_LEDGER; - const { usage } = terminal, at = new Date().toISOString(); + const at = new Date().toISOString(); + // The seal row is rendered for *every* terminal turn, including one whose + // usage is null. That is the whole point: a turn cancelled before any usage + // could be attributed is exactly the turn whose outstanding MCP call and + // redacted last words have no other route to the host. + const seal = renderBrokerTurnSealLine(terminal, { agent: detail.agentId, wake: detail.wakeId, at }); + if (terminal.usage === null) return { usage: null, requests: "", seal }; + const { usage } = terminal; return { + seal, usage: renderTurnUsageLine({ agent: detail.agentId, wake: detail.wakeId, engine: "grok", at, usage: { input: usage.input, output: usage.output, cacheRead: usage.cacheRead, cacheWrite: usage.cacheWrite, total: usage.total, calls: terminal.requests, notionalUsd: detail.notionalUsd, complete: detail.complete }, @@ -38,11 +47,20 @@ export function renderBrokerTurnLedger(terminal: EngineBrokerTerminalResponse, d const MAX_USAGE_LINE_BYTES = 4_096, MAX_REQUEST_LINES_BYTES = 262_144; const rows = (text: string): Record[] => text.split("\n").filter((line) => line.length > 0).map((line) => { const value: unknown = JSON.parse(line); if (value === null || typeof value !== "object" || Array.isArray(value)) throw new Error(); return value as Record; }); -/** Strict check of stored ledger bytes: exactly the turn's own rows, newline-terminated, bounded. */ +/** + * Strict check of stored ledger bytes: exactly the turn's own rows, + * newline-terminated, bounded. + * + * `seal` is optional so a record sealed before this stream existed still + * replays; its absence means the turn owes no seal row, never that one was + * lost. + */ export function parseBrokerTurnLedgerLines(value: unknown, turnId: string): BrokerTurnLedgerLines { const invalid = () => new Error("broker turn registry unavailable"); - if (value === null || typeof value !== "object" || Array.isArray(value) || Object.keys(value).length !== 2 || !Object.hasOwn(value, "usage") || !Object.hasOwn(value, "requests")) throw invalid(); - const { usage, requests } = value as { usage: unknown; requests: unknown }; + if (value === null || typeof value !== "object" || Array.isArray(value) || !Object.hasOwn(value, "usage") || !Object.hasOwn(value, "requests")) throw invalid(); + const sealed = Object.hasOwn(value, "seal"); + if (Object.keys(value).length !== (sealed ? 3 : 2)) throw invalid(); + const { usage, requests, seal } = value as { usage: unknown; requests: unknown; seal?: unknown }; try { if (usage !== null) { if (typeof usage !== "string" || !usage.endsWith("\n") || Buffer.byteLength(usage) > MAX_USAGE_LINE_BYTES) throw invalid(); @@ -51,21 +69,32 @@ export function parseBrokerTurnLedgerLines(value: unknown, turnId: string): Brok } if (typeof requests !== "string" || (requests.length > 0 && (usage === null || !requests.endsWith("\n"))) || Buffer.byteLength(requests) > MAX_REQUEST_LINES_BYTES) throw invalid(); if (rows(requests).some((row) => row.v !== TURN_REQUEST_LEDGER_VERSION || row.turn !== turnId)) throw invalid(); + if (sealed) { + if (typeof seal !== "string" || !seal.endsWith("\n") || Buffer.byteLength(seal) > TURN_SEAL_MAX_LINE_BYTES) throw invalid(); + const parsed = rows(seal); + if (parsed.length !== 1 || parsed[0]!.v !== TURN_SEAL_LEDGER_VERSION || parsed[0]!.turn !== turnId) throw invalid(); + } } catch { throw invalid(); } - return { usage: usage as string | null, requests }; + return { usage: usage as string | null, requests, ...(sealed ? { seal: seal as string } : {}) }; } +export type BrokerTurnLedgerPaths = Readonly<{ usageLedgerPath: string; requestLedgerPath: string; sealLedgerPath: string }>; + /** Appends sealed lines on the first metering: no presence scan is needed, nothing was appended before the record existed. */ -export async function appendBrokerTurnLedger(lines: BrokerTurnLedgerLines, paths: Readonly<{ usageLedgerPath: string; requestLedgerPath: string }>): Promise { +export async function appendBrokerTurnLedger(lines: BrokerTurnLedgerLines, paths: BrokerTurnLedgerPaths): Promise { if (lines.usage !== null) await recordLedgerLines(paths.usageLedgerPath, lines.usage); await recordLedgerLines(paths.requestLedgerPath, lines.requests); + // Last, and never conditional on usage: a turn with no attributable spend is + // precisely the one whose seal row is its only account of itself. + if (lines.seal !== undefined) await recordLedgerLines(paths.sealLedgerPath, lines.seal); } /** On replay: append each stream's sealed lines only when that stream (current file or its `.1`) holds no row for this turn. Never rejects. */ -export async function ensureBrokerTurnLedgered(lines: BrokerTurnLedgerLines, turnId: string, paths: Readonly<{ usageLedgerPath: string; requestLedgerPath: string }>): Promise { +export async function ensureBrokerTurnLedgered(lines: BrokerTurnLedgerLines, turnId: string, paths: BrokerTurnLedgerPaths): Promise { try { if (lines.usage !== null && !await ledgerHasTurn(paths.usageLedgerPath, turnId)) await recordLedgerLines(paths.usageLedgerPath, lines.usage); if (lines.requests.length > 0 && !await ledgerHasTurn(paths.requestLedgerPath, turnId)) await recordLedgerLines(paths.requestLedgerPath, lines.requests); + if (lines.seal !== undefined && !await ledgerHasTurn(paths.sealLedgerPath, turnId)) await recordLedgerLines(paths.sealLedgerPath, lines.seal); } catch { /* advisory: a replay never fails on its ledger */ } } diff --git a/src/runtime/grokEngineBrokerMetering.ts b/src/runtime/grokEngineBrokerMetering.ts index 8b8fb31..d0df740 100644 --- a/src/runtime/grokEngineBrokerMetering.ts +++ b/src/runtime/grokEngineBrokerMetering.ts @@ -7,6 +7,8 @@ import type { TurnUsageFailureReason } from "./turnUsageLedger.js"; export type BrokerTurnMetering = Readonly<{ usageLedgerPath: string; requestLedgerPath: string; + /** Where the sealed response's operator-visible projection is appended (`engineBrokerSealLedger.ts`). */ + sealLedgerPath: string; agentId: string; wakeId: string; }>; @@ -31,8 +33,9 @@ export type BrokerTurnMeteringDetail = Readonly<{ notionalUsd: number; complete: * * Both terminal kinds meter: a failed turn spent real tokens, so its partial * usage is written with `outcome: failed` and its closed `limitReason`. A turn - * with no usage at all (`usage: null`) writes nothing — a zero row is - * byte-identical to a measured zero. + * with no usage at all (`usage: null`) writes no *usage* row — a zero row is + * byte-identical to a measured zero — but it still writes its seal row, which + * is a record of what the turn did rather than of what it spent. * * Appends never reject, so an append failure cannot escape into the caller's * `catch` and rewrite a completed turn as failed; the caller also refuses to diff --git a/src/runtime/grokEngineBrokerTurn.ts b/src/runtime/grokEngineBrokerTurn.ts index eef3a88..61dfaa1 100644 --- a/src/runtime/grokEngineBrokerTurn.ts +++ b/src/runtime/grokEngineBrokerTurn.ts @@ -7,7 +7,7 @@ import type { EngineBrokerMcpCallObservation } from "./engineBrokerMcpCallLog.js import { lowerEngineBrokerTurnLimits, mapGrokReportedModel, type EngineBrokerTurnAccounting, type EngineBrokerTurnLimitOverrides, type EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; import { NativeBrokerTurnFailure, type NativeBrokerDiagnostic, type NativeBrokerTurn, type NativeBrokerTurnResult } from "./engineBrokerNativeClient.js"; import type { EngineBrokerTurnRegistry } from "./engineBrokerTurnRegistry.js"; -import { engineBrokerRequestLedgerPathFor, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +import { engineBrokerRequestLedgerPathFor, engineBrokerSealLedgerPathFor, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; import { ensureBrokerTurnLedgered } from "./grokEngineBrokerLedger.js"; import { finishBrokerTurnWithUsage, type BrokerTurnMetering, type BrokerTurnMeteringDetail } from "./grokEngineBrokerMetering.js"; import type { GrokBrokerProxyTurn } from "./grokBrokerProxy.js"; @@ -53,7 +53,7 @@ export async function runGrokEngineBrokerTurn(deps: GrokEngineBrokerTurnDependen const turnId = createHash("sha256").update(`${agentId}\0${wakeId}`).digest("hex"); const request = { version: ENGINE_BROKER_VERSION, kind: "start_turn", requestId: randomUUID(), turnId, agentId, wakeId, prompt, mcpEndpoint, ...(overrides === undefined ? {} : { limits: overrides }) } as const; const begun = await deps.turns.begin(request, declared); - const metering: BrokerTurnMetering = { usageLedgerPath: registration.usageLedgerPath, requestLedgerPath: engineBrokerRequestLedgerPathFor(registration.usageLedgerPath), agentId, wakeId }; + const metering: BrokerTurnMetering = { usageLedgerPath: registration.usageLedgerPath, requestLedgerPath: engineBrokerRequestLedgerPathFor(registration.usageLedgerPath), sealLedgerPath: engineBrokerSealLedgerPathFor(registration.usageLedgerPath), agentId, wakeId }; if (begun !== "start") { await ensureBrokerTurnLedgered(begun.ledger, turnId, metering); return replay(begun.replay); } const controller = new AbortController(); const meter = new GrokBrokerTurnMeter(limits, () => controller.abort()); diff --git a/src/runtime/index.ts b/src/runtime/index.ts index b3f2ed1..e16c59c 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -17,7 +17,8 @@ export { GROK_INFERENCE_AUTH_STALE_BODY } from "./grokInferenceProxy.js"; export { assertGrokWorkerDenyPathPlacement, assertGrokWorkerDenyPathShape, assertGrokWorkerDenyPathsPlaceable, GROK_WORKER_BASE_PROFILE_GRANTS, grokWorkerCanSearch, grokWorkerDenyPathChain, GrokWorkerDenyPlacementError, readGrokWorkerDenyPathChain } from "./grokWorkerDenyPlacement.js"; export type { GrokWorkerDenyPathEntry, GrokWorkerDenyPathStep, GrokWorkerDenyPathWorker } from "./grokWorkerDenyPlacement.js"; export { grokWorkerSandboxProfileSha256, renderGrokWorkerSandboxProfile } from "./grokWorkerSandboxProfile.js"; -export { engineBrokerRequestLedgerPathFor, parseEngineBrokerServiceConfig, type EngineBrokerServiceConfig, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +export { engineBrokerRequestLedgerPathFor, engineBrokerSealLedgerPathFor, parseEngineBrokerServiceConfig, type EngineBrokerServiceConfig, type EngineBrokerServiceRegistration } from "./engineBrokerServiceConfig.js"; +export { TURN_SEAL_LEDGER, TURN_SEAL_LEDGER_VERSION } from "./engineBrokerSealLedger.js"; export { DEFAULT_GROK_BROKER_TURN_LIMITS, ENGINE_BROKER_LIMIT_REASONS, type EngineBrokerLimitReason, type EngineBrokerTurnAccounting, type EngineBrokerTurnLimits, type EngineBrokerTurnUsage } from "./engineBrokerTurnAccounting.js"; export { dedupeTurnUsageRows } from "./turnUsageLedger.js"; From 0fe368a0a5d5ffe362c7e61cab34183af712d498 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 12:46:38 +0200 Subject: [PATCH 54/69] test: prove a worker still blocked in write past the launcher's output bound is stopped, not parked --- ...ngineBrokerLauncherIntegrationLauncher.inc | 37 +++++++++++++++++++ src/runtime/native/fixtureWorker.c | 6 ++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc index 2350808..5fb629b 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc @@ -201,6 +201,42 @@ static void worker_spill_case(void) { close(p); close(c); } +/* The bound is the WHOLE TURN's stdout, and a worker can cross it while it is + still working and still writing. `output_boundary_case` writes one byte past + the bound and then exits on its own account, so it proves the arithmetic and + not the termination: the worker was already gone when the trip fired. This + one straddles that boundary from the other side. The fixture writes eight + times the bound in ordinary frame-sized writes and then sleeps far longer + than this suite, so it fills its own pipe repeatedly and is asleep inside + `write()` when the trip happens, and NOTHING but the launcher can end it. + A launcher that stopped reading without killing — or killed without + answering — parks the worker and this client forever, which is why the + socket carries a deadline: a park has to fail red here, not hang the runner. + The deadline also bounds the answer: it must arrive while the fixture is + still sleeping, so a pass cannot be the fixture exiting by itself. */ +static void worker_stream_case(void) { + int s = connect_socket(), p = sealed("prompt"), + c = sealed_bundle("stream-output", "mcp.Stream-2"); + struct dbl_request q = request(); + struct dbl_result r; + struct timeval deadline = {.tv_sec = 30, .tv_usec = 0}; + char extra; + check(!setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &deadline, sizeof(deadline)), + "worker stream deadline"); + send_request(s, &q, p, c); + check(read_all(s, &r, sizeof(r)) && result_padding_zero(&r), + "worker stream answered before the deadline"); + check(r.status == DBL_STATUS_OUTPUT_FAILED && r.stage == DBL_STAGE_OUTPUT && + r.failure_class == DBL_FAILURE_OUTPUT_LIMIT && + r.output_length == 0 && r.diagnostic_length == 0 && + r.worker_pid > 0 && r.worker_uid == 2200 && r.start_ticks > 0 && + r.term_signal == SIGKILL, + "worker stream stopped at the bound"); + check(read(s, &extra, 1) == 0, "worker stream EOF"); + close(s); + close(p); + close(c); +} static void org_cases(void) { check(!setgid(DBL_BROKER_UID) && !setuid(DBL_BROKER_UID), "drop broker uid"); client_case(); @@ -211,6 +247,7 @@ static void org_cases(void) { worker_failure_case(); worker_flood_case(); worker_spill_case(); + worker_stream_case(); int s = connect_socket(), p = sealed("prompt"), c = sealed_bundle("provider.Token-1", "mcp.Token-2"); struct dbl_request q = request(); diff --git a/src/runtime/native/fixtureWorker.c b/src/runtime/native/fixtureWorker.c index 551bea5..19c211d 100644 --- a/src/runtime/native/fixtureWorker.c +++ b/src/runtime/native/fixtureWorker.c @@ -7,4 +7,8 @@ #include #include #include "engineBrokerLauncher.h" -int main(int argc,char**argv){unsigned char bundle[8200]={0};ssize_t size=pread(4,bundle,sizeof(bundle),0);if(size<6)return 21;size_t offset=0;unsigned provider_length=bundle[0]|(bundle[1]<<8);offset=2;if(offset+provider_length+2>(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output"),spill=!strcmp(provider,"spill-output");if(!strcmp(provider,"sleep"))sleep(30);if(!strcmp(provider,"stderr-failure")){fprintf(stderr,"grok: session store unwritable\n");return 1;}if(!strcmp(provider,"stderr-flood")){fprintf(stderr,"HEAD-OF-ERROR grok: profile refused ");for(size_t i=0;i<4096;i++)if(fputc(0x6d,stderr)==EOF)return 32;fprintf(stderr," TAIL-OF-ECHO");fflush(stderr);return 1;}if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;iDBL_MAX_OUTPUT)return 34;char*blob=malloc(count);if(!blob)return 35;memset(blob,'s',count);int head=snprintf(blob,count,"SPILL cap=%d count=%zu\n",cap,count);if(head<=0||(size_t)head+10u>=count)return 36;blob[head]='s';memcpy(blob+count-10,"SPILL-TAIL",10);if(write(1,blob,count)!=(ssize_t)count)return 37;free(blob);return 0;}char prompt_bytes[128]={0};FILE*prompt_file=fopen("/proc/self/fd/3","r");if(!prompt_file)return 26;size_t prompt_read=fread(prompt_bytes,1,sizeof(prompt_bytes)-1,prompt_file);fclose(prompt_file);if(!prompt_read)return 27;char fds[128]={0};size_t fds_used=0;DIR*fd_dir=opendir("/proc/self/fd");if(!fd_dir)return 29;struct dirent*fd_entry;int fd_seen[64]={0};while((fd_entry=readdir(fd_dir))){if(fd_entry->d_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s tmpdir=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target,getenv("TMPDIR")?getenv("TMPDIR"):"unset");for(int i=0;i(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output"),spill=!strcmp(provider,"spill-output");int stream=!strcmp(provider,"stream-output");if(!strcmp(provider,"sleep"))sleep(30);if(!strcmp(provider,"stderr-failure")){fprintf(stderr,"grok: session store unwritable\n");return 1;}if(!strcmp(provider,"stderr-flood")){fprintf(stderr,"HEAD-OF-ERROR grok: profile refused ");for(size_t i=0;i<4096;i++)if(fputc(0x6d,stderr)==EOF)return 32;fprintf(stderr," TAIL-OF-ECHO");fflush(stderr);return 1;}if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;iDBL_MAX_OUTPUT)return 34;char*blob=malloc(count);if(!blob)return 35;memset(blob,'s',count);int head=snprintf(blob,count,"SPILL cap=%d count=%zu\n",cap,count);if(head<=0||(size_t)head+10u>=count)return 36;blob[head]='s';memcpy(blob+count-10,"SPILL-TAIL",10);if(write(1,blob,count)!=(ssize_t)count)return 37;free(blob);return 0;}if(stream){char chunk[4096];memset(chunk,'t',sizeof(chunk));for(size_t i=0;id_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s tmpdir=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target,getenv("TMPDIR")?getenv("TMPDIR"):"unset");for(int i=0;i Date: Fri, 18 Sep 2026 12:47:40 +0200 Subject: [PATCH 55/69] fix: raise the launcher's whole-turn output bound to the control protocol's own text bound --- src/contracts/runtimeContractManifest.ts | 8 ++-- src/runtime/engineBrokerNativeClient.ts | 3 +- src/runtime/native/AGENTS.md | 42 +++++++++++++----- .../artifacts/daimon-engine-broker-arm64 | Bin 67456 -> 67456 bytes ...daimon-engine-broker-arm64.provenance.json | 2 +- .../native/artifacts/daimon-engine-broker-x64 | Bin 55392 -> 55392 bytes .../daimon-engine-broker-x64.provenance.json | 2 +- src/runtime/native/engineBrokerLauncher.h | 13 +++++- ...ngineBrokerLauncherIntegrationLauncher.inc | 39 ++++++++++++++++ src/runtime/native/fixtureWorker.c | 11 ++++- 10 files changed, 101 insertions(+), 19 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 3df8e89..95b0dfc 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -80,7 +80,7 @@ export const GROK_ENGINE_BROKER = { spillDirectory: { relativeToRuntimeHome: "tool-output", owner: "organization", group: "worker", mode: 0o2750, fileMode: 0o640 } } }, - bounds: { promptBytes: 65_536, capabilityBytes: 4_096, capabilityBundleBytes: 8_196, outputBytes: 65_536 }, + bounds: { promptBytes: 65_536, capabilityBytes: 4_096, capabilityBundleBytes: 8_196, outputBytes: 262_144 }, // Accounting and limits (P2). The broker is the single sealed usage writer. controlProtocolVersion: "noopolis.daimon.engine-broker.v2", turnRecordVersions: ["noopolis.daimon.engine-broker-turn.v1", "noopolis.daimon.engine-broker-turn.v2"], @@ -133,9 +133,9 @@ export const GROK_ENGINE_BROKER = { projectionVersion: "noopolis.daimon.grok-broker-projection.v1", slotPreflightVersion: "noopolis.daimon.grok-slot-preflight.v2", artifacts: { - sourceSha256: "d8c9640a2d0084f584721d0d4afdc524af7434e9461c1fc2f8adcdff6454ba6d", - x64Sha256: "4059ec576065130e857cc937b03b97a0c45fffab7d790fb153fcb170bfd0f310", - arm64Sha256: "eb0e2975cf3204bb80d25edc1fec00dbd623a95466176f3f36e7e83178deb8d2" + sourceSha256: "dd39aacfece496cc6528f6acdb4f1066a848a0fb5b0961f5c70b0ba00440dc24", + x64Sha256: "67e3624d3198e9c59e1ffafa4eca7c895dfe265d5b8bb0614cb547b68b8b93a7", + arm64Sha256: "c07d22225ff968bc289e5ddf0981cdd5d64040eee6e3ea45da7d03e1439dea98" } } as const; export const AGY_SUBSCRIPTION_REALM = { diff --git a/src/runtime/engineBrokerNativeClient.ts b/src/runtime/engineBrokerNativeClient.ts index de6406a..6a612e1 100644 --- a/src/runtime/engineBrokerNativeClient.ts +++ b/src/runtime/engineBrokerNativeClient.ts @@ -7,7 +7,8 @@ export const ENGINE_BROKER_NATIVE_REQUEST_BYTES = 396; export const ENGINE_BROKER_NATIVE_RESULT_BYTES = 128; /** `DBL_MAX_DIAGNOSTIC`: the launcher's bounded tail of a failed worker's own merged stdout/stderr. */ export const ENGINE_BROKER_NATIVE_DIAGNOSTIC_BYTES = 512; -const MAX_PROMPT = 65_536, MAX_CAPABILITY = 4_096, MAX_OUTPUT = 65_536; +/** `DBL_MAX_OUTPUT`: the launcher's bound on a whole turn's stdout, and the control protocol's own `text` bound, which is the next boundary this output crosses. */ +const MAX_PROMPT = 65_536, MAX_CAPABILITY = 4_096, MAX_OUTPUT = 262_144; const statuses = ["ok", "prelaunch_failed", "worker_failed", "output_failed", "cancelled"] as const; const stages = ["none", "peer", "request", "registration", "executable", "exec", "wait", "output", "attestation"] as const; const failures = ["none", "peer", "protocol", "registration", "executable", "exec", "wait", "output_limit", "cancelled", "profile_missing", "profile_invalid"] as const; diff --git a/src/runtime/native/AGENTS.md b/src/runtime/native/AGENTS.md index 86d0c78..be65878 100644 --- a/src/runtime/native/AGENTS.md +++ b/src/runtime/native/AGENTS.md @@ -99,16 +99,38 @@ adversarial cases that do cross the bound (`output_boundary_case`, add a test that claims to cover it by feeding the bound through the poll loop: that routes around the defect. -The bound is the whole turn's stdout, not one frame. A live single-tool-call -brokered turn already emitted 26,486 bytes, 23,320 of them one tool-result -frame (`.runtime/grok-p1b/worker-a2-output.jsonl`), against a `--max-turns` of -48 and a 16 KiB tool-result spill bound, so 64 KiB is reachable by an ordinary -working turn rather than only by a runaway one. **Known limit, deliberately not -raised yet:** the same day changed the pipe's blocking mode, and raising the -bound alongside it would mix two variables in the next live run; the measured -ceiling is 26 KB against 64 KiB, so it is not the thing in the way. Revisit -once a trial scores, and decide the number on the spread of real turns rather -than on headroom-by-guess. +The bound is the whole turn's stdout, not one frame, and it is now 256 KiB. +**This supersedes the "known limit, deliberately not raised yet" this file +carried while the pipe's blocking mode was the variable under test.** The +measurement that decided it: a live brokered turn emitted 26,482 bytes for +four tool calls, 23,320 of them one tool-result frame carrying all four +(`.runtime/grok-p1b/worker-a2-output.jsonl`); JSON framing and escaping +inflated those payloads by 1.007x, so a turn's stdout is close to the sum of +its tool results. The nine-tool-call turn this was raised for is about 210 KB +of the same shape, against a 64 KiB bound — so 64 KiB was reachable by an +ordinary working turn, and crossing it costs that turn its whole text. The new +number is not headroom-by-guess: it is the control protocol's own `text` bound +(`engineBrokerProtocol.ts`, 262144), the next boundary this output has to +cross, so a larger launcher bound would only move the refusal one layer up. +`worker_turn_case` writes exactly that measured shape — a 9,728-byte init +frame and nine 23,320-byte frames, 219,608 bytes — and asserts it is published +whole; restoring 65536 turns it red. + +**A bound that hangs would be worse than no bound, and this one does not.** +The hypothesis that a worker parks forever in `write()` once the bound is +crossed — plausible after the pipe became blocking, because a write that +cannot complete now blocks instead of erroring — was tested, not reasoned +about, and it is false. `worker_stream_case` writes eight times the bound in +frame-sized writes and then sleeps far longer than this suite, so it is asleep +inside `write()` with its pipe full when the trip fires and nothing but the +launcher can end it; the launcher answers `output_limit` with `term_signal` +SIGKILL in seconds. Its socket carries a 30-second deadline so a park fails +red instead of parking the runner. Deleting the `output_limited` half of +`if (disconnected || output_limited) kill(-pid, SIGKILL)` is the mutation that +proves it: the case then times out on that deadline, and +`output_boundary_case` does not notice, because its worker has already exited +by the time the trip fires. That is the boundary the two cases straddle — +a worker gone at the trip against a worker alive and blocked at it. The result frame's last word is `diagnostic_length`, not padding: on `DBL_STATUS_WORKER_FAILED` the supervisor keeps the last `DBL_MAX_DIAGNOSTIC` diff --git a/src/runtime/native/artifacts/daimon-engine-broker-arm64 b/src/runtime/native/artifacts/daimon-engine-broker-arm64 index 2dbe88d27e03d69ea06aa08648efa8f8bc54e551..7d34af45d6e98df65d0375adda12a2c361467705 100755 GIT binary patch delta 2778 zcmY*bdr*|u6~Fg>yU1f8;POz=?Beorts<2L6qQ}xhMHL1Y3evhpg3xRNrR|JFtZ<~ ziP`c;TrY~^Ynl8}cPml5&~!7^V00WCVyf#SnkLP+VyDUUF;Y<>2)DnxVkf;bcg{J# zbM86ko^!rCpZcCoeNVUha+JD*gqM2imU%W!@K4NsUk}n_Ud`xt{`;BNbS%cs2Q7BG zlgG~UNa4i1Z6N;aAN`&&V)uWuAR?kzv0s^JBi;#xxbLl|-jN2yZ6IoJtWoD3Q_klt!l&G~28x-%TqWHMUn!{2Eue4g~|1m1M2FPeZ)0pTY#1=5dlcqz{ zj;8iU98J}XXIo9tq%f3gZ#C*>Zyo~PK$0vy!0t1}vNe`iQ{XqGr1>-@S;nkwgP(60 zY19aD3_KDAYxxyx+8Pg{IGKG|W282;>u72TA+MN{iK75n?n9QJFvDOwGjwRmOiLa- z7zr7kpF|wZjC9Bd^6Yn$!Xb3zDQ%Z6KxdH6#&Z#mLfl?pN>S`?6^C!ESKLOEa{bBX zn+j&pt=v_R&lEfVxL~HeA8BZ~%m5<=X1FyWybP-%1x5^mGW<|83EYF7OrZ|phnj5x z!vOD=DMaz4!fbsNwEMZe(9FJgz}FX6(z(31&`1mUr9v}Zz=sQS=?$J^rNG{V-Q&t6 z`j|}(8nFA|_hsH&)a+zPWe(XHl-<>EY7&|!r!^(jl0S*=E+$s(9CcIbsm+4D4f1)| zJy3`RmuzZWLL4-UI5b>T`b($b!qm1%+j+$LH`p3=q)D^DPTm091YMnlFzW^0v)JhT z2qv16V^6Jue zsDzBkBHA-ZUucNB8C8RW<0ylfSVIbP%O(hCq<1sywME2=D+O_-puBn=WL-FpTFXGw zu?fmC4?6Y7#Cki<7Jv_nhW#`!Ee}|97>f&odVn^79sKJhDJy3}@+>4)>E?%S2Xt!f zbm)+-MW))GORS8kwP#he_9oJjo$=(37Rqwssklo_OWdzLFV*to z)X*}&Wk%jj?6nxRr z3TpGWE{&n|PyFp=W;(#V%gnPre1I`Ku4{^lC63JB>Er(i-oRY$-*n7xUib#p%C> zzbFi-wA%?Z)G)VW7?uy^fWlWdk1A=Q)x5nVg_iOIB{j3Ap%)R@e>cUtFVw}l@02n3 z&|8X|5psEtErVX=nYLX~c`_RE%;o*Ik`>dYvX<4*P4V?I56L5^9`KVG&+r_89|u3t zcE(FhYB2n^$B?mm$f(!9%xsz`noFKP}B#b__4&79zWoXgPczKH6*n8yhXy z7RHh-`z7WC26-=d5kc}$A%~`y;63HfQNoAI z4fH72SCr8g`HG5?=xrl7kGUBg;d?5U({K5eibd1A;nXxC@U);2sCU)FF ztuoUW_{u6{Qakixjpg3)<_hnZlOjddY6owxT13luf7SD}m&aCbpsV@T>MGjKPgmzs ztN&ls&6L`B{WC8m~q|c!ytkrkGymu{9-B&sWx@(KGzzni{&6_tzNd z`~K@SW||OQ|p#Xr%x4pL$h9MCmYNub&X2lLxkJaAx4qJNZW; zmV#!S7GfRfyP(@ab3PVg0Ms@hM5GS%ybxPJk9{V@MNrQrA^r(kepQHf^@M2Ggizsq z;dOWiRY8w|YBz*<1bPQFUQRh^2B_zzm@ZagZcvesS~qjjPwb{;NGs9f-Hb5qnJYA+L)a^tl*6((<%E9v|2`ezwJ+&jGiI-)b?> zTm|kA0h9xggT)#Edw M>BSH0_0-7z3w>l!q5uE@ delta 2755 zcmY*b3s98T6~6cXcL@(ck+)!XaRKpHie~_dq|c&Dd2}!>DO;-}Gs3-#T!J{KOCtFu$k;o)wJ1#~NjQ6>k@$vT zBhsir3JHF;|3;(g6KS~FRLewt6|^1f1&c{qbf0au%%^U4#$uqw>eRcF|SlZp91ur=sNgG(U}u@Aw@ zfzGH9$$XCaW*b`kFbT>@9 zNC9zlBRe`LrD_H^btW-ik5hf{P=Z)lz+5wcMSp|E03*Phh^`A~r&UX!;Ub}k2Iy;8 zd|^siDmdh(h`dUs5wj#|)u%MA`WkW&ZK#(a-?Z5)_Kz= zId5#0of7)OjA|DBl$EL5qGpTjkf??AsaE^wk+emTMvd^xz()xlNr_@bjNq-{W$=$S z{mD&@S|I#($B^N>Na>{ahb1OT|G-*H>m+HA{kt@$xC?LLIwG2osQh*S@iwVGRT|1) zAI6dum%v(pLGA$0BghPt$PtuNlzptqW}r@1Z>yu6_1V}9@rYIvL-K+-)R`Hn1!5rw zqaiDjGjogFhWwXYPV*@p6_{o}TnaXp9mxsEb5et3`dU zF?+d*E@k!QhNSJ#53ic%e$ZIvzVeu-h+4hII?HF#O4e7th@NCqE9&TrYYmRZ=%zW$WT2k?} zb-l;=bkMB#IbRI=3((b|xqs&T1gPrg94`XN?dE)m4*GMPcY$i>IqwC{y2$w&MD_P_ zz9NEY{faw|Hd?Dx^&=ua}FEjdsHF%PC zP(s#u5gR_FB+eC68s|Ezvvr)XAM)Bu+1vJA$L+S=UG_c4>}Bo0v+q4>-*?2m|1G;z z5&g-Pe_Zanbg}p9wURi^uPccoB{7W$-Sz_>JM%PsXS@zKYmJX{BO)WTu|s!A&H|eO zd;|NU$&@w^TsJI!A}sLD!&(e1k!5bM>G1VBr$4(wD9I#ubX(Vu?yt(3JVDI?rM ztO~^KMl)~|Hs)olrQfm^M>E3cAdK)TZ8Yfp;C{n)Z*13#ZXIDWUTM-_2Dh3WeWhut z1D#sin&uw~kY7e;Hk2(GWJ^6&kO<>)?B1f=X&>>*& zvGbcIj?Y5J)C=Eu2%|)-BH$ZXd~-sY1Dw{+d6rP&n};Q4=eD=FIi`gLX{xv4L25MZ L`u#FJHAw#h^mJ3@ diff --git a/src/runtime/native/artifacts/daimon-engine-broker-arm64.provenance.json b/src/runtime/native/artifacts/daimon-engine-broker-arm64.provenance.json index 8b9496e..a694add 100644 --- a/src/runtime/native/artifacts/daimon-engine-broker-arm64.provenance.json +++ b/src/runtime/native/artifacts/daimon-engine-broker-arm64.provenance.json @@ -1 +1 @@ -{"version":"daimon.engine-broker-native-build.v1","architecture":"arm64","target":"linux/arm64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:d8c9640a2d0084f584721d0d4afdc524af7434e9461c1fc2f8adcdff6454ba6d","binary_sha256":"sha256:eb0e2975cf3204bb80d25edc1fec00dbd623a95466176f3f36e7e83178deb8d2","install_path":"/opt/daimon/bin/daimon-engine-broker"} +{"version":"daimon.engine-broker-native-build.v1","architecture":"arm64","target":"linux/arm64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:dd39aacfece496cc6528f6acdb4f1066a848a0fb5b0961f5c70b0ba00440dc24","binary_sha256":"sha256:c07d22225ff968bc289e5ddf0981cdd5d64040eee6e3ea45da7d03e1439dea98","install_path":"/opt/daimon/bin/daimon-engine-broker"} diff --git a/src/runtime/native/artifacts/daimon-engine-broker-x64 b/src/runtime/native/artifacts/daimon-engine-broker-x64 index 3e42c7286b0330f78f6a9ef82948337c03c7dd50..f7c022d10d401f5dbe41751e5d41160afcd6c66c 100755 GIT binary patch delta 267 zcmaE`f%(A(<_&H7j4YGe^(}$qLH+eWvd|!gk!ABa0|iD#g~{&?Ll`9{yBOUN74Yfw zQQ>&e`QiV6pKc(_r`ttEV6vgHhNi~nr7oSAz8CNoDO#WkR$tW<{%w(M)3jJ!vLxC!8o2~-7 zpv_E^QDgH4vvx*Cfz1{c$C(*fCfSGq-6?MKR)B?pznzPjfx)NqgX11X2A0Vpw#Gmy t58D7BIn&k~NItb~1Clv*sz6UovOCTQq@o;NRsjuj*)0t;b8*5R82~rMS1bSk delta 267 zcmaE`f%(A(<_&H7jEs}p^(}$qLH+eWvd|!gk#X}m0|iD#jmhr~Ll^}nyBOUNmGJ5G zQQ>&e`QiV6pKc(_r`ttEVzQyJhNi;jr7oSJ!vLxC!8o2~-7 zpv_E^QDO52vvx*CiOm)k$C()!C)tPr-6?MKR)CR#znzPjfx)NqgX11X2FA%Ew#Gmy v58D7BIn&k~NItb~1Clv*sz6UovOCTQq@o;NRxz?{cG)e>$jGudVUG*|FYH$L diff --git a/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json b/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json index d973815..4dba2b9 100644 --- a/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json +++ b/src/runtime/native/artifacts/daimon-engine-broker-x64.provenance.json @@ -1 +1 @@ -{"version":"daimon.engine-broker-native-build.v1","architecture":"x64","target":"linux/amd64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:d8c9640a2d0084f584721d0d4afdc524af7434e9461c1fc2f8adcdff6454ba6d","binary_sha256":"sha256:4059ec576065130e857cc937b03b97a0c45fffab7d790fb153fcb170bfd0f310","install_path":"/opt/daimon/bin/daimon-engine-broker"} +{"version":"daimon.engine-broker-native-build.v1","architecture":"x64","target":"linux/amd64","builder_image":"gcc:14-bookworm@sha256:5e927c284bf55a7dc796262e311a0703344f62f41f5621eb56843111b1d37e15","compiler":"gcc-14","source_sha256":"sha256:dd39aacfece496cc6528f6acdb4f1066a848a0fb5b0961f5c70b0ba00440dc24","binary_sha256":"sha256:67e3624d3198e9c59e1ffafa4eca7c895dfe265d5b8bb0614cb547b68b8b93a7","install_path":"/opt/daimon/bin/daimon-engine-broker"} diff --git a/src/runtime/native/engineBrokerLauncher.h b/src/runtime/native/engineBrokerLauncher.h index 636e484..7542b51 100644 --- a/src/runtime/native/engineBrokerLauncher.h +++ b/src/runtime/native/engineBrokerLauncher.h @@ -8,7 +8,18 @@ #define DBL_MAX_PROMPT 65536u #define DBL_MAX_TOKEN 4096u #define DBL_MAX_CAPABILITY_BUNDLE (DBL_MAX_TOKEN * 2u + 4u) -#define DBL_MAX_OUTPUT 65536u +/* The WHOLE turn's stdout, not one frame, and sized against real turns rather + than headroom-by-guess. A live four-tool-call brokered turn emitted 26,482 + bytes, 23,320 of them one tool-result frame carrying four results + (`.runtime/grok-p1b/worker-a2-output.jsonl`), so 64 KiB was reachable by an + ordinary working turn: the nine-tool-call turn this was raised for lands + around 210 KB of the same shape, and a trip costs the turn its whole text. + The number is the control protocol's own `text` bound + (`engineBrokerProtocol.ts`, 262144), because that is the next boundary the + output must cross: a larger launcher bound would only move the refusal one + layer up. A runaway worker is still stopped here — crossing it stops + reading, SIGKILLs the worker's process group and reports `output_limit`. */ +#define DBL_MAX_OUTPUT 262144u /* Bounded tail of the worker's own merged stdout/stderr, kept only for a worker that exited on its own account (`DBL_STATUS_WORKER_FAILED`), so the reason reaches the host instead of `exit=1`. It is a diagnostic, never the diff --git a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc index 5fb629b..7b4bdc8 100644 --- a/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc +++ b/src/runtime/native/engineBrokerLauncherIntegrationLauncher.inc @@ -201,6 +201,44 @@ static void worker_spill_case(void) { close(p); close(c); } +/* An ordinary working turn must not be able to hit the bound, and its output + must arrive whole. The fixture writes the shape one live turn measured — a + 9,728-byte init frame and nine 23,320-byte tool-result frames, 219,608 + bytes in all — in frame-sized writes. That is more than three times the + 64 KiB this bound used to be, so at the old value this same turn was + published as `output_limit` with its whole text discarded; it is the case + that straddles the raise, and restoring 65536 turns it red. */ +static void worker_turn_case(void) { + int s = connect_socket(), p = sealed("prompt"), + c = sealed_bundle("turn-output", "mcp.Turn-2"); + struct dbl_request q = request(); + struct dbl_result r; + const uint32_t expected = 9728u + 23320u * 9u; + char *out, extra; + size_t index; + send_request(s, &q, p, c); + check(read_all(s, &r, sizeof(r)) && result_padding_zero(&r) && + r.status == DBL_STATUS_OK && r.stage == DBL_STAGE_OUTPUT && + r.failure_class == DBL_FAILURE_NONE && r.exit_code == 0 && + r.term_signal == 0 && r.diagnostic_length == 0 && + expected > 65536u && + r.output_length == expected, + "ordinary turn published whole"); + out = calloc(1, (size_t)expected + 1u); + check(out && read_all(s, out, expected), "ordinary turn output"); + check(!memcmp(out, "TURN-HEAD", 9) && + !memcmp(out + expected - 9, "TURN-TAIL", 9), + "ordinary turn ends intact"); + for (index = 9; index < (size_t)expected - 9u; index++) + if (out[index] != 'u') + break; + check(index == (size_t)expected - 9u, "ordinary turn bytes intact"); + check(read(s, &extra, 1) == 0, "ordinary turn EOF"); + free(out); + close(s); + close(p); + close(c); +} /* The bound is the WHOLE TURN's stdout, and a worker can cross it while it is still working and still writing. `output_boundary_case` writes one byte past the bound and then exits on its own account, so it proves the arithmetic and @@ -247,6 +285,7 @@ static void org_cases(void) { worker_failure_case(); worker_flood_case(); worker_spill_case(); + worker_turn_case(); worker_stream_case(); int s = connect_socket(), p = sealed("prompt"), c = sealed_bundle("provider.Token-1", "mcp.Token-2"); diff --git a/src/runtime/native/fixtureWorker.c b/src/runtime/native/fixtureWorker.c index 19c211d..14fd55a 100644 --- a/src/runtime/native/fixtureWorker.c +++ b/src/runtime/native/fixtureWorker.c @@ -10,5 +10,14 @@ /* Eight times the whole-turn output bound, written in ordinary frame-sized writes and followed by a sleep the launcher must cut short: the fixture never exits on its own account, so only the bound can end this turn. */ +/* The measured shape of a real brokered turn: a 9,728-byte init frame and + nine 23,320-byte tool-result frames, the frame size one live capture + actually produced (`.runtime/grok-p1b/worker-a2-output.jsonl`). It is far + past the 64 KiB this bound used to be and well inside what it is now, so it + is the case that straddles the raise. */ +#define TURN_INIT_BYTES 9728u +#define TURN_FRAME_BYTES 23320u +#define TURN_FRAMES 9u +#define TURN_BYTES (TURN_INIT_BYTES + TURN_FRAME_BYTES * TURN_FRAMES) #define STREAM_CHUNKS ((DBL_MAX_OUTPUT / 4096u) * 8u) -int main(int argc,char**argv){unsigned char bundle[8200]={0};ssize_t size=pread(4,bundle,sizeof(bundle),0);if(size<6)return 21;size_t offset=0;unsigned provider_length=bundle[0]|(bundle[1]<<8);offset=2;if(offset+provider_length+2>(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output"),spill=!strcmp(provider,"spill-output");int stream=!strcmp(provider,"stream-output");if(!strcmp(provider,"sleep"))sleep(30);if(!strcmp(provider,"stderr-failure")){fprintf(stderr,"grok: session store unwritable\n");return 1;}if(!strcmp(provider,"stderr-flood")){fprintf(stderr,"HEAD-OF-ERROR grok: profile refused ");for(size_t i=0;i<4096;i++)if(fputc(0x6d,stderr)==EOF)return 32;fprintf(stderr," TAIL-OF-ECHO");fflush(stderr);return 1;}if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;iDBL_MAX_OUTPUT)return 34;char*blob=malloc(count);if(!blob)return 35;memset(blob,'s',count);int head=snprintf(blob,count,"SPILL cap=%d count=%zu\n",cap,count);if(head<=0||(size_t)head+10u>=count)return 36;blob[head]='s';memcpy(blob+count-10,"SPILL-TAIL",10);if(write(1,blob,count)!=(ssize_t)count)return 37;free(blob);return 0;}if(stream){char chunk[4096];memset(chunk,'t',sizeof(chunk));for(size_t i=0;id_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s tmpdir=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target,getenv("TMPDIR")?getenv("TMPDIR"):"unset");for(int i=0;i(size_t)size)return 21;char provider[4097]={0};memcpy(provider,bundle+offset,provider_length);offset+=provider_length;unsigned mcp_length=bundle[offset]|(bundle[offset+1]<<8);offset+=2;if(offset+mcp_length!=(size_t)size)return 21;char mcp[4097]={0};memcpy(mcp,bundle+offset,mcp_length);memset(bundle,0,sizeof(bundle));int exact=!strcmp(provider,"exact-output"),overflow=!strcmp(provider,"overflow-output"),spill=!strcmp(provider,"spill-output");int stream=!strcmp(provider,"stream-output");int turn=!strcmp(provider,"turn-output");if(!strcmp(provider,"sleep"))sleep(30);if(!strcmp(provider,"stderr-failure")){fprintf(stderr,"grok: session store unwritable\n");return 1;}if(!strcmp(provider,"stderr-flood")){fprintf(stderr,"HEAD-OF-ERROR grok: profile refused ");for(size_t i=0;i<4096;i++)if(fputc(0x6d,stderr)==EOF)return 32;fprintf(stderr," TAIL-OF-ECHO");fflush(stderr);return 1;}if(!getenv("DAIMON_MCP_CAPABILITY")||strcmp(getenv("DAIMON_MCP_CAPABILITY"),mcp)||strstr(getenv("DAIMON_MCP_CAPABILITY"),provider))return 24;if(!getenv("DAIMON_PROVIDER_CAPABILITY")||strcmp(getenv("DAIMON_PROVIDER_CAPABILITY"),provider))return 28;int output[2];if(pipe(output))return 22;pid_t child=fork();if(!child){dup2(output[1],1);close(output[0]);execl("/opt/daimon/bin/daimon-engine-broker","daimon-engine-broker","--auth-provider",NULL);_exit(127);}close(output[1]);char auth[8192]={0};ssize_t got=read(output[0],auth,sizeof(auth)-1);close(output[0]);int status;waitpid(child,&status,0);char expected[8192];snprintf(expected,sizeof(expected),"{\"access_token\":\"%s\",\"expires_in\":600}\n",provider);memset(provider,0,sizeof(provider));memset(mcp,0,sizeof(mcp));if(!WIFEXITED(status)||WEXITSTATUS(status)||got<1||strcmp(auth,expected))return 23;memset(auth,0,sizeof(auth));memset(expected,0,sizeof(expected));if(exact||overflow){size_t count=DBL_MAX_OUTPUT+(size_t)overflow;for(size_t i=0;iDBL_MAX_OUTPUT)return 34;char*blob=malloc(count);if(!blob)return 35;memset(blob,'s',count);int head=snprintf(blob,count,"SPILL cap=%d count=%zu\n",cap,count);if(head<=0||(size_t)head+10u>=count)return 36;blob[head]='s';memcpy(blob+count-10,"SPILL-TAIL",10);if(write(1,blob,count)!=(ssize_t)count)return 37;free(blob);return 0;}if(turn){char*blob=malloc(TURN_BYTES);if(!blob)return 39;memset(blob,'u',TURN_BYTES);memcpy(blob,"TURN-HEAD",9);memcpy(blob+TURN_BYTES-9,"TURN-TAIL",9);size_t sent=0;while(sentd_name[0]=='.')continue;int fd_number=atoi(fd_entry->d_name);if(fd_number==dirfd(fd_dir))continue;if(fd_number<0||fd_number>=64)return 30;fd_seen[fd_number]=1;}closedir(fd_dir);for(int fd_number=0;fd_number<64;fd_number++)if(fd_seen[fd_number])fds_used+=(size_t)snprintf(fds+fds_used,sizeof(fds)-fds_used,"%s%d",fds_used?",":"",fd_number);char stdin_target[64]={0};if(readlink("/proc/self/fd/0",stdin_target,sizeof(stdin_target)-1)<0)return 31;printf("uid=%ld argc=%d home=%s auth=ok mcp=ok prompt=%s fds=%s stdin=%s tmpdir=%s\n",(long)getuid(),argc,getenv("HOME"),prompt_bytes,fds,stdin_target,getenv("TMPDIR")?getenv("TMPDIR"):"unset");for(int i=0;i Date: Fri, 18 Sep 2026 13:00:25 +0200 Subject: [PATCH 56/69] feat: observe the brokered worker's standalone MCP GET tunnel --- src/runtime/engineBrokerControlClient.ts | 6 +- src/runtime/engineBrokerMcpCallLog.test.ts | 81 ++++++++++++++++++++-- src/runtime/engineBrokerMcpCallLog.ts | 65 ++++++++++++++++- src/runtime/engineBrokerMcpFacade.test.ts | 55 +++++++++++++++ src/runtime/engineBrokerMcpFacade.ts | 20 ++++-- src/runtime/engineBrokerProtocol.test.ts | 23 ++++++ src/runtime/engineBrokerProtocol.ts | 33 ++++++++- src/runtime/engineBrokerSealLedger.test.ts | 38 ++++++++++ src/runtime/engineBrokerSealLedger.ts | 21 +++++- 9 files changed, 323 insertions(+), 19 deletions(-) diff --git a/src/runtime/engineBrokerControlClient.ts b/src/runtime/engineBrokerControlClient.ts index 6744615..fd491f1 100644 --- a/src/runtime/engineBrokerControlClient.ts +++ b/src/runtime/engineBrokerControlClient.ts @@ -2,7 +2,7 @@ import { createHash, randomUUID } from "node:crypto"; import { createConnection } from "node:net"; import { ENGINE_BROKER_VERSION, encodeEngineBrokerFrame,EngineBrokerFrameDecoder,parseEngineBrokerResponse } from "./engineBrokerProtocol.js"; import type { EngineBrokerInferenceFailureCode, EngineBrokerInferenceRequest, EngineBrokerInferenceResponse } from "./engineBrokerInferenceProtocol.js"; -import type { EngineBrokerMcpCallObservation } from "./engineBrokerMcpCallLog.js"; +import type { EngineBrokerMcpCallObservation, EngineBrokerMcpTunnelObservation } from "./engineBrokerMcpCallLog.js"; import type { EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; import type { GrokBrokerModel, GrokBrokerReasoningEffort } from "./grokBrokerModelPolicy.js"; import type { GrokInferencePurpose } from "./inferenceUsageLedger.js"; @@ -14,7 +14,9 @@ import type { GrokInferencePurpose } from "./inferenceUsageLedger.js"; * long it had been waiting — the one thing a completion-only tool receipt can * never say. */ -const renderMcpCalls=(calls:EngineBrokerMcpCallObservation|undefined):string=>calls===undefined?"":`; mcp=${calls.answered}/${calls.started} answered${calls.undecoded===0?"":`; mcp_undecoded=${calls.undecoded}`}${calls.outstanding.length===0?"":`; mcp_outstanding=${calls.outstanding.map((call)=>`${call.name}@${call.outstandingMs}ms`).join(",")}`}`; +const renderMcpCalls=(calls:EngineBrokerMcpCallObservation|undefined):string=>calls===undefined?"":`; mcp=${calls.answered}/${calls.started} answered${calls.undecoded===0?"":`; mcp_undecoded=${calls.undecoded}`}${calls.outstanding.length===0?"":`; mcp_outstanding=${calls.outstanding.map((call)=>`${call.name}@${call.outstandingMs}ms`).join(",")}`}${renderMcpTunnels(calls.tunnels)}`; +/** The session's GET SSE tunnels: how many closed of how many opened, and each one still open with its age and whether the mount ever pushed through it. */ +const renderMcpTunnels=(tunnels:EngineBrokerMcpTunnelObservation|undefined):string=>tunnels===undefined?"":`; mcp_get=${tunnels.closed}/${tunnels.opened} closed, ${tunnels.delivered} delivered${tunnels.open.length===0?"":`; mcp_get_open=${tunnels.open.map((tunnel)=>`${tunnel.openMs}ms/${tunnel.delivered?"delivered":"silent"}`).join(",")}`}`; export type EngineBrokerInferenceGrant = Omit, "version" | "kind" | "requestId">; /** A refused grant request; `code` is closed (`auth_stale` is the stale shared realm, `grant_limit` the live-grant cap). */ diff --git a/src/runtime/engineBrokerMcpCallLog.test.ts b/src/runtime/engineBrokerMcpCallLog.test.ts index 8550608..7b0ab26 100644 --- a/src/runtime/engineBrokerMcpCallLog.test.ts +++ b/src/runtime/engineBrokerMcpCallLog.test.ts @@ -1,12 +1,15 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { EngineBrokerMcpCallLog, ENGINE_BROKER_MCP_CALL_INVALID, ENGINE_BROKER_MCP_CALL_TRUNCATED, ENGINE_BROKER_MCP_OUTSTANDING_MAX } from "./engineBrokerMcpCallLog.js"; +import { EngineBrokerMcpCallLog, ENGINE_BROKER_MCP_CALL_INVALID, ENGINE_BROKER_MCP_CALL_TRUNCATED, ENGINE_BROKER_MCP_OUTSTANDING_MAX, ENGINE_BROKER_MCP_TUNNEL_MAX } from "./engineBrokerMcpCallLog.js"; const body = (value: unknown): Buffer => Buffer.from(JSON.stringify(value), "utf8"); const call = (name: unknown, id = 1): Buffer => body({ jsonrpc: "2.0", id, method: "tools/call", params: { name, arguments: { text: "argument-bytes" } } }); /** A controllable clock: an outstanding call's whole value is how long it has been outstanding. */ +/** An observed turn whose facade never relayed a GET tunnel — a measurement, not an absence. */ +const NO_TUNNEL = { opened: 0, closed: 0, delivered: 0, open: [] }; + const clock = (): { log: EngineBrokerMcpCallLog; advance: (ms: number) => void } => { let at = 1_000; return { log: new EngineBrokerMcpCallLog(() => at), advance: (ms: number): void => { at += ms; } }; @@ -33,12 +36,12 @@ test("absence stays absence: an unopened turn observes undefined, a turn that ca const { log } = clock(); assert.equal(log.observe("turn"), undefined); log.open("turn"); - assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 0, outstanding: [] }); + assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 0, outstanding: [], tunnels: NO_TUNNEL }); // Everything that is not a tool call records nothing at all, so `started` // stays a count of tool calls and not of traffic. for (const value of [body({ jsonrpc: "2.0", id: 1, method: "tools/list" }), body({ jsonrpc: "2.0", method: "notifications/initialized" }), Buffer.alloc(0)]) log.begin("turn", value).answer(); log.begin("turn", undefined).answer(); - assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 0, outstanding: [] }); + assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 0, outstanding: [], tunnels: NO_TUNNEL }); log.close("turn"); assert.equal(log.observe("turn"), undefined, "a revoked turn keeps nothing"); }); @@ -48,7 +51,7 @@ test("a body the facade could not read counts as undecoded, never as a call with log.open("turn"); log.begin("turn", Buffer.from("{not json", "utf8")); log.undecodable("turn"); - assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 2, outstanding: [] }); + assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 2, outstanding: [], tunnels: NO_TUNNEL }); log.undecodable("absent"); }); @@ -71,3 +74,73 @@ test("the outstanding list is bounded, and the earliest calls are the ones kept" assert.equal(observed?.outstanding[0]?.name, "tool_0"); assert.equal(observed?.outstanding.at(-1)?.name, ENGINE_BROKER_MCP_CALL_TRUNCATED); }); + +/** + * The channel a brokered turn had no light on at all. + * + * Seven live runs closed every provider request, answered every tool call, and + * still sat idle to the deadline. The one thing none of them could say is + * whether the worker was parked on the session's standalone GET SSE tunnel, + * because the facade relayed it and recorded nothing. The boundary these + * assertions straddle is exactly that: a tunnel still open at observation + * against one that ended before it — one is a worker that may still be reading, + * the other is a channel already closed and therefore not the blocker. + * + * Mutation: drop `openTunnels.delete(record)` from `close`, and the closed + * tunnel keeps reporting itself open; drop `tunnelsOpened += 1`, and an open + * tunnel becomes indistinguishable from a turn that never opened one. + */ +test("a GET tunnel still open reports its age; one that closed first reports closed and nothing open", () => { + const { log, advance } = clock(); + log.open("turn"); + const parked = log.openTunnel("turn"); + advance(430_000); + const open = log.observe("turn")?.tunnels; + assert.deepEqual(open, { opened: 1, closed: 0, delivered: 0, open: [{ openMs: 430_000, delivered: false }] }); + + parked.close(); + advance(5_000); + assert.deepEqual(log.observe("turn")?.tunnels, { opened: 1, closed: 1, delivered: 0, open: [] }); +}); + +test("a turn that never opened a GET tunnel is not a turn that opened one, and neither is an unobserved turn", () => { + const { log, advance } = clock(); + log.open("turn"); + // Observed, and it measured zero: every count is a measurement. + assert.deepEqual(log.observe("turn")?.tunnels, { opened: 0, closed: 0, delivered: 0, open: [] }); + log.begin("turn", call("daimon__moltnet_read")).answer(); + assert.deepEqual(log.observe("turn")?.tunnels, { opened: 0, closed: 0, delivered: 0, open: [] }, "a POST is not a tunnel"); + const tunnel = log.openTunnel("turn"); + advance(1_000); + assert.equal(log.observe("turn")?.tunnels?.open.length, 1); + tunnel.close(); + // And the third state, which is not zero: a turn the facade never registered. + assert.equal(log.observe("absent"), undefined); + log.openTunnel("absent").deliver(); + assert.equal(log.observe("absent"), undefined, "an unopened turn keeps nothing"); +}); + +test("a tunnel the mount pushed through is a different fact from one held open in silence", () => { + const { log, advance } = clock(); + log.open("turn"); + const silent = log.openTunnel("turn"); + const pushing = log.openTunnel("turn"); + advance(90_000); + pushing.deliver(); pushing.deliver(); + assert.deepEqual(log.observe("turn")?.tunnels, { + opened: 2, closed: 0, delivered: 1, + open: [{ openMs: 90_000, delivered: false }, { openMs: 90_000, delivered: true }] + }); + silent.close(); silent.close(); + assert.deepEqual(log.observe("turn")?.tunnels?.closed, 1, "closing twice closes one tunnel"); +}); + +test("the open-tunnel list is bounded, and the counts still name every tunnel beyond it", () => { + const { log, advance } = clock(); + log.open("turn"); + for (let index = 0; index < ENGINE_BROKER_MCP_TUNNEL_MAX + 3; index += 1) { log.openTunnel("turn"); advance(1); } + const tunnels = log.observe("turn")?.tunnels; + assert.equal(tunnels?.opened, ENGINE_BROKER_MCP_TUNNEL_MAX + 3); + assert.equal(tunnels?.open.length, ENGINE_BROKER_MCP_TUNNEL_MAX); + assert.equal(tunnels?.open[0]?.openMs, ENGINE_BROKER_MCP_TUNNEL_MAX + 3, "the earliest tunnels are the ones kept"); +}); diff --git a/src/runtime/engineBrokerMcpCallLog.ts b/src/runtime/engineBrokerMcpCallLog.ts index fc49dbf..b680c48 100644 --- a/src/runtime/engineBrokerMcpCallLog.ts +++ b/src/runtime/engineBrokerMcpCallLog.ts @@ -30,6 +30,24 @@ * the one parse is wrapped, and the facade treats a missing handle as a * no-op. * + * The same map carries the facade's *other* channel, and for the same reason. + * A `tools/call` is a POST that answers; the standalone `GET` SSE tunnel the + * Streamable HTTP transport opens once per session is the route a server + * notification or progress frame takes, and it stays open for the whole + * session by design. A worker parked reading that tunnel is, in every artifact + * the broker writes, indistinguishable from a worker doing nothing at all: + * every provider request closed, every tool call answered, and the turn idle + * until its deadline. {@link EngineBrokerMcpCallLog.openTunnel} records the + * lifecycle — how many the facade relayed, how many ended, and for the ones + * still open at seal time how long each has been open and whether the mount + * ever pushed a single byte through it. A tunnel held open having delivered + * nothing is a different fact from one actively carrying frames, and it is the + * difference that decides whether the tunnel is the blocker. + * + * Observing is all it does. The facade's behaviour is unchanged: nothing here + * closes, times out or refuses a tunnel, because an instrument that tore down + * the stream would destroy the evidence it exists to gather. + * * "Answered" means the facade wrote a complete response back to the worker — * the relay reached its own `end()`. A relay that was torn down (the worker * died, the tunnel broke, the turn aborted) did *not* answer, so its calls stay @@ -44,27 +62,45 @@ export const ENGINE_BROKER_MCP_CALL_TRUNCATED = ""; export const ENGINE_BROKER_MCP_CALL_NAME = /^(?:||[A-Za-z0-9_.-]{1,64})$/u; const TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/u; +/** Open GET tunnels reported: a session opens one, so more than a handful is already the anomaly. */ +export const ENGINE_BROKER_MCP_TUNNEL_MAX = 8; + export type EngineBrokerOutstandingMcpCall = Readonly<{ name: string; outstandingMs: number }>; +/** One GET SSE tunnel still open at observation: how long it has been open, and whether the mount ever pushed through it. */ +export type EngineBrokerOpenMcpTunnel = Readonly<{ openMs: number; delivered: boolean }>; +/** + * What the facade saw of one turn's standalone GET SSE tunnels: how many it + * relayed, how many ended, how many ever carried a byte from the mount, and + * the ones still open with the age of each. + */ +export type EngineBrokerMcpTunnelObservation = Readonly<{ opened: number; closed: number; delivered: number; open: readonly EngineBrokerOpenMcpTunnel[] }>; /** * What the facade saw of one turn's tool calls: how many started, how many the * facade answered, how many POST bodies it could not read, and the ones still * unanswered with the time each has been outstanding. */ -export type EngineBrokerMcpCallObservation = Readonly<{ started: number; answered: number; undecoded: number; outstanding: readonly EngineBrokerOutstandingMcpCall[] }>; +export type EngineBrokerMcpCallObservation = Readonly<{ started: number; answered: number; undecoded: number; outstanding: readonly EngineBrokerOutstandingMcpCall[]; tunnels?: EngineBrokerMcpTunnelObservation }>; /** One relayed POST's calls. `answer` marks a complete relay; `close` ends it unanswered. Both are idempotent. */ export interface EngineBrokerMcpCallHandle { answer(): void; close(): void } const INERT: EngineBrokerMcpCallHandle = { answer: () => undefined, close: () => undefined }; +/** One relayed GET tunnel. `deliver` marks the first byte the mount pushed; `close` ends it. Both are idempotent. */ +export interface EngineBrokerMcpTunnelHandle { deliver(): void; close(): void } +const INERT_TUNNEL: EngineBrokerMcpTunnelHandle = { deliver: () => undefined, close: () => undefined }; type CallRecord = { readonly name: string; readonly startedAt: number; endedAt?: number }; -type TurnLog = { started: number; answered: number; undecoded: number; readonly live: Set; readonly ended: CallRecord[] }; +type TunnelRecord = { readonly openedAt: number; delivered: boolean }; +type TurnLog = { + started: number; answered: number; undecoded: number; readonly live: Set; readonly ended: CallRecord[]; + tunnelsOpened: number; tunnelsClosed: number; tunnelsDelivered: number; readonly openTunnels: Set; +}; export class EngineBrokerMcpCallLog { private readonly turns = new Map(); constructor(private readonly now: () => number = Date.now) {} /** A turn the facade registered. Re-opening an id resets it: a turn id is unique per turn. */ - open(turnId: string): void { this.turns.set(turnId, { started: 0, answered: 0, undecoded: 0, live: new Set(), ended: [] }); } + open(turnId: string): void { this.turns.set(turnId, { started: 0, answered: 0, undecoded: 0, live: new Set(), ended: [], tunnelsOpened: 0, tunnelsClosed: 0, tunnelsDelivered: 0, openTunnels: new Set() }); } close(turnId: string): void { this.turns.delete(turnId); } /** A POST body the facade is about to relay. Anything that is not a `tools/call` records nothing. */ @@ -98,6 +134,24 @@ export class EngineBrokerMcpCallLog { }; } + /** + * A GET SSE tunnel the facade is about to relay. Counted when it opens, not + * when it succeeds: a tunnel the mount refused still ends, so `opened` and + * `closed` stay a pair and an open one is exactly `opened - closed`. + */ + openTunnel(turnId: string): EngineBrokerMcpTunnelHandle { + const log = this.turns.get(turnId); + if (log === undefined) return INERT_TUNNEL; + const record: TunnelRecord = { openedAt: this.now(), delivered: false }; + log.tunnelsOpened += 1; + log.openTunnels.add(record); + let ended = false; + return { + deliver: (): void => { if (record.delivered) return; record.delivered = true; log.tunnelsDelivered += 1; }, + close: (): void => { if (ended) return; ended = true; log.tunnelsClosed += 1; log.openTunnels.delete(record); } + }; + } + /** A POST whose body the facade refused to read (over its own bound), which is a call it cannot name. */ undecodable(turnId: string): void { const log = this.turns.get(turnId); if (log !== undefined) log.undecoded += 1; } @@ -107,8 +161,13 @@ export class EngineBrokerMcpCallLog { const at = this.now(); const pending = [...log.live, ...log.ended].sort((left, right) => left.startedAt - right.startedAt); const outstanding = pending.map((record): EngineBrokerOutstandingMcpCall => ({ name: record.name, outstandingMs: Math.max(0, (record.endedAt ?? at) - record.startedAt) })); + const open = [...log.openTunnels] + .sort((left, right) => left.openedAt - right.openedAt) + .slice(0, ENGINE_BROKER_MCP_TUNNEL_MAX) + .map((record): EngineBrokerOpenMcpTunnel => ({ openMs: Math.max(0, at - record.openedAt), delivered: record.delivered })); return { started: log.started, answered: log.answered, undecoded: log.undecoded, + tunnels: { opened: log.tunnelsOpened, closed: log.tunnelsClosed, delivered: log.tunnelsDelivered, open }, outstanding: outstanding.length > ENGINE_BROKER_MCP_OUTSTANDING_MAX ? [...outstanding.slice(0, ENGINE_BROKER_MCP_OUTSTANDING_MAX - 1), { name: ENGINE_BROKER_MCP_CALL_TRUNCATED, outstandingMs: 0 }] : outstanding diff --git a/src/runtime/engineBrokerMcpFacade.test.ts b/src/runtime/engineBrokerMcpFacade.test.ts index 54261ed..ff7d85e 100644 --- a/src/runtime/engineBrokerMcpFacade.test.ts +++ b/src/runtime/engineBrokerMcpFacade.test.ts @@ -12,6 +12,7 @@ import { Type } from "@earendil-works/pi-ai"; import { defineTool } from "@earendil-works/pi-coding-agent"; import { createPiToolMcpServer } from "../mcp/toolServer.js"; +import type { EngineBrokerMcpTunnelObservation } from "./engineBrokerMcpCallLog.js"; import { awaitMcpTunnelDrain, ENGINE_BROKER_MCP_FACADE_PORT, startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; const FACADE_URL = `http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}/mcp`; @@ -265,6 +266,60 @@ test("the facade carries the mount's server-initiated SSE stream, which only the } }); +/** + * The channel the call log could not see. + * + * A brokered turn's tool calls all answer and its provider requests all close, + * and the worker can still sit idle to the deadline — parked on the standalone + * GET SSE tunnel, which stays open for the whole session and, until now, wrote + * nothing anywhere. This drives the real transport: the tunnel the real client + * opens must observe as open with an age while it is open, as *delivered* once + * the mount pushes a frame through it, and as closed once the client ends it. + * + * Mutation: remove `calls.openTunnel` from the facade's route and the first + * assertion goes red (an open tunnel reads as a turn that opened none); remove + * `tunnel?.close()` from the relay's `finally` and the last one does (a closed + * tunnel reads as still open, which is the reading the whole instrument is + * meant to make trustworthy). + */ +test("the facade observes the standalone GET tunnel: open with its age, whether it delivered, and closed when it ends", async () => { + const rig = await startRig(); + const tunnels = (): EngineBrokerMcpTunnelObservation | undefined => rig.facade.observe(rig.turnId)?.tunnels; + const until = async (reason: string, ready: () => boolean): Promise => { + for (let attempt = 0; attempt < 200 && !ready(); attempt += 1) await new Promise((resolve) => setTimeout(resolve, 25)); + assert.ok(ready(), reason); + }; + try { + // Before any request the turn is registered and has relayed nothing: a + // measured zero, which is not the same statement as an open tunnel. + assert.deepEqual(tunnels(), { opened: 0, closed: 0, delivered: 0, open: [] }); + const { client } = await connectClient(rig.capability); + const notified = new Promise((resolve) => client.setNotificationHandler(ToolListChangedNotificationSchema, () => resolve())); + try { + await until("the client never opened its GET tunnel", () => (tunnels()?.open.length ?? 0) > 0); + const open = tunnels(); + assert.equal(open?.opened, 1); assert.equal(open?.closed, 0); + assert.ok((open?.open[0]?.openMs ?? -1) >= 0, "an open tunnel reports how long it has been open"); + assert.equal(open?.open[0]?.delivered, false, "a tunnel the mount has not pushed through is held open in silence"); + assert.equal(open?.delivered, 0); + + // The same tunnel, now actually carrying a server frame. "Held open + // having delivered nothing" and "in use" are different facts about it. + rig.server.sendToolListChanged(); + await withDeadline(notified, 4_000, "no server notification reached the client"); + await until("the delivered frame was never attributed to the tunnel", () => (tunnels()?.delivered ?? 0) === 1); + assert.equal(tunnels()?.open[0]?.delivered, true); + assert.equal(tunnels()?.closed, 0, "a tunnel that delivered is still open"); + } finally { + await client.close(); + } + await until("the closed GET tunnel still reads as open", () => (tunnels()?.closed ?? 0) === 1); + assert.deepEqual(tunnels(), { opened: 1, closed: 1, delivered: 1, open: [] }); + } finally { + await rig.close(); + } +}); + test("the facade carries the session-closing DELETE, and the mount then refuses the stale session", async () => { const rig = await startRig(); try { diff --git a/src/runtime/engineBrokerMcpFacade.ts b/src/runtime/engineBrokerMcpFacade.ts index 25a741f..51e4e0e 100644 --- a/src/runtime/engineBrokerMcpFacade.ts +++ b/src/runtime/engineBrokerMcpFacade.ts @@ -1,7 +1,7 @@ import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import { Readable } from "node:stream"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; -import { EngineBrokerMcpCallLog, type EngineBrokerMcpCallObservation } from "./engineBrokerMcpCallLog.js"; +import { EngineBrokerMcpCallLog, type EngineBrokerMcpCallObservation, type EngineBrokerMcpTunnelHandle } from "./engineBrokerMcpCallLog.js"; /** * The brokered worker's only route to its own per-wake Daimon MCP mount. The @@ -90,6 +90,11 @@ export async function startEngineBrokerMcpFacade() { catch (error) { calls.undecodable(scope.turnId); throw error; } const payload = body === undefined ? undefined : (body.buffer.slice(body.byteOffset, body.byteOffset + body.byteLength) as ArrayBuffer); const call = calls.begin(scope.turnId, body); + // The GET SSE tunnel is the session's other channel and the one that + // outlives every request: it stays open until the worker or the mount ends + // it, so a turn sealed with one still open is a turn whose worker may be + // parked on it. Observed only — never closed, timed out or refused here. + const tunnel = method === "GET" ? calls.openTunnel(scope.turnId) : undefined; const controller = new AbortController(); const open = inflight.get(scope.turnId) ?? new Set(); @@ -101,9 +106,10 @@ export async function startEngineBrokerMcpFacade() { // Answered only on a relay that reached its own end: a tunnel torn down // by the worker's death must not mark the call it was blocked on as // finished. - if (await forward(target, method, headersFor(method, request), payload, controller.signal, response)) call.answer(); + if (await forward(target, method, headersFor(method, request), payload, controller.signal, response, tunnel)) call.answer(); } finally { call.close(); + tunnel?.close(); response.off("close", abort); open.delete(controller); if (open.size === 0) inflight.delete(scope.turnId); @@ -136,7 +142,8 @@ export async function startEngineBrokerMcpFacade() { calls.close(turnId); }, /** - * What the facade saw of this turn's tool calls, or `undefined` for a turn + * What the facade saw of this turn's tool calls and GET tunnels, or + * `undefined` for a turn * it never registered. Read on the failure path, before `revoke`. */ observe: (turnId: string): EngineBrokerMcpCallObservation | undefined => calls.observe(turnId), @@ -179,7 +186,8 @@ async function forward( headers: Record, body: ArrayBuffer | undefined, signal: AbortSignal, - response: ServerResponse + response: ServerResponse, + tunnel?: EngineBrokerMcpTunnelHandle ): Promise { const upstream = await fetch(target, { method, headers, body, signal, redirect: "manual" }); // MCP never redirects, and following one would let the mount aim the facade @@ -196,8 +204,12 @@ async function forward( response.writeHead(upstream.status, outbound); if (upstream.body === null) { response.end(); return true; } const stream = Readable.fromWeb(upstream.body as Parameters[0]); + let delivered = false; try { for await (const chunk of stream) { + // The first byte the mount pushes: a tunnel that carried frames is a + // different fact from one held open having delivered nothing. + if (!delivered) { delivered = true; tunnel?.deliver(); } if (response.destroyed || response.writableEnded) throw new Error("MCP tunnel closed"); if (!response.write(chunk as Uint8Array)) await awaitMcpTunnelDrain(response, signal); } diff --git a/src/runtime/engineBrokerProtocol.test.ts b/src/runtime/engineBrokerProtocol.test.ts index 6f57c10..835493c 100644 --- a/src/runtime/engineBrokerProtocol.test.ts +++ b/src/runtime/engineBrokerProtocol.test.ts @@ -101,4 +101,27 @@ test("a failed frame carries the broker's in-flight MCP tool-call observation, b ]) assert.throws(() => parseEngineBrokerResponse({ ...value, mcpCalls }), /invalid broker frame/u, JSON.stringify(mcpCalls)); // A v1 record predates the instrument; a v1 frame that carries it is forged. assert.throws(() => parseEngineBrokerV1TerminalResponse({ version: "noopolis.daimon.engine-broker.v1", kind: "failed", requestId: "request-1", turnId: "turn-1", code: "engine_failed", mcpCalls: value.mcpCalls }), /invalid broker frame/u); + + /** + * The session's standalone GET SSE tunnel rides the same member, under the + * same rules. It is optional for exactly one reason — a turn sealed before + * the facade observed that channel replays without it — so its absence means + * "not measured" and never zero, and a record that carries it must still be + * a measurement: nothing closes before it opens, nothing delivers without + * opening, and no more can be open than `opened - closed`. + */ + const tunnels = { opened: 2, closed: 1, delivered: 1, open: [{ openMs: 428_004, delivered: false }] } as const; + const observed = { ...value, mcpCalls: { ...value.mcpCalls, tunnels } } as const; + assert.deepEqual(parseEngineBrokerResponse(observed), observed); + assert.deepEqual(parseEngineBrokerResponse(value), value, "a frame sealed before the tunnel was observed still replays, without the member"); + for (const forged of [ + { ...tunnels, closed: 3 }, + { ...tunnels, delivered: 3 }, + { ...tunnels, opened: 1, closed: 1, open: [{ openMs: 1, delivered: false }] }, + { ...tunnels, open: Array.from({ length: 9 }, () => ({ openMs: 1, delivered: false })), opened: 12, closed: 0 }, + { ...tunnels, open: [{ openMs: -1, delivered: false }] }, + { ...tunnels, open: [{ openMs: 1, delivered: "yes" }] }, + { ...tunnels, open: [{ openMs: 1, delivered: false, sessionId: "mcp-session-0" }] }, + { opened: 1, closed: 0, open: [] } + ]) assert.throws(() => parseEngineBrokerResponse({ ...value, mcpCalls: { ...value.mcpCalls, tunnels: forged } }), /invalid broker frame/u, JSON.stringify(forged)); }); diff --git a/src/runtime/engineBrokerProtocol.ts b/src/runtime/engineBrokerProtocol.ts index 643e41c..201d005 100644 --- a/src/runtime/engineBrokerProtocol.ts +++ b/src/runtime/engineBrokerProtocol.ts @@ -1,5 +1,5 @@ import { isEngineBrokerInferenceRequestKind, isEngineBrokerInferenceResponseKind, parseEngineBrokerInferenceRequest, parseEngineBrokerInferenceResponse, type EngineBrokerInferenceRequest, type EngineBrokerInferenceResponse } from "./engineBrokerInferenceProtocol.js"; -import { ENGINE_BROKER_MCP_CALL_NAME, ENGINE_BROKER_MCP_OUTSTANDING_MAX, type EngineBrokerMcpCallObservation } from "./engineBrokerMcpCallLog.js"; +import { ENGINE_BROKER_MCP_CALL_NAME, ENGINE_BROKER_MCP_OUTSTANDING_MAX, ENGINE_BROKER_MCP_TUNNEL_MAX, type EngineBrokerMcpCallObservation, type EngineBrokerMcpTunnelObservation } from "./engineBrokerMcpCallLog.js"; import { parseEngineBrokerTurnAccounting, parseEngineBrokerTurnLimitOverrides, type EngineBrokerTurnAccounting, type EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; /** @@ -143,7 +143,10 @@ function parseTerminal(input: JsonRecord, expected: typeof VERSION | typeof V1): */ function parseMcpCallObservation(value: unknown): EngineBrokerMcpCallObservation { const input = record(value); - exact(input, ["started", "answered", "undecoded", "outstanding"]); + // `tunnels` is optional for one reason only: a turn sealed before the GET + // tunnel was observed carries no such member, and its record must still + // replay. Absence there means "the instrument did not exist", never zero. + exact(input, ["started", "answered", "undecoded", "outstanding", ...(input.tunnels === undefined ? [] : ["tunnels"])]); const started = input.started, answered = input.answered, undecoded = input.undecoded; if (![started, answered, undecoded].every((count) => Number.isSafeInteger(count) && (count as number) >= 0)) throw new TypeError("invalid broker frame"); if (!Array.isArray(input.outstanding) || input.outstanding.length > ENGINE_BROKER_MCP_OUTSTANDING_MAX) throw new TypeError("invalid broker frame"); @@ -155,7 +158,31 @@ function parseMcpCallObservation(value: unknown): EngineBrokerMcpCallObservation return { name: call.name, outstandingMs: call.outstandingMs as number }; }); if ((answered as number) > (started as number) || outstanding.length > (started as number) - (answered as number)) throw new TypeError("invalid broker frame"); - return { started: started as number, answered: answered as number, undecoded: undecoded as number, outstanding }; + const tunnels = input.tunnels === undefined ? undefined : parseMcpTunnelObservation(input.tunnels); + return { started: started as number, answered: answered as number, undecoded: undecoded as number, outstanding, ...(tunnels === undefined ? {} : { tunnels }) }; +} + +/** + * The GET SSE tunnels, under the call observation's rules: counts and elapsed + * milliseconds, bounded, and internally consistent. A tunnel cannot close + * before it opened, cannot deliver without having opened, and no more can be + * reported open than `opened - closed` — a report claiming otherwise is a + * frame, not a measurement. + */ +function parseMcpTunnelObservation(value: unknown): EngineBrokerMcpTunnelObservation { + const input = record(value); + exact(input, ["opened", "closed", "delivered", "open"]); + const opened = input.opened, closed = input.closed, delivered = input.delivered; + if (![opened, closed, delivered].every((count) => Number.isSafeInteger(count) && (count as number) >= 0)) throw new TypeError("invalid broker frame"); + if ((closed as number) > (opened as number) || (delivered as number) > (opened as number)) throw new TypeError("invalid broker frame"); + if (!Array.isArray(input.open) || input.open.length > ENGINE_BROKER_MCP_TUNNEL_MAX || input.open.length > (opened as number) - (closed as number)) throw new TypeError("invalid broker frame"); + const open = input.open.map((entry) => { + const tunnel = record(entry); + exact(tunnel, ["openMs", "delivered"]); + if (!Number.isSafeInteger(tunnel.openMs) || (tunnel.openMs as number) < 0 || typeof tunnel.delivered !== "boolean") throw new TypeError("invalid broker frame"); + return { openMs: tunnel.openMs as number, delivered: tunnel.delivered }; + }); + return { opened: opened as number, closed: closed as number, delivered: delivered as number, open }; } function closedDiagnostic(value:JsonRecord):boolean{ diff --git a/src/runtime/engineBrokerSealLedger.test.ts b/src/runtime/engineBrokerSealLedger.test.ts index 9b5ecc2..60754e3 100644 --- a/src/runtime/engineBrokerSealLedger.test.ts +++ b/src/runtime/engineBrokerSealLedger.test.ts @@ -120,3 +120,41 @@ test("a cancelled turn that called no tool is distinguishable from one the facad try { await assert.rejects(readFile(engineBrokerSealLedgerPathFor(path.join(root, "usage.jsonl")), "utf8")); } finally { await rm(root, { recursive: true, force: true }); } }); + +/** + * The channel the seal row could not see, on the same durable route. + * + * Seven live runs sealed with every provider request closed and every tool call + * answered, and still went idle to the deadline. The facade relays one more + * thing for the whole session — the standalone GET SSE tunnel — and recorded + * nothing about it, so a worker parked reading that stream and a worker doing + * nothing wrote identical rows. These three seal the boundary that separates + * them: still open at seal time, closed before it, and never opened at all. + * + * Mutation: drop the `tunnels` member from `renderBrokerTurnSealLine` and the + * first three go red; render it unconditionally as zeros when the observation + * carries none, and the fourth does — a zero nobody measured reads exactly + * like a zero somebody did. + */ +test("a cancelled turn's GET tunnel is sealed open with its age, closed, or never opened — three distinct rows", async () => { + const mcp = (tunnels: Record): EngineBrokerMcpCallObservation => + ({ started: 1, answered: 1, undecoded: 0, outstanding: [], ...tunnels } as EngineBrokerMcpCallObservation); + + const parked = await cancelledTurn(() => mcp({ tunnels: { opened: 1, closed: 0, delivered: 0, open: [{ openMs: 428_004, delivered: false }] } })); + assert.deepEqual((parked.seal?.mcp as Record).tunnels, { + opened: 1, closed: 0, delivered: 0, open: [{ open_ms: 428_004, delivered: false }] + }, "a turn sealed with a tunnel still open must say so, and say how long it had been open"); + + const ended = await cancelledTurn(() => mcp({ tunnels: { opened: 1, closed: 1, delivered: 2, open: [] } })); + assert.deepEqual((ended.seal?.mcp as Record).tunnels, { opened: 1, closed: 1, delivered: 2, open: [] }, + "a tunnel that closed before the seal is not an open one"); + + const never = await cancelledTurn(() => mcp({ tunnels: { opened: 0, closed: 0, delivered: 0, open: [] } })); + assert.deepEqual((never.seal?.mcp as Record).tunnels, { opened: 0, closed: 0, delivered: 0, open: [] }, + "a turn whose facade never relayed a GET measured zero, which is not the same as not having looked"); + + // And the fourth state, which is the absence: a turn sealed before this + // channel was observed at all carries no `tunnels` member. + const unobserved = await cancelledTurn(() => ({ started: 1, answered: 1, undecoded: 0, outstanding: [] })); + assert.equal(Object.hasOwn(unobserved.seal?.mcp as Record, "tunnels"), false); +}); diff --git a/src/runtime/engineBrokerSealLedger.ts b/src/runtime/engineBrokerSealLedger.ts index 52d38c1..d9fd3f4 100644 --- a/src/runtime/engineBrokerSealLedger.ts +++ b/src/runtime/engineBrokerSealLedger.ts @@ -1,4 +1,4 @@ -import { ENGINE_BROKER_MCP_CALL_NAME, ENGINE_BROKER_MCP_OUTSTANDING_MAX } from "./engineBrokerMcpCallLog.js"; +import { ENGINE_BROKER_MCP_CALL_NAME, ENGINE_BROKER_MCP_OUTSTANDING_MAX, ENGINE_BROKER_MCP_TUNNEL_MAX } from "./engineBrokerMcpCallLog.js"; import type { EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; import { TURN_USAGE_LEDGER, TURN_USAGE_MAX_IDENTIFIER_CHARS } from "./turnUsageLedger.js"; @@ -27,7 +27,8 @@ import { TURN_USAGE_LEDGER, TURN_USAGE_MAX_IDENTIFIER_CHARS } from "./turnUsageL * the accounting, the diagnostic's closed `status`/`stage`/`failure_class` * and its already-redacted, already-bounded, control-character-free `reason` * (`engineBrokerNativeClient.ts` produced it; nothing here re-derives it), - * plus tool-call names and elapsed milliseconds. Never a prompt, a body, a + * plus tool-call names, GET-tunnel counts and elapsed milliseconds. Never a + * prompt, a body, a * reply, a bearer, a capability or a session id — none of which the terminal * response carries in the first place. * - **Absence stays absence.** `mcp` is written only when the facade actually @@ -102,7 +103,21 @@ export const renderBrokerTurnSealLine = (terminal: EngineBrokerTerminalResponse, undecoded: terminal.mcpCalls.undecoded, outstanding: terminal.mcpCalls.outstanding .slice(0, ENGINE_BROKER_MCP_OUTSTANDING_MAX) - .map((call) => ({ name: ENGINE_BROKER_MCP_CALL_NAME.test(call.name) ? call.name : "", outstanding_ms: call.outstandingMs })) + .map((call) => ({ name: ENGINE_BROKER_MCP_CALL_NAME.test(call.name) ? call.name : "", outstanding_ms: call.outstandingMs })), + // The session's standalone GET tunnel, absent for a turn sealed before + // the facade observed that channel at all. + ...(terminal.mcpCalls.tunnels === undefined + ? {} + : { + tunnels: { + opened: terminal.mcpCalls.tunnels.opened, + closed: terminal.mcpCalls.tunnels.closed, + delivered: terminal.mcpCalls.tunnels.delivered, + open: terminal.mcpCalls.tunnels.open + .slice(0, ENGINE_BROKER_MCP_TUNNEL_MAX) + .map((tunnel) => ({ open_ms: tunnel.openMs, delivered: tunnel.delivered })) + } + }) } } : {}) From edbbb998d6185c1ecd40ed312709d60a5cb8d052 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 13:02:12 +0200 Subject: [PATCH 57/69] docs: record the MCP GET tunnel observation and what the real CLI does with it --- src/runtime/AGENTS.md | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index f4243e1..db88d50 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -314,6 +314,31 @@ answer, so the call it was blocked on stays outstanding with the elapsed time it had reached — otherwise the turn's death would erase the evidence the instrument exists to keep. +The facade relays one more thing, for the whole session, and until now wrote +nothing about it. A `tools/call` is a POST that answers; the standalone `GET` +SSE tunnel is the route a server notification or progress frame takes, and it +stays open from `initialize` to the worker's own shutdown. A worker parked +reading it was, in every artifact the broker wrote, identical to a worker doing +nothing: every provider request closed, every tool call answered, idle to the +deadline. `EngineBrokerMcpCallLog.openTunnel` records that lifecycle on the same +observation — `tunnels: {opened, closed, delivered, open: [{openMs, delivered}]}` +— so a turn sealed with one still open says so and says how long it had been +open, and `delivered` separates a tunnel actively carrying frames from one held +open having received nothing, which is the difference that decides whether it is +the blocker. Bounded at `ENGINE_BROKER_MCP_TUNNEL_MAX` open entries (a session +opens one), counts and elapsed milliseconds only, never a frame, an event +payload or a session id. It is *observation only*: nothing here closes, times +out or refuses a tunnel, because an instrument that tore the stream down would +destroy the evidence it exists to gather. The member is optional on the wire for +one reason — a turn sealed before it existed must still replay — so its absence +means "not measured" and never zero, exactly as `mcp`'s own absence does. +Measured against the real CLI (rig, grok 1.0.34, real facade and mount): the +tunnel opens ~3 ms after `initialize`, carries nothing for its whole life, and +**closes 16 ms before the worker exits** — the close is the worker's own +shutdown, not the facade's. A turn that never reaches that shutdown is the one +that seals with it open; a deliberately stalled `tools/call` sealed +`open: [{openMs: 14652, delivered: false}]` beside its outstanding call. + That seam is enough for a turn that *fails with a reply* and not for the turn the instrument was built for. A worker that crashes still produces a terminal response; a worker that HANGS is cancelled by its client's deadline, and a @@ -328,7 +353,8 @@ terminal turn (`turns.jsonl`, `engineBrokerSealLedgerPathFor`), rendered from the sealed response and nothing else. Its members are the accounting, the failure `code`, the diagnostic's closed `status`/`stage`/`failure_class` with the reason `engineBrokerNativeClient.ts` already redacted and bounded, and the -facade's `mcp` observation — names, counts and elapsed milliseconds. Never a +facade's `mcp` observation — names, counts, GET-tunnel lifecycle and elapsed +milliseconds. Never a prompt, body, reply, bearer, capability or session id; the terminal response carries none of those in the first place, and the projection is an allow-list rather than a spread, so a future additive member of the response cannot become From 1a5ba8dc13b30e5809a39de118bf950156874ba6 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 13:04:49 +0200 Subject: [PATCH 58/69] docs: name the tunnel timings in the seal line's own bound --- src/runtime/engineBrokerSealLedger.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/engineBrokerSealLedger.ts b/src/runtime/engineBrokerSealLedger.ts index d9fd3f4..c386838 100644 --- a/src/runtime/engineBrokerSealLedger.ts +++ b/src/runtime/engineBrokerSealLedger.ts @@ -56,7 +56,7 @@ export const TURN_SEAL_LEDGER = { fileMode: TURN_USAGE_LEDGER.fileMode } as const; -/** A rendered seal line is bounded by its own contents: a 768-byte reason plus 16 bounded names. */ +/** A rendered seal line is bounded by its own contents: a 768-byte reason, 16 bounded names and 8 tunnel timings. */ export const TURN_SEAL_MAX_LINE_BYTES = 8_192; export type BrokerTurnSealEntry = Readonly<{ agent: string; wake: string; at: string }>; From 130b548b5a64d6f4584a2fa66f040547d9896d3f Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 13:11:50 +0200 Subject: [PATCH 59/69] test: split the facade's observation suite and its rig out of the facade tests --- src/runtime/engineBrokerMcpFacade.test.ts | 268 +----------------- src/runtime/engineBrokerMcpFacadeRig.test.ts | 164 +++++++++++ .../engineBrokerMcpObservation.test.ts | 155 ++++++++++ 3 files changed, 322 insertions(+), 265 deletions(-) create mode 100644 src/runtime/engineBrokerMcpFacadeRig.test.ts create mode 100644 src/runtime/engineBrokerMcpObservation.test.ts diff --git a/src/runtime/engineBrokerMcpFacade.test.ts b/src/runtime/engineBrokerMcpFacade.test.ts index ff7d85e..6ad19ca 100644 --- a/src/runtime/engineBrokerMcpFacade.test.ts +++ b/src/runtime/engineBrokerMcpFacade.test.ts @@ -1,50 +1,13 @@ import assert from "node:assert/strict"; -import { createServer, type Server as HttpServer, type IncomingMessage, type ServerResponse } from "node:http"; +import { createServer } from "node:http"; -import { randomUUID } from "node:crypto"; import test from "node:test"; -import { Client } from "@modelcontextprotocol/sdk/client/index.js"; -import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; -import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; import { ToolListChangedNotificationSchema } from "@modelcontextprotocol/sdk/types.js"; -import { Type } from "@earendil-works/pi-ai"; -import { defineTool } from "@earendil-works/pi-coding-agent"; -import { createPiToolMcpServer } from "../mcp/toolServer.js"; -import type { EngineBrokerMcpTunnelObservation } from "./engineBrokerMcpCallLog.js"; -import { awaitMcpTunnelDrain, ENGINE_BROKER_MCP_FACADE_PORT, startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; +import { ENGINE_BROKER_MCP_FACADE_PORT } from "./engineBrokerMcpFacade.js"; -const FACADE_URL = `http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}/mcp`; -type Facade = Awaited>; - -/** - * One facade serves every turn of a broker, so the tests share one too. It - * also keeps the fixed port free: a facade per test would leave the HTTP - * client pooling a socket onto a server that no longer exists. - */ -let shared: Facade | undefined; -const sharedFacade = async (): Promise => (shared ??= await startFacade()); -const releaseShared = async (): Promise => { const facade = shared; shared = undefined; if (facade) await facade.close(); }; -test.after(releaseShared); - -/** - * Closing a facade destroys its sockets, and the port is fixed, so the HTTP - * client can still hold a pooled connection to the server that just went away. - * That is a test-harness artifact — one facade outlives a whole broker — so a - * fresh facade is probed until a refusal proves the route is live again. - */ -const startFacade = async (): Promise => { - const facade = await startEngineBrokerMcpFacade(); - for (let attempt = 0; attempt < 20; attempt += 1) { - try { - const probe = await fetch(FACADE_URL, { method: "PUT" }); - await probe.body?.cancel(); - if (probe.status === 403) return facade; - } catch { /* a pooled socket onto the previous facade: try the next one */ } - } - throw new Error("facade did not answer after starting"); -}; +import { connectClient, FACADE_URL, releaseShared, sharedFacade, startFacade, startRig, withDeadline } from "./engineBrokerMcpFacadeRig.test.js"; test("MCP facade routes only valid active capabilities to the registered mount", async () => { const facade=await sharedFacade(); @@ -53,177 +16,6 @@ test("MCP facade routes only valid active capabilities to the registered mount", try{assert.equal((await call("wrong-token-abcdefghijklmnopqrstuvwxyz0123456789")).status,403);assert.equal((await call(token)).status,200);assert.equal(calls,1);facade.revoke("turn-capabilities");assert.equal((await call(token)).status,403);assert.equal(calls,1);}finally{facade.revoke("turn-capabilities");await new Promise((resolve)=>target.close(()=>resolve()));} }); -/** - * Daimon writes a tool receipt only on completion, so a call that started and - * never returned reads exactly like a call that was never made — the one path - * a seven-minute live hang left unlit. The facade is where that difference is - * visible, and it has to survive the tear-down that ends the turn: a tunnel - * destroyed when the worker dies must not mark the call it was blocked on as - * answered, or the instrument erases the very evidence it exists to keep. - */ -test("MCP facade reports a tool call that started and never returned as outstanding, by name", async () => { - const facade = await sharedFacade(); - const held: ServerResponse[] = []; - // Two ways for a mount not to answer: never reply at all (`moltnet_read`), - // or open the stream and never deliver the result (`memory_recall`). - const target = createServer((request, response) => { - const chunks: Buffer[] = []; - request.on("data", (chunk: Buffer) => chunks.push(chunk)); - request.on("end", () => { - const asked = Buffer.concat(chunks).toString("utf8"); - if (asked.includes("moltnet_read")) { held.push(response); return; } - if (asked.includes("memory_recall")) { response.writeHead(200, { "content-type": "text/event-stream" }); response.write(": open\n\n"); held.push(response); return; } - response.writeHead(200, { "content-type": "application/json" }); response.end('{"jsonrpc":"2.0","id":1,"result":{}}'); - }); - }); - await new Promise((resolve) => target.listen(0, "127.0.0.1", resolve)); - const address = target.address(); if (address === null || typeof address === "string") throw new Error(); - const turnId = "turn-outstanding", token = facade.register("agent", turnId, `http://127.0.0.1:${address.port}/mcp`); - const post = (name: string, signal?: AbortSignal): Promise => fetch(FACADE_URL, { method: "POST", signal, headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name, arguments: { text: "argument-bytes-that-must-never-be-recorded" } } }) }); - const pending = new AbortController(); - /** - * A live call's elapsed time grows with every read, so two identical reads - * mean every relay has settled — the only moment at which "answered" is - * final. Polling for a name instead would read the log mid-teardown. - */ - const settled = async (): Promise> => { - for (let attempt = 0; attempt < 100; attempt += 1) { - const before = JSON.stringify(facade.observe(turnId)); - await new Promise((resolve) => setTimeout(resolve, 60)); - if (JSON.stringify(facade.observe(turnId)) === before) return facade.observe(turnId); - } - throw new Error("the facade's observation never settled"); - }; - const names = (observed: ReturnType): readonly string[] => (observed?.outstanding ?? []).map((call) => call.name); - try { - const answered = await post("daimon__moltnet_send"); - assert.equal(answered.status, 200); await answered.text(); - const hanging = post("daimon__moltnet_read", pending.signal).catch(() => undefined); - for (let attempt = 0; attempt < 200 && held.length === 0; attempt += 1) await new Promise((resolve) => setTimeout(resolve, 25)); - assert.equal(held.length, 1, "the mount never received the hung tool call"); - - const observed = facade.observe(turnId); - assert.deepEqual(names(observed), ["daimon__moltnet_read"], "the unanswered call must be outstanding, by name"); - assert.equal(observed?.started, 2); assert.equal(observed?.answered, 1, "the completed call must not be outstanding"); assert.equal(observed?.undecoded, 0); - assert.ok((observed?.outstanding[0]?.outstandingMs ?? -1) >= 0, "an outstanding call reports how long it has been waiting"); - assert.ok(!JSON.stringify(observed).includes("argument-bytes"), "names and timings only: no arguments"); - - // The turn's own death tears the tunnel down. The call was still never - // answered, and must still say so. - pending.abort(); await hanging; - const afterTeardown = await settled(); - assert.deepEqual(names(afterTeardown), ["daimon__moltnet_read"], "a torn-down relay is not an answer"); - assert.equal(afterTeardown?.answered, 1); - - // A stream the facade opened and never finished relaying is not an answer - // either. Awaiting the headers and one chunk puts the facade inside its own - // streaming relay before the client walks away, which is the branch that - // decides whether a half-written tunnel counts as an answer. - const halted = new AbortController(); - const half = await post("memory_recall", halted.signal); - assert.equal(half.status, 200); await half.body!.getReader().read(); halted.abort(); - const afterHalfRelay = await settled(); - assert.deepEqual(names(afterHalfRelay), ["daimon__moltnet_read", "memory_recall"], "a half-relayed stream is not an answer"); - assert.equal(afterHalfRelay?.answered, 1); - - facade.revoke(turnId); - assert.equal(facade.observe(turnId), undefined, "a turn the facade never registered observes as absence, not as zero"); - } finally { - pending.abort(); facade.revoke(turnId); - for (const response of held) response.destroy(); - target.closeAllConnections(); - await new Promise((resolve) => target.close(() => resolve())); - } -}); - -/** - * The brokered worker's real route: a real Daimon MCP mount behind a real - * Streamable HTTP transport, reached by a real MCP client through the facade. - * Asserting that a header is copied would pass while the route stayed broken, - * so every case below drives the transport end to end. - */ -type Rig = Readonly<{ - facade: Facade; - turnId: string; - server: ReturnType; - capability: string; - observed: IncomingMessage[]; - /** Resolves when the mount's standalone GET stream is torn down. */ - getStreamClosed: Promise; - close: () => Promise; -}>; - -const echoTool = defineTool({ - name: "moltnet_read", - label: "Read a scoped Moltnet surface", - description: "Reads the fixture room.", - parameters: Type.Object({ target: Type.String() }, { additionalProperties: false }), - async execute(_toolCallId: string, params: { target: string }) { - return { content: [{ type: "text" as const, text: `read ${params.target}` }], details: { target: params.target } }; - } -}); - -let turns = 0; - -const startRig = async (facade?: Facade): Promise => { - const host = facade ?? await sharedFacade(); - const turnId = `turn-${++turns}`; - const server = createPiToolMcpServer([echoTool], {}); - const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: () => randomUUID() }); - await server.connect(transport); - const observed: IncomingMessage[] = []; - let noteGetStreamClosed = (): void => undefined; - const getStreamClosed = new Promise((resolve) => { noteGetStreamClosed = resolve; }); - const mount = createServer((request, response) => { - observed.push(request); - if (request.method === "GET") response.on("close", () => noteGetStreamClosed()); - const chunks: Buffer[] = []; - request.on("data", (chunk: Buffer) => chunks.push(chunk)); - request.on("end", () => { - const raw = Buffer.concat(chunks); - let parsed: unknown; - try { parsed = raw.length === 0 ? undefined : JSON.parse(raw.toString("utf8")); } catch { parsed = undefined; } - void transport.handleRequest(request, response, parsed); - }); - }); - await new Promise((resolve) => mount.listen(0, "127.0.0.1", resolve)); - const address = mount.address(); - if (address === null || typeof address === "string") throw new Error("mount address unavailable"); - const capability = host.register("alpha", turnId, `http://127.0.0.1:${address.port}/mcp`); - return { - facade: host, turnId, server, capability, observed, getStreamClosed, - close: async () => { - host.revoke(turnId); - await closeMount(mount, transport, server); - } - }; -}; - -const closeMount = async (mount: HttpServer, transport: StreamableHTTPServerTransport, server: ReturnType): Promise => { - mount.closeAllConnections(); - await new Promise((resolve) => mount.close(() => resolve())); - await transport.close().catch(() => undefined); - await server.close().catch(() => undefined); -}; - -const connectClient = async (capability: string): Promise<{ client: Client; transport: StreamableHTTPClientTransport }> => { - const transport = new StreamableHTTPClientTransport(new URL(FACADE_URL), { - requestInit: { headers: { authorization: `Bearer ${capability}` } } - }); - const client = new Client({ name: "daimon-facade-test-client", version: "0.1.0" }); - await client.connect(transport); - return { client, transport }; -}; - -const withDeadline = async (work: Promise, ms: number, reason: string): Promise => { - let timer: NodeJS.Timeout | undefined; - try { - return await Promise.race([work, new Promise((_resolve, reject) => { timer = setTimeout(() => reject(new Error(reason)), ms); })]); - } finally { - if (timer) clearTimeout(timer); - } -}; - test("a brokered worker completes the whole MCP handshake through the facade and calls a mounted tool", async () => { const rig = await startRig(); try { @@ -266,60 +58,6 @@ test("the facade carries the mount's server-initiated SSE stream, which only the } }); -/** - * The channel the call log could not see. - * - * A brokered turn's tool calls all answer and its provider requests all close, - * and the worker can still sit idle to the deadline — parked on the standalone - * GET SSE tunnel, which stays open for the whole session and, until now, wrote - * nothing anywhere. This drives the real transport: the tunnel the real client - * opens must observe as open with an age while it is open, as *delivered* once - * the mount pushes a frame through it, and as closed once the client ends it. - * - * Mutation: remove `calls.openTunnel` from the facade's route and the first - * assertion goes red (an open tunnel reads as a turn that opened none); remove - * `tunnel?.close()` from the relay's `finally` and the last one does (a closed - * tunnel reads as still open, which is the reading the whole instrument is - * meant to make trustworthy). - */ -test("the facade observes the standalone GET tunnel: open with its age, whether it delivered, and closed when it ends", async () => { - const rig = await startRig(); - const tunnels = (): EngineBrokerMcpTunnelObservation | undefined => rig.facade.observe(rig.turnId)?.tunnels; - const until = async (reason: string, ready: () => boolean): Promise => { - for (let attempt = 0; attempt < 200 && !ready(); attempt += 1) await new Promise((resolve) => setTimeout(resolve, 25)); - assert.ok(ready(), reason); - }; - try { - // Before any request the turn is registered and has relayed nothing: a - // measured zero, which is not the same statement as an open tunnel. - assert.deepEqual(tunnels(), { opened: 0, closed: 0, delivered: 0, open: [] }); - const { client } = await connectClient(rig.capability); - const notified = new Promise((resolve) => client.setNotificationHandler(ToolListChangedNotificationSchema, () => resolve())); - try { - await until("the client never opened its GET tunnel", () => (tunnels()?.open.length ?? 0) > 0); - const open = tunnels(); - assert.equal(open?.opened, 1); assert.equal(open?.closed, 0); - assert.ok((open?.open[0]?.openMs ?? -1) >= 0, "an open tunnel reports how long it has been open"); - assert.equal(open?.open[0]?.delivered, false, "a tunnel the mount has not pushed through is held open in silence"); - assert.equal(open?.delivered, 0); - - // The same tunnel, now actually carrying a server frame. "Held open - // having delivered nothing" and "in use" are different facts about it. - rig.server.sendToolListChanged(); - await withDeadline(notified, 4_000, "no server notification reached the client"); - await until("the delivered frame was never attributed to the tunnel", () => (tunnels()?.delivered ?? 0) === 1); - assert.equal(tunnels()?.open[0]?.delivered, true); - assert.equal(tunnels()?.closed, 0, "a tunnel that delivered is still open"); - } finally { - await client.close(); - } - await until("the closed GET tunnel still reads as open", () => (tunnels()?.closed ?? 0) === 1); - assert.deepEqual(tunnels(), { opened: 1, closed: 1, delivered: 1, open: [] }); - } finally { - await rig.close(); - } -}); - test("the facade carries the session-closing DELETE, and the mount then refuses the stale session", async () => { const rig = await startRig(); try { diff --git a/src/runtime/engineBrokerMcpFacadeRig.test.ts b/src/runtime/engineBrokerMcpFacadeRig.test.ts new file mode 100644 index 0000000..fd336a4 --- /dev/null +++ b/src/runtime/engineBrokerMcpFacadeRig.test.ts @@ -0,0 +1,164 @@ +import { createServer, type Server as HttpServer, type IncomingMessage } from "node:http"; + +import { randomUUID } from "node:crypto"; +import test from "node:test"; + +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; +import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; +import { Type } from "@earendil-works/pi-ai"; +import { defineTool } from "@earendil-works/pi-coding-agent"; + +import { createPiToolMcpServer } from "../mcp/toolServer.js"; +import { ENGINE_BROKER_MCP_FACADE_PORT, startEngineBrokerMcpFacade } from "./engineBrokerMcpFacade.js"; + +/** + * The facade's shared test rig, and not a suite of its own. + * + * Two suites drive the same boundary — the facade's routing and header + * contract, and the observations it records while relaying — and both need one + * facade on the protocol's fixed port plus a real Daimon MCP mount behind a + * real Streamable HTTP transport. Splitting them into one file each kept both + * readable; duplicating the rig into each would have left two copies of the + * thing every assertion depends on. It carries a `.test.ts` name so it never + * reaches production `dist` (`tsconfig.build.json` excludes exactly that), and + * running it on its own asserts nothing, which is what it is. + * + * Each suite is its own process, so each holds its own shared facade and each + * takes the fixed port for the length of its file. + */ +export const FACADE_URL = `http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}/mcp`; +export type Facade = Awaited>; + +/** + * One facade serves every turn of a broker, so the tests share one too. It + * also keeps the fixed port free: a facade per test would leave the HTTP + * client pooling a socket onto a server that no longer exists. + */ +let shared: Facade | undefined; +export const sharedFacade = async (): Promise => (shared ??= await startFacade()); +export const releaseShared = async (): Promise => { const facade = shared; shared = undefined; if (facade) await facade.close(); }; +test.after(releaseShared); + +/** + * Closing a facade destroys its sockets, and the port is fixed, so the HTTP + * client can still hold a pooled connection to the server that just went away. + * That is a test-harness artifact — one facade outlives a whole broker — so a + * fresh facade is probed until a refusal proves the route is live again. + */ +/** + * The facade's port is the control protocol's own, so the two suites that + * drive it cannot each hold one at the same time. Whichever binds first runs; + * the other waits for it to release the port rather than failing on the + * collision, which is the whole cost of splitting this boundary in two. + */ +const bindFacade = async (): Promise => { + for (let attempt = 0; attempt < 240; attempt += 1) { + try { return await startEngineBrokerMcpFacade(); } + catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EADDRINUSE") throw error; + await new Promise((resolve) => setTimeout(resolve, 250)); + } + } + throw new Error("the MCP facade port never came free"); +}; + +export const startFacade = async (): Promise => { + const facade = await bindFacade(); + for (let attempt = 0; attempt < 20; attempt += 1) { + try { + const probe = await fetch(FACADE_URL, { method: "PUT" }); + await probe.body?.cancel(); + if (probe.status === 403) return facade; + } catch { /* a pooled socket onto the previous facade: try the next one */ } + } + throw new Error("facade did not answer after starting"); +}; + +/** + * The brokered worker's real route: a real Daimon MCP mount behind a real + * Streamable HTTP transport, reached by a real MCP client through the facade. + * Asserting that a header is copied would pass while the route stayed broken, + * so every case below drives the transport end to end. + */ +export type Rig = Readonly<{ + facade: Facade; + turnId: string; + server: ReturnType; + capability: string; + observed: IncomingMessage[]; + /** Resolves when the mount's standalone GET stream is torn down. */ + getStreamClosed: Promise; + close: () => Promise; +}>; + +export const echoTool = defineTool({ + name: "moltnet_read", + label: "Read a scoped Moltnet surface", + description: "Reads the fixture room.", + parameters: Type.Object({ target: Type.String() }, { additionalProperties: false }), + async execute(_toolCallId: string, params: { target: string }) { + return { content: [{ type: "text" as const, text: `read ${params.target}` }], details: { target: params.target } }; + } +}); + +let turns = 0; + +export const startRig = async (facade?: Facade): Promise => { + const host = facade ?? await sharedFacade(); + const turnId = `turn-${++turns}`; + const server = createPiToolMcpServer([echoTool], {}); + const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: () => randomUUID() }); + await server.connect(transport); + const observed: IncomingMessage[] = []; + let noteGetStreamClosed = (): void => undefined; + const getStreamClosed = new Promise((resolve) => { noteGetStreamClosed = resolve; }); + const mount = createServer((request, response) => { + observed.push(request); + if (request.method === "GET") response.on("close", () => noteGetStreamClosed()); + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.on("end", () => { + const raw = Buffer.concat(chunks); + let parsed: unknown; + try { parsed = raw.length === 0 ? undefined : JSON.parse(raw.toString("utf8")); } catch { parsed = undefined; } + void transport.handleRequest(request, response, parsed); + }); + }); + await new Promise((resolve) => mount.listen(0, "127.0.0.1", resolve)); + const address = mount.address(); + if (address === null || typeof address === "string") throw new Error("mount address unavailable"); + const capability = host.register("alpha", turnId, `http://127.0.0.1:${address.port}/mcp`); + return { + facade: host, turnId, server, capability, observed, getStreamClosed, + close: async () => { + host.revoke(turnId); + await closeMount(mount, transport, server); + } + }; +}; + +export const closeMount = async (mount: HttpServer, transport: StreamableHTTPServerTransport, server: ReturnType): Promise => { + mount.closeAllConnections(); + await new Promise((resolve) => mount.close(() => resolve())); + await transport.close().catch(() => undefined); + await server.close().catch(() => undefined); +}; + +export const connectClient = async (capability: string): Promise<{ client: Client; transport: StreamableHTTPClientTransport }> => { + const transport = new StreamableHTTPClientTransport(new URL(FACADE_URL), { + requestInit: { headers: { authorization: `Bearer ${capability}` } } + }); + const client = new Client({ name: "daimon-facade-test-client", version: "0.1.0" }); + await client.connect(transport); + return { client, transport }; +}; + +export const withDeadline = async (work: Promise, ms: number, reason: string): Promise => { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([work, new Promise((_resolve, reject) => { timer = setTimeout(() => reject(new Error(reason)), ms); })]); + } finally { + if (timer) clearTimeout(timer); + } +}; diff --git a/src/runtime/engineBrokerMcpObservation.test.ts b/src/runtime/engineBrokerMcpObservation.test.ts new file mode 100644 index 0000000..b70a6f2 --- /dev/null +++ b/src/runtime/engineBrokerMcpObservation.test.ts @@ -0,0 +1,155 @@ +import assert from "node:assert/strict"; +import { createServer, type ServerResponse } from "node:http"; + +import test from "node:test"; + +import { ToolListChangedNotificationSchema } from "@modelcontextprotocol/sdk/types.js"; + +import type { EngineBrokerMcpTunnelObservation } from "./engineBrokerMcpCallLog.js"; +import { connectClient, FACADE_URL, sharedFacade, startRig, withDeadline } from "./engineBrokerMcpFacadeRig.test.js"; + +/** + * What the facade saw while it was relaying, which is the only place a call or + * a stream is observable *while it is still running*. + * + * Daimon writes a tool receipt on completion and nothing at all for the + * session's GET tunnel, so a call that never returned, a worker parked on an + * open stream, and a worker doing nothing were one indistinguishable silence. + * Both instruments are driven here through the real facade. + */ +/** + * Daimon writes a tool receipt only on completion, so a call that started and + * never returned reads exactly like a call that was never made — the one path + * a seven-minute live hang left unlit. The facade is where that difference is + * visible, and it has to survive the tear-down that ends the turn: a tunnel + * destroyed when the worker dies must not mark the call it was blocked on as + * answered, or the instrument erases the very evidence it exists to keep. + */ +test("MCP facade reports a tool call that started and never returned as outstanding, by name", async () => { + const facade = await sharedFacade(); + const held: ServerResponse[] = []; + // Two ways for a mount not to answer: never reply at all (`moltnet_read`), + // or open the stream and never deliver the result (`memory_recall`). + const target = createServer((request, response) => { + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.on("end", () => { + const asked = Buffer.concat(chunks).toString("utf8"); + if (asked.includes("moltnet_read")) { held.push(response); return; } + if (asked.includes("memory_recall")) { response.writeHead(200, { "content-type": "text/event-stream" }); response.write(": open\n\n"); held.push(response); return; } + response.writeHead(200, { "content-type": "application/json" }); response.end('{"jsonrpc":"2.0","id":1,"result":{}}'); + }); + }); + await new Promise((resolve) => target.listen(0, "127.0.0.1", resolve)); + const address = target.address(); if (address === null || typeof address === "string") throw new Error(); + const turnId = "turn-outstanding", token = facade.register("agent", turnId, `http://127.0.0.1:${address.port}/mcp`); + const post = (name: string, signal?: AbortSignal): Promise => fetch(FACADE_URL, { method: "POST", signal, headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name, arguments: { text: "argument-bytes-that-must-never-be-recorded" } } }) }); + const pending = new AbortController(); + /** + * A live call's elapsed time grows with every read, so two identical reads + * mean every relay has settled — the only moment at which "answered" is + * final. Polling for a name instead would read the log mid-teardown. + */ + const settled = async (): Promise> => { + for (let attempt = 0; attempt < 100; attempt += 1) { + const before = JSON.stringify(facade.observe(turnId)); + await new Promise((resolve) => setTimeout(resolve, 60)); + if (JSON.stringify(facade.observe(turnId)) === before) return facade.observe(turnId); + } + throw new Error("the facade's observation never settled"); + }; + const names = (observed: ReturnType): readonly string[] => (observed?.outstanding ?? []).map((call) => call.name); + try { + const answered = await post("daimon__moltnet_send"); + assert.equal(answered.status, 200); await answered.text(); + const hanging = post("daimon__moltnet_read", pending.signal).catch(() => undefined); + for (let attempt = 0; attempt < 200 && held.length === 0; attempt += 1) await new Promise((resolve) => setTimeout(resolve, 25)); + assert.equal(held.length, 1, "the mount never received the hung tool call"); + + const observed = facade.observe(turnId); + assert.deepEqual(names(observed), ["daimon__moltnet_read"], "the unanswered call must be outstanding, by name"); + assert.equal(observed?.started, 2); assert.equal(observed?.answered, 1, "the completed call must not be outstanding"); assert.equal(observed?.undecoded, 0); + assert.ok((observed?.outstanding[0]?.outstandingMs ?? -1) >= 0, "an outstanding call reports how long it has been waiting"); + assert.ok(!JSON.stringify(observed).includes("argument-bytes"), "names and timings only: no arguments"); + + // The turn's own death tears the tunnel down. The call was still never + // answered, and must still say so. + pending.abort(); await hanging; + const afterTeardown = await settled(); + assert.deepEqual(names(afterTeardown), ["daimon__moltnet_read"], "a torn-down relay is not an answer"); + assert.equal(afterTeardown?.answered, 1); + + // A stream the facade opened and never finished relaying is not an answer + // either. Awaiting the headers and one chunk puts the facade inside its own + // streaming relay before the client walks away, which is the branch that + // decides whether a half-written tunnel counts as an answer. + const halted = new AbortController(); + const half = await post("memory_recall", halted.signal); + assert.equal(half.status, 200); await half.body!.getReader().read(); halted.abort(); + const afterHalfRelay = await settled(); + assert.deepEqual(names(afterHalfRelay), ["daimon__moltnet_read", "memory_recall"], "a half-relayed stream is not an answer"); + assert.equal(afterHalfRelay?.answered, 1); + + facade.revoke(turnId); + assert.equal(facade.observe(turnId), undefined, "a turn the facade never registered observes as absence, not as zero"); + } finally { + pending.abort(); facade.revoke(turnId); + for (const response of held) response.destroy(); + target.closeAllConnections(); + await new Promise((resolve) => target.close(() => resolve())); + } +}); + +/** + * The channel the call log could not see. + * + * A brokered turn's tool calls all answer and its provider requests all close, + * and the worker can still sit idle to the deadline — parked on the standalone + * GET SSE tunnel, which stays open for the whole session and, until now, wrote + * nothing anywhere. This drives the real transport: the tunnel the real client + * opens must observe as open with an age while it is open, as *delivered* once + * the mount pushes a frame through it, and as closed once the client ends it. + * + * Mutation: remove `calls.openTunnel` from the facade's route and the first + * assertion goes red (an open tunnel reads as a turn that opened none); remove + * `tunnel?.close()` from the relay's `finally` and the last one does (a closed + * tunnel reads as still open, which is the reading the whole instrument is + * meant to make trustworthy). + */ +test("the facade observes the standalone GET tunnel: open with its age, whether it delivered, and closed when it ends", async () => { + const rig = await startRig(); + const tunnels = (): EngineBrokerMcpTunnelObservation | undefined => rig.facade.observe(rig.turnId)?.tunnels; + const until = async (reason: string, ready: () => boolean): Promise => { + for (let attempt = 0; attempt < 200 && !ready(); attempt += 1) await new Promise((resolve) => setTimeout(resolve, 25)); + assert.ok(ready(), reason); + }; + try { + // Before any request the turn is registered and has relayed nothing: a + // measured zero, which is not the same statement as an open tunnel. + assert.deepEqual(tunnels(), { opened: 0, closed: 0, delivered: 0, open: [] }); + const { client } = await connectClient(rig.capability); + const notified = new Promise((resolve) => client.setNotificationHandler(ToolListChangedNotificationSchema, () => resolve())); + try { + await until("the client never opened its GET tunnel", () => (tunnels()?.open.length ?? 0) > 0); + const open = tunnels(); + assert.equal(open?.opened, 1); assert.equal(open?.closed, 0); + assert.ok((open?.open[0]?.openMs ?? -1) >= 0, "an open tunnel reports how long it has been open"); + assert.equal(open?.open[0]?.delivered, false, "a tunnel the mount has not pushed through is held open in silence"); + assert.equal(open?.delivered, 0); + + // The same tunnel, now actually carrying a server frame. "Held open + // having delivered nothing" and "in use" are different facts about it. + rig.server.sendToolListChanged(); + await withDeadline(notified, 4_000, "no server notification reached the client"); + await until("the delivered frame was never attributed to the tunnel", () => (tunnels()?.delivered ?? 0) === 1); + assert.equal(tunnels()?.open[0]?.delivered, true); + assert.equal(tunnels()?.closed, 0, "a tunnel that delivered is still open"); + } finally { + await client.close(); + } + await until("the closed GET tunnel still reads as open", () => (tunnels()?.closed ?? 0) === 1); + assert.deepEqual(tunnels(), { opened: 1, closed: 1, delivered: 1, open: [] }); + } finally { + await rig.close(); + } +}); From 4f4922a1212298af2743dd0ba1ae2bf88b46a112 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 15:00:25 +0200 Subject: [PATCH 60/69] feat: count and seal the MCP facade's refused requests by reason class --- src/runtime/AGENTS.md | 27 +++++++- src/runtime/engineBrokerCapabilities.ts | 22 ++++++ src/runtime/engineBrokerControlClient.ts | 6 +- src/runtime/engineBrokerMcpCallLog.test.ts | 50 +++++++++++++- src/runtime/engineBrokerMcpCallLog.ts | 42 +++++++++++- src/runtime/engineBrokerMcpFacade.ts | 62 ++++++++++++++--- .../engineBrokerMcpObservation.test.ts | 68 +++++++++++++++++++ src/runtime/engineBrokerProtocol.test.ts | 19 ++++++ src/runtime/engineBrokerProtocol.ts | 23 ++++++- src/runtime/engineBrokerSealLedger.test.ts | 27 ++++++++ src/runtime/engineBrokerSealLedger.ts | 9 ++- 11 files changed, 331 insertions(+), 24 deletions(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index db88d50..20b4e62 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -332,6 +332,25 @@ out or refuses a tunnel, because an instrument that tore the stream down would destroy the evidence it exists to gather. The member is optional on the wire for one reason — a turn sealed before it existed must still replay — so its absence means "not measured" and never zero, exactly as `mcp`'s own absence does. +A request the facade *refuses* is the sharpest form of the same silence, and +it used to observe as nothing at all: `route()` threw before `calls.begin`, so +a turn 403'd on every request sealed `answered == started, outstanding: []` — +byte-identical to a healthy turn. `EngineBrokerMcpCallLog.refuse` now counts +each one by a closed reason class (`route`, `expired`, `exhausted`, +`unrouted`, `oversized`), because the classes call for opposite fixes: an +exhausted per-turn capability is a budget, an unserved route is a worker +asking for something that does not exist. The budget is reachable rather than +theoretical — `ENGINE_BROKER_MCP_CAPABILITY_REQUESTS` (128) covers every POST, +the GET tunnel and the DELETE, and a `search_tool`+`use_tool` round spends two, +so a 48-round wake asks for ~96 plus its handshake. Attribution comes from +`EngineBrokerCapabilities.classifyToken`, which names the token's turn and why +it would be refused *without spending its budget*; a bearer no grant matches +names no turn and stays unattributed, because guessing an owner would be +inventing the measurement. Counts only: never the token, the capability, the +URL or the body. The member is optional on the wire for `tunnels`' one reason, +and reaches the operator as `mcp_refused=exhausted:41` and the seal row's +`mcp.refusals`. + Measured against the real CLI (rig, grok 1.0.34, real facade and mount): the tunnel opens ~3 ms after `initialize`, carries nothing for its whole life, and **closes 16 ms before the worker exits** — the close is the worker's own @@ -353,8 +372,12 @@ terminal turn (`turns.jsonl`, `engineBrokerSealLedgerPathFor`), rendered from the sealed response and nothing else. Its members are the accounting, the failure `code`, the diagnostic's closed `status`/`stage`/`failure_class` with the reason `engineBrokerNativeClient.ts` already redacted and bounded, and the -facade's `mcp` observation — names, counts, GET-tunnel lifecycle and elapsed -milliseconds. Never a +facade's `mcp` observation — names, counts, refusals by reason class, +GET-tunnel lifecycle and elapsed milliseconds. That projection is a closed +allow-list and `engineBrokerSealLedger.test.ts` asserts the *exact key set* of +a written row for a completed turn: replacing it with `...terminal` writes +`usage`, `diagnostic`, `mcpCalls` and the model's entire reply into the +ledger, and that mutation is what the assertion exists to catch. Never a prompt, body, reply, bearer, capability or session id; the terminal response carries none of those in the first place, and the projection is an allow-list rather than a spread, so a future additive member of the response cannot become diff --git a/src/runtime/engineBrokerCapabilities.ts b/src/runtime/engineBrokerCapabilities.ts index 0bbd102..1f99f8a 100644 --- a/src/runtime/engineBrokerCapabilities.ts +++ b/src/runtime/engineBrokerCapabilities.ts @@ -23,6 +23,28 @@ export class EngineBrokerCapabilities { } return undefined; } + /** + * Which grant a token names and why it would be refused, *without* spending + * it. + * + * The facade needs this on its refusal path alone. A 403 it cannot attribute + * to a turn is a 403 that seals as nothing at all, and a turn whose every + * request was refused then reads exactly like a healthy one — the silence + * `engineBrokerMcpCallLog.ts` exists to end. A token no grant matches names + * no turn and stays unattributed; nothing here returns the token, the grant + * or the agent's capability, only the turn id and a closed reason. + */ + classifyToken(token: string): Readonly<{ turnId: string; state: "live" | "expired" | "exhausted" }> | undefined { + const candidate = hash(token); + for (const grant of this.grants.values()) { + if (!timingSafeEqual(grant.digest, candidate)) continue; + // Budget before expiry: the TTL outlives every declared turn limit, so an + // exhausted grant is the reachable refusal and the actionable answer. + if (grant.requests >= grant.maxRequests) return { turnId: grant.turnId, state: "exhausted" }; + return { turnId: grant.turnId, state: grant.expiresAt <= Date.now() ? "expired" : "live" }; + } + return undefined; + } inspectToken(token:string):Readonly<{agentId:string;turnId:string}>|undefined{const candidate=hash(token);for(const grant of this.grants.values()){if(grant.expiresAt>Date.now()&&grant.requestscalls===undefined?"":`; mcp=${calls.answered}/${calls.started} answered${calls.undecoded===0?"":`; mcp_undecoded=${calls.undecoded}`}${calls.outstanding.length===0?"":`; mcp_outstanding=${calls.outstanding.map((call)=>`${call.name}@${call.outstandingMs}ms`).join(",")}`}${renderMcpTunnels(calls.tunnels)}`; +const renderMcpCalls=(calls:EngineBrokerMcpCallObservation|undefined):string=>calls===undefined?"":`; mcp=${calls.answered}/${calls.started} answered${calls.undecoded===0?"":`; mcp_undecoded=${calls.undecoded}`}${calls.outstanding.length===0?"":`; mcp_outstanding=${calls.outstanding.map((call)=>`${call.name}@${call.outstandingMs}ms`).join(",")}`}${renderMcpRefusals(calls.refusals)}${renderMcpTunnels(calls.tunnels)}`; +/** The refusals the facade never relayed, by reason class, and only the classes that happened: a turn refused 403 must not read as a turn that was served. */ +const renderMcpRefusals=(refusals:EngineBrokerMcpRefusalObservation|undefined):string=>{if(refusals===undefined)return"";const named=ENGINE_BROKER_MCP_REFUSAL_REASONS.filter((reason)=>refusals[reason]>0);return named.length===0?"":`; mcp_refused=${named.map((reason)=>`${reason}:${refusals[reason]}`).join(",")}`;}; /** The session's GET SSE tunnels: how many closed of how many opened, and each one still open with its age and whether the mount ever pushed through it. */ const renderMcpTunnels=(tunnels:EngineBrokerMcpTunnelObservation|undefined):string=>tunnels===undefined?"":`; mcp_get=${tunnels.closed}/${tunnels.opened} closed, ${tunnels.delivered} delivered${tunnels.open.length===0?"":`; mcp_get_open=${tunnels.open.map((tunnel)=>`${tunnel.openMs}ms/${tunnel.delivered?"delivered":"silent"}`).join(",")}`}`; diff --git a/src/runtime/engineBrokerMcpCallLog.test.ts b/src/runtime/engineBrokerMcpCallLog.test.ts index 7b0ab26..dbf51ac 100644 --- a/src/runtime/engineBrokerMcpCallLog.test.ts +++ b/src/runtime/engineBrokerMcpCallLog.test.ts @@ -9,6 +9,8 @@ const call = (name: unknown, id = 1): Buffer => body({ jsonrpc: "2.0", id, metho /** A controllable clock: an outstanding call's whole value is how long it has been outstanding. */ /** An observed turn whose facade never relayed a GET tunnel — a measurement, not an absence. */ const NO_TUNNEL = { opened: 0, closed: 0, delivered: 0, open: [] }; +/** An observed turn the facade never refused — also a measurement, and not the same statement as a turn it never observed. */ +const NO_REFUSALS = { route: 0, expired: 0, exhausted: 0, unrouted: 0, oversized: 0 }; const clock = (): { log: EngineBrokerMcpCallLog; advance: (ms: number) => void } => { let at = 1_000; @@ -36,12 +38,12 @@ test("absence stays absence: an unopened turn observes undefined, a turn that ca const { log } = clock(); assert.equal(log.observe("turn"), undefined); log.open("turn"); - assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 0, outstanding: [], tunnels: NO_TUNNEL }); + assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 0, outstanding: [], refusals: NO_REFUSALS, tunnels: NO_TUNNEL }); // Everything that is not a tool call records nothing at all, so `started` // stays a count of tool calls and not of traffic. for (const value of [body({ jsonrpc: "2.0", id: 1, method: "tools/list" }), body({ jsonrpc: "2.0", method: "notifications/initialized" }), Buffer.alloc(0)]) log.begin("turn", value).answer(); log.begin("turn", undefined).answer(); - assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 0, outstanding: [], tunnels: NO_TUNNEL }); + assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 0, outstanding: [], refusals: NO_REFUSALS, tunnels: NO_TUNNEL }); log.close("turn"); assert.equal(log.observe("turn"), undefined, "a revoked turn keeps nothing"); }); @@ -51,7 +53,7 @@ test("a body the facade could not read counts as undecoded, never as a call with log.open("turn"); log.begin("turn", Buffer.from("{not json", "utf8")); log.undecodable("turn"); - assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 2, outstanding: [], tunnels: NO_TUNNEL }); + assert.deepEqual(log.observe("turn"), { started: 0, answered: 0, undecoded: 2, outstanding: [], refusals: NO_REFUSALS, tunnels: NO_TUNNEL }); log.undecodable("absent"); }); @@ -144,3 +146,45 @@ test("the open-tunnel list is bounded, and the counts still name every tunnel be assert.equal(tunnels?.open.length, ENGINE_BROKER_MCP_TUNNEL_MAX); assert.equal(tunnels?.open[0]?.openMs, ENGINE_BROKER_MCP_TUNNEL_MAX + 3, "the earliest tunnels are the ones kept"); }); + +/** + * The refusal is the sharpest form of the silence this log exists to end. + * + * A refused request never reaches the relay, so a turn every one of whose + * requests was 403'd observed as `started: 0, answered: 0, outstanding: []` — + * the same three numbers a turn that simply had nothing to call observes. The + * boundary these assertions straddle is that one: a turn refused against a + * turn served, and an exhausted capability budget against a route the facade + * does not serve, because the two call for opposite fixes. + * + * Mutation: drop the `refusals` member from `observe`, or make `refuse` a + * no-op, and a refused turn reads as an idle one again. + */ +test("a refused request is counted by reason class, and refusing is not calling", () => { + const { log } = clock(); + log.open("turn"); + assert.deepEqual(log.observe("turn")?.refusals, NO_REFUSALS, "an observed turn that was never refused measured zero"); + log.refuse("turn", "exhausted"); log.refuse("turn", "exhausted"); log.refuse("turn", "route"); + const observed = log.observe("turn"); + assert.deepEqual(observed?.refusals, { ...NO_REFUSALS, exhausted: 2, route: 1 }); + // The reading the instrument has to keep honest: a refused turn is not a + // turn that called nothing and answered everything. + assert.deepEqual([observed?.started, observed?.answered, observed?.outstanding], [0, 0, []]); + // Every reason class is its own count, and the observation is a copy: a + // later refusal cannot rewrite a report already handed out. + log.refuse("turn", "unrouted"); + assert.deepEqual(observed?.refusals, { ...NO_REFUSALS, exhausted: 2, route: 1 }); + assert.deepEqual(log.observe("turn")?.refusals, { ...NO_REFUSALS, exhausted: 2, route: 1, unrouted: 1 }); +}); + +test("a refusal the facade cannot attribute is recorded against no turn at all", () => { + const { log } = clock(); + log.refuse("absent", "expired"); + assert.equal(log.observe("absent"), undefined, "an unopened turn keeps nothing"); + log.open("absent"); + assert.deepEqual(log.observe("absent")?.refusals, NO_REFUSALS, "a refusal before the turn existed is not this turn's"); + log.refuse("absent", "oversized"); + log.close("absent"); + log.open("absent"); + assert.deepEqual(log.observe("absent")?.refusals, NO_REFUSALS, "re-opening an id resets its refusals with everything else"); +}); diff --git a/src/runtime/engineBrokerMcpCallLog.ts b/src/runtime/engineBrokerMcpCallLog.ts index b680c48..6654102 100644 --- a/src/runtime/engineBrokerMcpCallLog.ts +++ b/src/runtime/engineBrokerMcpCallLog.ts @@ -55,6 +55,21 @@ * the turn's death must not retroactively mark the call it was blocked on as * finished. */ +/** + * The same map carries the facade's *refusals*, for the sharpest form of the + * same problem. A refused request never reaches the relay at all, so a turn + * every one of whose requests was 403'd sealed as `answered == started, + * outstanding: []` — byte-identical to a healthy turn, which is precisely the + * reading this instrument exists to make trustworthy. The reason class is what + * makes it actionable: an exhausted per-turn capability budget (a worker's + * `search_tool`+`use_tool` pair per round is two requests of the facade's + * `ENGINE_BROKER_MCP_CAPABILITY_REQUESTS`) is a different fault from a + * mount that was never registered, and both differ from a worker asking for a + * route the facade does not serve. Counts only, keyed by a closed vocabulary: + * never the token, the capability, the URL or the body. A refusal the facade + * cannot attribute to a turn — a bearer no live grant matches — is recorded + * nowhere, because attributing it to a turn would be inventing the fact. + */ export const ENGINE_BROKER_MCP_OUTSTANDING_MAX = 16; export const ENGINE_BROKER_MCP_CALL_INVALID = ""; export const ENGINE_BROKER_MCP_CALL_TRUNCATED = ""; @@ -65,6 +80,19 @@ const TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/u; /** Open GET tunnels reported: a session opens one, so more than a handful is already the anomaly. */ export const ENGINE_BROKER_MCP_TUNNEL_MAX = 8; +/** + * Why the facade refused one relayed request, as a closed vocabulary: + * - `route`: a path or method the facade does not serve. + * - `expired`: the turn capability's TTL had passed. + * - `exhausted`: the turn capability's request budget was spent. + * - `unrouted`: a live capability whose turn has no registered mount. + * - `oversized`: a POST body past the facade's own request bound. + */ +export const ENGINE_BROKER_MCP_REFUSAL_REASONS = ["route", "expired", "exhausted", "unrouted", "oversized"] as const; +export type EngineBrokerMcpRefusalReason = (typeof ENGINE_BROKER_MCP_REFUSAL_REASONS)[number]; +/** How many requests of this turn the facade refused, by reason class. Every member is a measurement; the whole member is absent only where it was never measured. */ +export type EngineBrokerMcpRefusalObservation = Readonly>; + export type EngineBrokerOutstandingMcpCall = Readonly<{ name: string; outstandingMs: number }>; /** One GET SSE tunnel still open at observation: how long it has been open, and whether the mount ever pushed through it. */ export type EngineBrokerOpenMcpTunnel = Readonly<{ openMs: number; delivered: boolean }>; @@ -79,7 +107,7 @@ export type EngineBrokerMcpTunnelObservation = Readonly<{ opened: number; closed * facade answered, how many POST bodies it could not read, and the ones still * unanswered with the time each has been outstanding. */ -export type EngineBrokerMcpCallObservation = Readonly<{ started: number; answered: number; undecoded: number; outstanding: readonly EngineBrokerOutstandingMcpCall[]; tunnels?: EngineBrokerMcpTunnelObservation }>; +export type EngineBrokerMcpCallObservation = Readonly<{ started: number; answered: number; undecoded: number; outstanding: readonly EngineBrokerOutstandingMcpCall[]; tunnels?: EngineBrokerMcpTunnelObservation; refusals?: EngineBrokerMcpRefusalObservation }>; /** One relayed POST's calls. `answer` marks a complete relay; `close` ends it unanswered. Both are idempotent. */ export interface EngineBrokerMcpCallHandle { answer(): void; close(): void } @@ -93,14 +121,16 @@ type TunnelRecord = { readonly openedAt: number; delivered: boolean }; type TurnLog = { started: number; answered: number; undecoded: number; readonly live: Set; readonly ended: CallRecord[]; tunnelsOpened: number; tunnelsClosed: number; tunnelsDelivered: number; readonly openTunnels: Set; + readonly refusals: Record; }; +const noRefusals = (): Record => ({ route: 0, expired: 0, exhausted: 0, unrouted: 0, oversized: 0 }); export class EngineBrokerMcpCallLog { private readonly turns = new Map(); constructor(private readonly now: () => number = Date.now) {} /** A turn the facade registered. Re-opening an id resets it: a turn id is unique per turn. */ - open(turnId: string): void { this.turns.set(turnId, { started: 0, answered: 0, undecoded: 0, live: new Set(), ended: [], tunnelsOpened: 0, tunnelsClosed: 0, tunnelsDelivered: 0, openTunnels: new Set() }); } + open(turnId: string): void { this.turns.set(turnId, { started: 0, answered: 0, undecoded: 0, live: new Set(), ended: [], tunnelsOpened: 0, tunnelsClosed: 0, tunnelsDelivered: 0, openTunnels: new Set(), refusals: noRefusals() }); } close(turnId: string): void { this.turns.delete(turnId); } /** A POST body the facade is about to relay. Anything that is not a `tools/call` records nothing. */ @@ -152,6 +182,13 @@ export class EngineBrokerMcpCallLog { }; } + /** + * A request the facade refused before it could ever be relayed. A refusal it + * cannot attribute to a turn is never recorded against one, so an unknown + * turn is a no-op here exactly as every other operation is. + */ + refuse(turnId: string, reason: EngineBrokerMcpRefusalReason): void { const log = this.turns.get(turnId); if (log !== undefined) log.refusals[reason] += 1; } + /** A POST whose body the facade refused to read (over its own bound), which is a call it cannot name. */ undecodable(turnId: string): void { const log = this.turns.get(turnId); if (log !== undefined) log.undecoded += 1; } @@ -167,6 +204,7 @@ export class EngineBrokerMcpCallLog { .map((record): EngineBrokerOpenMcpTunnel => ({ openMs: Math.max(0, at - record.openedAt), delivered: record.delivered })); return { started: log.started, answered: log.answered, undecoded: log.undecoded, + refusals: { ...log.refusals }, tunnels: { opened: log.tunnelsOpened, closed: log.tunnelsClosed, delivered: log.tunnelsDelivered, open }, outstanding: outstanding.length > ENGINE_BROKER_MCP_OUTSTANDING_MAX ? [...outstanding.slice(0, ENGINE_BROKER_MCP_OUTSTANDING_MAX - 1), { name: ENGINE_BROKER_MCP_CALL_TRUNCATED, outstandingMs: 0 }] diff --git a/src/runtime/engineBrokerMcpFacade.ts b/src/runtime/engineBrokerMcpFacade.ts index 51e4e0e..44eccfe 100644 --- a/src/runtime/engineBrokerMcpFacade.ts +++ b/src/runtime/engineBrokerMcpFacade.ts @@ -1,7 +1,7 @@ import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import { Readable } from "node:stream"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; -import { EngineBrokerMcpCallLog, type EngineBrokerMcpCallObservation, type EngineBrokerMcpTunnelHandle } from "./engineBrokerMcpCallLog.js"; +import { EngineBrokerMcpCallLog, type EngineBrokerMcpCallObservation, type EngineBrokerMcpRefusalReason, type EngineBrokerMcpTunnelHandle } from "./engineBrokerMcpCallLog.js"; /** * The brokered worker's only route to its own per-wake Daimon MCP mount. The @@ -46,6 +46,18 @@ const FORWARDED_RESPONSE_HEADERS = ["content-type", "mcp-session-id", "mcp-proto const FORWARDED_METHODS = new Set(["POST", "GET", "DELETE"]); const MAX_REQUEST_BYTES = 1024 * 1024; export const ENGINE_BROKER_MCP_FACADE_PORT = 43_124; +/** + * Requests one turn capability may spend, across all three methods. + * + * A worker's round is a `search_tool` and a `use_tool`, so a 48-turn wake is + * ~96 POSTs plus the handshake, the standalone GET tunnel and the closing + * DELETE: exhaustion is reachable rather than theoretical, and every request + * past it is a 403 the worker cannot explain. That is why the refusal is + * counted and sealed (`engineBrokerMcpCallLog.ts`) rather than being an + * absence in the turn's row. + */ +export const ENGINE_BROKER_MCP_CAPABILITY_REQUESTS = 128; +export const ENGINE_BROKER_MCP_CAPABILITY_TTL_MS = 15 * 60_000; class FacadeRefusal extends Error {} @@ -71,15 +83,37 @@ export async function startEngineBrokerMcpFacade() { }); }); + /** + * A 403 the call log can read, whenever the bearer names a turn. + * + * The refusal itself is unchanged — same status, same body, same silence + * towards the worker — but it is attributed first, through a lookup that + * does not spend the capability's budget. A bearer no grant matches names no + * turn and stays unattributed, because guessing whose it was would be + * inventing the measurement. A capability that is *also* spent or expired + * reports that instead of the route it asked for: it is the fault the + * operator can act on. + */ + function refuse(bearer: string | undefined, live: EngineBrokerMcpRefusalReason): FacadeRefusal { + const classified = bearer === undefined ? undefined : capabilities.classifyToken(bearer); + if (classified !== undefined) calls.refuse(classified.turnId, classified.state === "live" ? live : classified.state); + return new FacadeRefusal(); + } + async function route(request: IncomingMessage, response: ServerResponse): Promise { const method = request.method ?? ""; - if (request.url !== "/mcp" || !FORWARDED_METHODS.has(method)) throw new FacadeRefusal(); - const match = request.headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u); - if (!match) throw new FacadeRefusal(); - const scope = capabilities.authorizeToken(match[1]!); - if (!scope) throw new FacadeRefusal(); + const bearer = request.headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{40,})$/u)?.[1]; + if (request.url !== "/mcp" || !FORWARDED_METHODS.has(method)) throw refuse(bearer, "route"); + if (bearer === undefined) throw new FacadeRefusal(); + const scope = capabilities.authorizeToken(bearer); + // A live grant that authorizes is the only way past here; anything else is + // classified so an exhausted budget and an expired TTL reach the turn's + // sealed row as themselves. `live` cannot be the answer on this path — + // `authorizeToken` read the same grant a moment ago, expiry only moves + // forward and a budget only spends — so it is the unreachable arm. + if (!scope) throw refuse(bearer, "expired"); const target = targets.get(scope.turnId); - if (target === undefined) throw new FacadeRefusal(); + if (target === undefined) { calls.refuse(scope.turnId, "unrouted"); throw new FacadeRefusal(); } // Only POST carries a JSON-RPC body; drain anything else so the socket // never stalls waiting for a body the facade will not forward. @@ -87,7 +121,13 @@ export async function startEngineBrokerMcpFacade() { // A body refused for size is a call the log can never name, and counting // it keeps "no tool call started" an honest reading rather than a gap. try { body = method === "POST" ? await bounded(request) : (request.resume(), undefined); } - catch (error) { calls.undecodable(scope.turnId); throw error; } + catch (error) { + calls.undecodable(scope.turnId); + // A body past the bound is refused, not merely unreadable: the worker + // gets a 403 for it, so it is counted as one too. + if (error instanceof FacadeRefusal) calls.refuse(scope.turnId, "oversized"); + throw error; + } const payload = body === undefined ? undefined : (body.buffer.slice(body.byteOffset, body.byteOffset + body.byteLength) as ArrayBuffer); const call = calls.begin(scope.turnId, body); // The GET SSE tunnel is the session's other channel and the one that @@ -133,7 +173,7 @@ export async function startEngineBrokerMcpFacade() { if (targets.has(turnId)) throw new Error("MCP turn already registered"); targets.set(turnId, url.href); calls.open(turnId); - return capabilities.issue(agentId, turnId, 15 * 60_000, 128); + return capabilities.issue(agentId, turnId, ENGINE_BROKER_MCP_CAPABILITY_TTL_MS, ENGINE_BROKER_MCP_CAPABILITY_REQUESTS); }, revoke(turnId: string): void { targets.delete(turnId); @@ -142,8 +182,8 @@ export async function startEngineBrokerMcpFacade() { calls.close(turnId); }, /** - * What the facade saw of this turn's tool calls and GET tunnels, or - * `undefined` for a turn + * What the facade saw of this turn's tool calls, GET tunnels and refusals, + * or `undefined` for a turn * it never registered. Read on the failure path, before `revoke`. */ observe: (turnId: string): EngineBrokerMcpCallObservation | undefined => calls.observe(turnId), diff --git a/src/runtime/engineBrokerMcpObservation.test.ts b/src/runtime/engineBrokerMcpObservation.test.ts index b70a6f2..fc2b42d 100644 --- a/src/runtime/engineBrokerMcpObservation.test.ts +++ b/src/runtime/engineBrokerMcpObservation.test.ts @@ -6,6 +6,7 @@ import test from "node:test"; import { ToolListChangedNotificationSchema } from "@modelcontextprotocol/sdk/types.js"; import type { EngineBrokerMcpTunnelObservation } from "./engineBrokerMcpCallLog.js"; +import { ENGINE_BROKER_MCP_CAPABILITY_REQUESTS, ENGINE_BROKER_MCP_FACADE_PORT } from "./engineBrokerMcpFacade.js"; import { connectClient, FACADE_URL, sharedFacade, startRig, withDeadline } from "./engineBrokerMcpFacadeRig.test.js"; /** @@ -153,3 +154,70 @@ test("the facade observes the standalone GET tunnel: open with its age, whether await rig.close(); } }); + +/** + * The refusal, which was the one relay outcome that observed as nothing. + * + * `route()` refuses before the call log is ever touched, so a request the + * facade 403'd never reached `started`, `undecoded` or `outstanding`: a turn + * whose every request was refused sealed as `answered == started, + * outstanding: []`, which is exactly what a healthy turn seals as. The budget + * makes that reachable rather than theoretical — one capability buys + * {@link ENGINE_BROKER_MCP_CAPABILITY_REQUESTS} requests across all three + * methods, and a worker spends two of them per round. + * + * The boundary these assertions straddle: a turn served against a turn + * refused, and within the refusals, a spent capability against a route the + * facade does not serve — the first is a budget to raise, the second is a + * worker asking for something that does not exist. + * + * Mutation: restore `throw new FacadeRefusal()` in place of either `refuse` + * call in `route()`, and a 403'd turn reads as an idle one again. + */ +test("a request the facade refused is counted against its turn, by reason, and is never a call it served", async () => { + const facade = await sharedFacade(); + let served = 0; + const target = createServer((request, response) => { + served += 1; + request.resume(); + request.on("end", () => { response.writeHead(200, { "content-type": "application/json" }); response.end('{"jsonrpc":"2.0","id":1,"result":{}}'); }); + }); + await new Promise((resolve) => target.listen(0, "127.0.0.1", resolve)); + const address = target.address(); if (address === null || typeof address === "string") throw new Error(); + const turnId = "turn-refused", token = facade.register("agent", turnId, `http://127.0.0.1:${address.port}/mcp`); + const listed = JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list", params: {} }); + const send = async (url: string, bearer: string, method = "POST"): Promise => { + const response = await fetch(url, { method, headers: { authorization: `Bearer ${bearer}`, "content-type": "application/json" }, ...(method === "POST" ? { body: listed } : {}) }); + await response.text(); + return response.status; + }; + try { + // A route the facade does not serve, asked for with a live capability. + assert.equal(await send(`http://127.0.0.1:${ENGINE_BROKER_MCP_FACADE_PORT}/nope`, token), 403); + assert.equal(await send(FACADE_URL, token, "PUT"), 403); + assert.deepEqual(facade.observe(turnId)?.refusals, { route: 2, expired: 0, exhausted: 0, unrouted: 0, oversized: 0 }); + + // A bearer no live grant matches names no turn, so it is recorded against + // none: inventing an owner would be worse than the silence. + assert.equal(await send(FACADE_URL, "wrong-token-abcdefghijklmnopqrstuvwxyz0123456789"), 403); + assert.equal(facade.observe(turnId)?.refusals?.route, 2, "an unattributable refusal belongs to no turn"); + + // Neither refusal spent the capability, so the budget is exactly what was + // issued — and spending it all is reachable: a 48-round worker asks for + // ~96 of these plus its handshake, tunnel and DELETE. + for (let spent = 0; spent < ENGINE_BROKER_MCP_CAPABILITY_REQUESTS; spent += 1) assert.equal(await send(FACADE_URL, token), 200); + assert.equal(served, ENGINE_BROKER_MCP_CAPABILITY_REQUESTS); + assert.equal(await send(FACADE_URL, token), 403, "the capability's budget is spent"); + assert.equal(served, ENGINE_BROKER_MCP_CAPABILITY_REQUESTS, "an exhausted capability never reaches the mount"); + + const observed = facade.observe(turnId); + assert.deepEqual(observed?.refusals, { route: 2, expired: 0, exhausted: 1, unrouted: 0, oversized: 0 }); + // And the reading the seal row used to publish for all of it: nothing. + assert.deepEqual([observed?.started, observed?.answered, observed?.undecoded, observed?.outstanding], [0, 0, 0, []]); + assert.ok(!JSON.stringify(observed).includes(token), "counts and reason classes only: never the capability"); + } finally { + facade.revoke(turnId); + target.closeAllConnections(); + await new Promise((resolve) => target.close(() => resolve())); + } +}); diff --git a/src/runtime/engineBrokerProtocol.test.ts b/src/runtime/engineBrokerProtocol.test.ts index 835493c..8d59256 100644 --- a/src/runtime/engineBrokerProtocol.test.ts +++ b/src/runtime/engineBrokerProtocol.test.ts @@ -124,4 +124,23 @@ test("a failed frame carries the broker's in-flight MCP tool-call observation, b { ...tunnels, open: [{ openMs: 1, delivered: false, sessionId: "mcp-session-0" }] }, { opened: 1, closed: 0, open: [] } ]) assert.throws(() => parseEngineBrokerResponse({ ...value, mcpCalls: { ...value.mcpCalls, tunnels: forged } }), /invalid broker frame/u, JSON.stringify(forged)); + + /** + * The refusals ride the same member under the same rules, and they are the + * counts that make a 403'd turn readable: without them a turn the facade + * refused every request of publishes `started: 0, answered: 0` — an idle + * turn's numbers. Every reason class is its own measurement, so a partial + * member is refused rather than zero-filled, and the whole member is optional + * for the one reason `tunnels` is. + */ + const refusals = { route: 1, expired: 0, exhausted: 41, unrouted: 0, oversized: 2 } as const; + const refused = { ...value, mcpCalls: { ...value.mcpCalls, refusals } } as const; + assert.deepEqual(parseEngineBrokerResponse(refused), refused); + for (const forged of [ + { ...refusals, exhausted: -1 }, + { ...refusals, exhausted: 1.5 }, + { ...refusals, exhausted: "41" }, + { route: 1, expired: 0, unrouted: 0, oversized: 0 }, + { ...refusals, capability: 3 } + ]) assert.throws(() => parseEngineBrokerResponse({ ...value, mcpCalls: { ...value.mcpCalls, refusals: forged } }), /invalid broker frame/u, JSON.stringify(forged)); }); diff --git a/src/runtime/engineBrokerProtocol.ts b/src/runtime/engineBrokerProtocol.ts index 201d005..f5e37b4 100644 --- a/src/runtime/engineBrokerProtocol.ts +++ b/src/runtime/engineBrokerProtocol.ts @@ -1,5 +1,5 @@ import { isEngineBrokerInferenceRequestKind, isEngineBrokerInferenceResponseKind, parseEngineBrokerInferenceRequest, parseEngineBrokerInferenceResponse, type EngineBrokerInferenceRequest, type EngineBrokerInferenceResponse } from "./engineBrokerInferenceProtocol.js"; -import { ENGINE_BROKER_MCP_CALL_NAME, ENGINE_BROKER_MCP_OUTSTANDING_MAX, ENGINE_BROKER_MCP_TUNNEL_MAX, type EngineBrokerMcpCallObservation, type EngineBrokerMcpTunnelObservation } from "./engineBrokerMcpCallLog.js"; +import { ENGINE_BROKER_MCP_CALL_NAME, ENGINE_BROKER_MCP_OUTSTANDING_MAX, ENGINE_BROKER_MCP_REFUSAL_REASONS, ENGINE_BROKER_MCP_TUNNEL_MAX, type EngineBrokerMcpCallObservation, type EngineBrokerMcpRefusalObservation, type EngineBrokerMcpTunnelObservation } from "./engineBrokerMcpCallLog.js"; import { parseEngineBrokerTurnAccounting, parseEngineBrokerTurnLimitOverrides, type EngineBrokerTurnAccounting, type EngineBrokerTurnLimitOverrides } from "./engineBrokerTurnAccounting.js"; /** @@ -146,7 +146,7 @@ function parseMcpCallObservation(value: unknown): EngineBrokerMcpCallObservation // `tunnels` is optional for one reason only: a turn sealed before the GET // tunnel was observed carries no such member, and its record must still // replay. Absence there means "the instrument did not exist", never zero. - exact(input, ["started", "answered", "undecoded", "outstanding", ...(input.tunnels === undefined ? [] : ["tunnels"])]); + exact(input, ["started", "answered", "undecoded", "outstanding", ...(input.tunnels === undefined ? [] : ["tunnels"]), ...(input.refusals === undefined ? [] : ["refusals"])]); const started = input.started, answered = input.answered, undecoded = input.undecoded; if (![started, answered, undecoded].every((count) => Number.isSafeInteger(count) && (count as number) >= 0)) throw new TypeError("invalid broker frame"); if (!Array.isArray(input.outstanding) || input.outstanding.length > ENGINE_BROKER_MCP_OUTSTANDING_MAX) throw new TypeError("invalid broker frame"); @@ -159,7 +159,24 @@ function parseMcpCallObservation(value: unknown): EngineBrokerMcpCallObservation }); if ((answered as number) > (started as number) || outstanding.length > (started as number) - (answered as number)) throw new TypeError("invalid broker frame"); const tunnels = input.tunnels === undefined ? undefined : parseMcpTunnelObservation(input.tunnels); - return { started: started as number, answered: answered as number, undecoded: undecoded as number, outstanding, ...(tunnels === undefined ? {} : { tunnels }) }; + const refusals = input.refusals === undefined ? undefined : parseMcpRefusalObservation(input.refusals); + return { started: started as number, answered: answered as number, undecoded: undecoded as number, outstanding, ...(tunnels === undefined ? {} : { tunnels }), ...(refusals === undefined ? {} : { refusals }) }; +} + +/** + * The refused requests, by reason class: one count per closed reason, all of + * them required once the member is present. Optional for the same single + * reason `tunnels` is — a turn sealed before the facade counted its refusals + * must still replay — so its absence means "not measured" and never zero, and + * a partial member is refused rather than zero-filled. + */ +function parseMcpRefusalObservation(value: unknown): EngineBrokerMcpRefusalObservation { + const input = record(value); + exact(input, ENGINE_BROKER_MCP_REFUSAL_REASONS); + for (const reason of ENGINE_BROKER_MCP_REFUSAL_REASONS) { + if (!Number.isSafeInteger(input[reason]) || (input[reason] as number) < 0) throw new TypeError("invalid broker frame"); + } + return Object.fromEntries(ENGINE_BROKER_MCP_REFUSAL_REASONS.map((reason) => [reason, input[reason] as number])) as EngineBrokerMcpRefusalObservation; } /** diff --git a/src/runtime/engineBrokerSealLedger.test.ts b/src/runtime/engineBrokerSealLedger.test.ts index 60754e3..a609601 100644 --- a/src/runtime/engineBrokerSealLedger.test.ts +++ b/src/runtime/engineBrokerSealLedger.test.ts @@ -158,3 +158,30 @@ test("a cancelled turn's GET tunnel is sealed open with its age, closed, or neve const unobserved = await cancelledTurn(() => ({ started: 1, answered: 1, undecoded: 0, outstanding: [] })); assert.equal(Object.hasOwn(unobserved.seal?.mcp as Record, "tunnels"), false); }); + +/** + * The refusal, on the same durable route as the hang it looks like. + * + * A request the facade 403'd never reached the relay, so before it was counted + * a turn whose capability was spent — 128 requests, two per worker round — + * sealed `answered == started, outstanding: []`, which is exactly what a + * healthy turn seals. The row has to carry the reason class, because an + * exhausted budget and an unserved route are opposite fixes. + * + * Mutation: drop the `refusals` member from `renderBrokerTurnSealLine` and the + * first assertion goes red; render it unconditionally as zeros for an + * observation that carries none, and the second does — a zero nobody measured + * reads exactly like a zero somebody did. + */ +test("a turn whose MCP requests were refused seals the refusals by reason, and a turn sealed before they were counted seals none", async () => { + const refused = await cancelledTurn(() => ({ + started: 0, answered: 0, undecoded: 0, outstanding: [], + refusals: { route: 0, expired: 0, exhausted: 41, unrouted: 1, oversized: 0 } + })); + assert.deepEqual((refused.seal?.mcp as Record).refusals, { route: 0, expired: 0, exhausted: 41, unrouted: 1, oversized: 0 }); + // Without it this row is `started: 0, answered: 0` — an idle turn's row. + assert.deepEqual([(refused.seal?.mcp as Record).started, (refused.seal?.mcp as Record).answered], [0, 0]); + + const unobserved = await cancelledTurn(() => ({ started: 1, answered: 1, undecoded: 0, outstanding: [] })); + assert.equal(Object.hasOwn(unobserved.seal?.mcp as Record, "refusals"), false); +}); diff --git a/src/runtime/engineBrokerSealLedger.ts b/src/runtime/engineBrokerSealLedger.ts index c386838..a1d119a 100644 --- a/src/runtime/engineBrokerSealLedger.ts +++ b/src/runtime/engineBrokerSealLedger.ts @@ -1,4 +1,4 @@ -import { ENGINE_BROKER_MCP_CALL_NAME, ENGINE_BROKER_MCP_OUTSTANDING_MAX, ENGINE_BROKER_MCP_TUNNEL_MAX } from "./engineBrokerMcpCallLog.js"; +import { ENGINE_BROKER_MCP_CALL_NAME, ENGINE_BROKER_MCP_OUTSTANDING_MAX, ENGINE_BROKER_MCP_REFUSAL_REASONS, ENGINE_BROKER_MCP_TUNNEL_MAX } from "./engineBrokerMcpCallLog.js"; import type { EngineBrokerTerminalResponse } from "./engineBrokerProtocol.js"; import { TURN_USAGE_LEDGER, TURN_USAGE_MAX_IDENTIFIER_CHARS } from "./turnUsageLedger.js"; @@ -104,6 +104,13 @@ export const renderBrokerTurnSealLine = (terminal: EngineBrokerTerminalResponse, outstanding: terminal.mcpCalls.outstanding .slice(0, ENGINE_BROKER_MCP_OUTSTANDING_MAX) .map((call) => ({ name: ENGINE_BROKER_MCP_CALL_NAME.test(call.name) ? call.name : "", outstanding_ms: call.outstandingMs })), + // Every request the facade refused before it could relay it, by reason + // class. Without it a turn whose capability was spent — 128 requests, + // two per worker round — seals as `answered == started, outstanding: + // []`, which is what a healthy turn seals as. + ...(terminal.mcpCalls.refusals === undefined + ? {} + : { refusals: Object.fromEntries(ENGINE_BROKER_MCP_REFUSAL_REASONS.map((reason) => [reason, terminal.mcpCalls!.refusals![reason]])) }), // The session's standalone GET tunnel, absent for a turn sealed before // the facade observed that channel at all. ...(terminal.mcpCalls.tunnels === undefined From 35c9bd588fcd3d0d20d52aca1a40c54a3204be6b Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 15:01:04 +0200 Subject: [PATCH 61/69] test: pin the seal row's exact field set for a completed turn --- src/runtime/engineBrokerSealLedger.test.ts | 60 ++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/src/runtime/engineBrokerSealLedger.test.ts b/src/runtime/engineBrokerSealLedger.test.ts index a609601..d2a299f 100644 --- a/src/runtime/engineBrokerSealLedger.test.ts +++ b/src/runtime/engineBrokerSealLedger.test.ts @@ -185,3 +185,63 @@ test("a turn whose MCP requests were refused seals the refusals by reason, and a const unobserved = await cancelledTurn(() => ({ started: 1, answered: 1, undecoded: 0, outstanding: [] })); assert.equal(Object.hasOwn(unobserved.seal?.mcp as Record, "refusals"), false); }); + +/** + * The projection is an allow-list, and this is the assertion that makes it one. + * + * `renderBrokerTurnSealLine` copies a closed field set out of the sealed + * terminal response. Replacing that copy with `...terminal` passed every other + * test in this suite while writing `usage`, `diagnostic`, `mcpCalls` — and, for + * a completed turn, `text`: the model's entire reply, into a ledger whose whole + * rule is that it carries no prompt, body or reply. Nothing sealed a completed + * turn and read the file back, so nothing was watching the one row that + * carries a reply at all. + * + * The boundary: the exact key set of a written row, for the turn kind that has + * the most to leak. + * + * Mutation: spread the terminal into the row (`...terminal, v: ..., agent: ...`) + * and this goes red on both halves — the key set gains `text`, `kind`, + * `version`, `requestId`, `workerPid`, `workerUid`, `workerStartTime` and + * `usage`, and the reply itself appears in the file's bytes. + */ +const reply = "TANGERINE-7-IS-THE-MODELS-OWN-REPLY"; +const answered = (text: string): string => { + const session = "01a0ad21-a90f-7f71-8054-93fdb4334d6a"; + const usage = { input_tokens: 2_677, output_tokens: 92, cache_read_input_tokens: 2_816, cache_creation_input_tokens: 0 }; + return [ + { type: "system", subtype: "init", session_id: session }, + { type: "assistant", message: { id: "msg_0", type: "message", role: "assistant", model: "daimon-broker-grok", content: [{ type: "text", text }], stop_reason: "end_turn", usage }, parent_tool_use_id: null, session_id: session }, + { type: "result", subtype: "success", is_error: false, num_turns: 1, result: text, stop_reason: "end_turn", total_cost_usd: 0.0024, usage, modelUsage: { "grok-4.6-build": {} }, session_id: session } + ].map((frame) => JSON.stringify(frame)).join("\n"); +}; + +test("a completed turn's seal row carries exactly its declared fields, and never the model's reply", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daimon-broker-seal-completed-")); + try { + const usageLedgerPath = path.join(root, "usage.jsonl"); + const deps: GrokEngineBrokerTurnDependencies = { + turns: new EngineBrokerTurnRegistry(path.join(root, "turns")), proxy, credentialStale: () => false, + mcp: { register: () => "mcp-capability-0123456789abcdef", revoke: () => undefined, observe: () => ({ started: 1, answered: 1, undecoded: 0, outstanding: [] }) }, + prepareIsolation: async () => async () => undefined, + runNative: async () => ({ text: answered(reply), workerPid: 4_242, workerUid: 2_200, startTicks: 99n }) + }; + const result = await runGrokEngineBrokerTurn(deps, registration(usageLedgerPath), "wake-done", "prompt", "http://127.0.0.1:43124/mcp"); + assert.equal(result.text, reply, "the turn itself still answers with the model's reply"); + + const bytes = await readFile(engineBrokerSealLedgerPathFor(usageLedgerPath), "utf8"); + const lines = bytes.split("\n").filter((line) => line.length > 0); + assert.equal(lines.length, 1); + const row = JSON.parse(lines[0]!) as Record; + assert.deepEqual(Object.keys(row).sort(), ["agent", "at", "engine", "limit_reason", "model", "outcome", "requests", "turn", "v", "wake"]); + assert.deepEqual([row.v, row.agent, row.wake, row.engine, row.outcome, row.model, row.limit_reason, row.requests], [TURN_SEAL_LEDGER_VERSION, "foreman", "wake-done", "grok", "completed", "grok-4.6", "none", 1]); + // The second half of the same guarantee, on the bytes rather than the keys: + // a reply that reached the ledger under any name is the failure. + assert.ok(!bytes.includes(reply), "the model's reply must never reach the ledger"); + // A completed turn carries no failure members at all, and the facade's + // observation is a failed turn's member: neither may appear here. + for (const absent of ["text", "code", "diagnostic", "mcp", "usage", "workerPid", "workerUid", "kind", "version"]) { + assert.equal(Object.hasOwn(row, absent), false, absent); + } + } finally { await rm(root, { recursive: true, force: true }); } +}); From 3c7e74f65e51e0cdafe1283ec791acfcfbe96fc1 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 15:02:53 +0200 Subject: [PATCH 62/69] test: type the sealed completed turn's native result frame --- src/runtime/engineBrokerSealLedger.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/engineBrokerSealLedger.test.ts b/src/runtime/engineBrokerSealLedger.test.ts index d2a299f..b3d60c6 100644 --- a/src/runtime/engineBrokerSealLedger.test.ts +++ b/src/runtime/engineBrokerSealLedger.test.ts @@ -224,7 +224,7 @@ test("a completed turn's seal row carries exactly its declared fields, and never turns: new EngineBrokerTurnRegistry(path.join(root, "turns")), proxy, credentialStale: () => false, mcp: { register: () => "mcp-capability-0123456789abcdef", revoke: () => undefined, observe: () => ({ started: 1, answered: 1, undecoded: 0, outstanding: [] }) }, prepareIsolation: async () => async () => undefined, - runNative: async () => ({ text: answered(reply), workerPid: 4_242, workerUid: 2_200, startTicks: 99n }) + runNative: async () => ({ text: answered(reply), workerPid: 4_242, workerUid: 2_200, startTicks: 99n, diagnostic: { status: "ok", stage: "output", failureClass: "none", profileApplied: false, exitCode: 0, termSignal: 0, workerPid: 4_242, workerUid: 2_200, startTicks: "99" } }) }; const result = await runGrokEngineBrokerTurn(deps, registration(usageLedgerPath), "wake-done", "prompt", "http://127.0.0.1:43124/mcp"); assert.equal(result.text, reply, "the turn itself still answers with the model's reply"); From 074c75c924410981857b0abe77adf5645797e796 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 15:04:38 +0200 Subject: [PATCH 63/69] fix: assert and correct the mode of existing runtime-home subdirectories --- src/observability/causalEvents.ts | 11 +-- src/observability/orgObserver.ts | 7 +- src/pi/cliSession.ts | 14 +-- src/pi/piHarness.ts | 16 +--- src/pi/turnTrace.ts | 7 +- src/pi/worldTrajectory.ts | 7 +- src/runtime/AGENTS.md | 15 ++- src/runtime/runtimeHomeLayout.test.ts | 130 ++++++++++++++++++++++++-- src/runtime/runtimeHomeLayout.ts | 71 ++++++++++++++ 9 files changed, 228 insertions(+), 50 deletions(-) diff --git a/src/observability/causalEvents.ts b/src/observability/causalEvents.ts index 9a29dd5..b594157 100644 --- a/src/observability/causalEvents.ts +++ b/src/observability/causalEvents.ts @@ -1,8 +1,8 @@ import { createHash, randomUUID } from "node:crypto"; import { constants } from "node:fs"; -import { appendFile, mkdir, open, readFile, rename, stat, unlink } from "node:fs/promises"; +import { appendFile, open, readFile, rename, stat, unlink } from "node:fs/promises"; import path from "node:path"; -import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "../runtime/runtimeHomeLayout.js"; +import { ensureRuntimeHomeDirectory } from "../runtime/runtimeHomeLayout.js"; /** * Daimon's own copy of the `noopolis.causal-event.v1` wire envelope. Field- @@ -107,8 +107,7 @@ const readSeqStore = async (runtimeHomePath: string): Promise => }; const writeSeqStore = async (runtimeHomePath: string, store: CausalSeqStore): Promise => { - const directory = telemetryDir(runtimeHomePath); - await mkdir(directory, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); + const directory = await ensureRuntimeHomeDirectory(runtimeHomePath, "telemetry"); const file = seqFilePath(runtimeHomePath); const temporary = `${file}.${randomUUID()}.tmp`; const handle = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); @@ -183,7 +182,7 @@ export const nextCausalSeq = async (input: { const lockPath = path.resolve(telemetryDir(input.runtimeHomePath), "causal.seq.lock"); const previous = seqAllocationQueues.get(lockPath) ?? Promise.resolve(); const allocation = previous.catch(() => undefined).then(async () => { - await mkdir(telemetryDir(input.runtimeHomePath), { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); + await ensureRuntimeHomeDirectory(input.runtimeHomePath, "telemetry"); await acquireSeqLock(lockPath); try { const store = await readSeqStore(input.runtimeHomePath); @@ -208,7 +207,7 @@ export const nextCausalSeq = async (input: { /** Appends one CausalEvent record as a line of `runtimeHome/telemetry/causal.jsonl`. */ export const appendCausalEvent = async (runtimeHomePath: string, event: CausalEvent): Promise => { - await mkdir(telemetryDir(runtimeHomePath), { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); + await ensureRuntimeHomeDirectory(runtimeHomePath, "telemetry"); await appendFile(jsonlFilePath(runtimeHomePath), `${JSON.stringify(event)}\n`, "utf8"); }; diff --git a/src/observability/orgObserver.ts b/src/observability/orgObserver.ts index 2d21ef8..bc55516 100644 --- a/src/observability/orgObserver.ts +++ b/src/observability/orgObserver.ts @@ -1,8 +1,8 @@ -import { mkdir, writeFile } from "node:fs/promises"; +import { writeFile } from "node:fs/promises"; import path from "node:path"; import type { MemoryRecallAudit } from "@noopolis/mneme"; -import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "../runtime/runtimeHomeLayout.js"; +import { ensureRuntimeHomeDirectory } from "../runtime/runtimeHomeLayout.js"; export interface WakeBenchRow { agent: string; @@ -207,8 +207,7 @@ export class OrgObserver { } async write(runtimeRoot: string): Promise { - const telemetryDir = path.join(runtimeRoot, "telemetry"); - await mkdir(telemetryDir, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); + const telemetryDir = await ensureRuntimeHomeDirectory(runtimeRoot, "telemetry"); const summaryRecord = { assertions: this.assertions, behavior: this.behaviorSummary(), diff --git a/src/pi/cliSession.ts b/src/pi/cliSession.ts index f161009..b045f4b 100644 --- a/src/pi/cliSession.ts +++ b/src/pi/cliSession.ts @@ -1,7 +1,6 @@ import { randomUUID } from "node:crypto"; import { createServer, type Server } from "node:http"; import { spawn, type ChildProcess } from "node:child_process"; -import { mkdir } from "node:fs/promises"; import type { ToolDefinition } from "@earendil-works/pi-coding-agent"; import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; @@ -35,7 +34,7 @@ import { decodeGrokHeadlessResult } from "./grokHeadlessResult.js"; import { terminateChild, trackCliChild } from "./cliProcess.js"; import type { PiSessionLike } from "./piAgentHandle.js"; import type { PiSessionFactoryInput } from "./piHarness.js"; -import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "../runtime/runtimeHomeLayout.js"; +import { ensureRuntimeHome, ensureRuntimeHomeDirectory } from "../runtime/runtimeHomeLayout.js"; export type CliEngineKind = "agy" | "codex" | "grok"; @@ -139,14 +138,9 @@ type CliTurnEnd = Extract; export const prepareCliRuntimeHome = async (runtimeHomePath: string | undefined): Promise => { if (runtimeHomePath === undefined) return; - await Promise.all([ - runtimeHomePath, - `${runtimeHomePath}/.config`, - `${runtimeHomePath}/.local/share`, - `${runtimeHomePath}/.local/state`, - `${runtimeHomePath}/.cache`, - `${runtimeHomePath}/.tmp` - ].map((directory) => mkdir(directory, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }))); + await ensureRuntimeHome(runtimeHomePath); + await Promise.all([".config", ".local/share", ".local/state", ".cache", ".tmp"] + .map((relative) => ensureRuntimeHomeDirectory(runtimeHomePath, relative))); }; const childSecretValues = (redactedNames: readonly string[]): readonly string[] => diff --git a/src/pi/piHarness.ts b/src/pi/piHarness.ts index 69096d9..cc4e235 100644 --- a/src/pi/piHarness.ts +++ b/src/pi/piHarness.ts @@ -23,7 +23,7 @@ import { type PiWakeEnvironmentContextRef } from "./piAgentWakeSupport.js"; import { DAIMON_WAKE_ID_ENV } from "./cliEnvironment.js"; import { createPiWorldTools, piWorldToolNames, type PiWorldBinding } from "./worldTools.js"; import type { PiWorldToolContextRef } from "./worldNudge.js"; -import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "../runtime/runtimeHomeLayout.js"; +import { ensureRuntimeHome, ensureRuntimeHomeDirectory } from "../runtime/runtimeHomeLayout.js"; import { bindPiRawTrainingCapture, validatePiRawTrainingCaptureOptions, @@ -98,18 +98,12 @@ export class PiHarnessAdapter implements AgentHarnessAdapter { async startAgent(input: AgentStartInput): Promise { validatePiRawTrainingCaptureOptions(this.options.rawTrainingCapture); - await Promise.all([ - input.runtimeHomePath, - `${input.runtimeHomePath}/.config`, - `${input.runtimeHomePath}/.local/share`, - `${input.runtimeHomePath}/.local/state`, - `${input.runtimeHomePath}/.cache`, - `${input.runtimeHomePath}/.tmp`, - `${input.runtimeHomePath}/tool-state` - ].map((directory) => mkdir(directory, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }))); + await ensureRuntimeHome(input.runtimeHomePath); + await Promise.all([".config", ".local/share", ".local/state", ".cache", ".tmp", "tool-state"] + .map((relative) => ensureRuntimeHomeDirectory(input.runtimeHomePath, relative))); await mkdir(input.workspacePath, { recursive: true }); const memoryRuntimeHomePath = this.options.memory?.runtimeHomePath ?? input.runtimeHomePath; - await mkdir(memoryRuntimeHomePath, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); + await ensureRuntimeHome(memoryRuntimeHomePath); const modelSpec = this.options.model ?? { auth: { method: "codex" as const }, provider: "openai", diff --git a/src/pi/turnTrace.ts b/src/pi/turnTrace.ts index 5ac198b..1833ed0 100644 --- a/src/pi/turnTrace.ts +++ b/src/pi/turnTrace.ts @@ -1,12 +1,12 @@ import { createHash } from "node:crypto"; -import { appendFile, mkdir, writeFile } from "node:fs/promises"; +import { appendFile, writeFile } from "node:fs/promises"; import path from "node:path"; import type { MemoryPrepareTurnResult, MemoryWakeMode } from "@noopolis/mneme"; import type { HarnessModelSpec, WakeEvent } from "../core/types.js"; import { redactCredentialText } from "../core/credentialRedaction.js"; -import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "../runtime/runtimeHomeLayout.js"; +import { ensureRuntimeHomeDirectory } from "../runtime/runtimeHomeLayout.js"; export interface PiTurnTraceModel { authMethod: NonNullable["method"]; @@ -269,8 +269,7 @@ export const writeTurnTraceRecord = async ( record: PiTurnTraceRecord ): Promise => { const telemetryPath = path.join(runtimeHomePath, "telemetry"); - const turnsPath = path.join(telemetryPath, "turns"); - await mkdir(turnsPath, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); + const turnsPath = await ensureRuntimeHomeDirectory(runtimeHomePath, "telemetry/turns"); const body = `${JSON.stringify(record, null, 2)}\n`; await writeFile(path.join(turnsPath, `${sanitizeTraceFileId(record.turn_id)}.json`), body, "utf8"); await appendFile(path.join(telemetryPath, "turns.ndjson"), `${JSON.stringify(record)}\n`, "utf8"); diff --git a/src/pi/worldTrajectory.ts b/src/pi/worldTrajectory.ts index 6883ed0..9ee6f8b 100644 --- a/src/pi/worldTrajectory.ts +++ b/src/pi/worldTrajectory.ts @@ -1,11 +1,11 @@ import { createHash } from "node:crypto"; -import { appendFile, mkdir, writeFile } from "node:fs/promises"; +import { appendFile, writeFile } from "node:fs/promises"; import path from "node:path"; import type { PiTurnTraceModel } from "./turnTrace.js"; import { redactTraceText, sanitizeTraceFileId } from "./turnTrace.js"; import type { PiWorldTurnContext } from "./worldNudge.js"; -import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "../runtime/runtimeHomeLayout.js"; +import { ensureRuntimeHomeDirectory } from "../runtime/runtimeHomeLayout.js"; export const WORLD_TRAJECTORY_SCHEMA = "daimon.world_trajectory.v1" as const; @@ -189,8 +189,7 @@ export const persistPiWorldTrajectory = async ( } }; const telemetryPath = path.join(input.runtimeHomePath, "telemetry"); - const trajectoriesPath = path.join(telemetryPath, "world-trajectories"); - await mkdir(trajectoriesPath, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); + const trajectoriesPath = await ensureRuntimeHomeDirectory(input.runtimeHomePath, "telemetry/world-trajectories"); const bytes = `${JSON.stringify(record, null, 2)}\n`; await writeFile( path.join(trajectoriesPath, `${sanitizeTraceFileId(input.turnId)}.json`), diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 20b4e62..a84e495 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -462,7 +462,20 @@ in `deny` (verified: `/tmp`, `/var/tmp`, `/run`, `/etc`, `sessions` all fail; (`runtimeHomeLayout.ts`: telemetry, turn traces, world trajectories, `tool-state`, the engine XDG directories, `.tmp`), so a traversable home still exposes nothing but `tool-output/`. A deployment-provisioned memory - home under that runtime home must stay `0700` for the same reason; + home under that runtime home must stay `0700` for the same reason. That mode + is *asserted and corrected*, not merely passed to `mkdir`, because `mkdir`'s + `mode` decides nothing for a directory that already exists: a `telemetry/` + left at `0755` by a pre-branch Daimon or pre-created by a deployment stayed + `0755` forever, and under a `0710` home that is the worker reading its own + agent's prompts, replies and causal history. `ensureRuntimeHomeDirectory` + walks every level below the home, opens each through + `O_DIRECTORY|O_NOFOLLOW` and `fchmod`s the directory it stat'd; one owned by + another uid is **refused**, never widened, and a symlink planted where a + directory belongs is refused rather than followed. The home itself is + create-only (`ensureRuntimeHome`) — whether it should be `0700` or a Grok + agent's `0710` is `physicalReadiness.ts`'s judgement, not the layout's. The + mode constant lives only in that module, and a test fails the build if any + writer imports it again; - spills (`toolResultSpill.ts`) are written `0640`; provision `/tool-output` as `2000: 2750` (setgid) under a runtime home the worker can traverse, so each spill carries that agent's diff --git a/src/runtime/runtimeHomeLayout.test.ts b/src/runtime/runtimeHomeLayout.test.ts index 6c4363d..7be7c9b 100644 --- a/src/runtime/runtimeHomeLayout.test.ts +++ b/src/runtime/runtimeHomeLayout.test.ts @@ -1,12 +1,12 @@ import assert from "node:assert/strict"; -import { mkdir, mkdtemp, readFile, readdir, rm, stat } from "node:fs/promises"; +import { chmod, mkdir, mkdtemp, readFile, readdir, rm, stat, symlink } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import test from "node:test"; import { appendCausalEvent, CAUSAL_EVENT_VERSION, nextCausalSeq } from "../observability/causalEvents.js"; import { summarizePrompt, writeTurnTraceRecord } from "../pi/turnTrace.js"; -import { RUNTIME_HOME_SUBDIRECTORY_MODE } from "./runtimeHomeLayout.js"; +import { ensureRuntimeHome, ensureRuntimeHomeDirectory, RUNTIME_HOME_SUBDIRECTORY_MODE } from "./runtimeHomeLayout.js"; /** * A brokered Grok runtime home is traversable by its worker uid (0710), so @@ -22,6 +22,15 @@ const withTraversableHome = async (body: (home: string) => Promise): Promi }; const mode = async (target: string): Promise => (await stat(target)).mode & 0o7777; +const traceRecord = { + agent_id: "mapper", completed_at: "2026-01-01T00:00:01.000Z", + engine: { auth_method: "none", kind: "pi", model: "llama3.2", provider: "local" }, + memory: { enabled: false }, prompt: summarizePrompt("hi"), reply: { output_chars: 2, reply_given: true }, + schema: "daimon.turn_trace.v1", session: { dispose_after_wake: false, mode: "awake", thread_id: "t" }, + started_at: "2026-01-01T00:00:00.000Z", status: "completed", timings_ms: { total: 1 }, tools: [], + turn_id: "turn-1", wake: { event_id: "w", kind: "message" } +} as unknown as Parameters[1]; + test("the runtime-home subdirectory mode grants nobody but the runtime user", () => { assert.equal(RUNTIME_HOME_SUBDIRECTORY_MODE, 0o700); }); @@ -39,18 +48,119 @@ test("telemetry directories Daimon creates in a traversable runtime home are pri await nextCausalSeq({ runtimeHomePath: home, agentId: "a", turnId: "t1", count: 1 } as never); assert.equal(await mode(path.join(home, "telemetry")), RUNTIME_HOME_SUBDIRECTORY_MODE); - await writeTurnTraceRecord(home, { - agent_id: "mapper", completed_at: "2026-01-01T00:00:01.000Z", - engine: { auth_method: "none", kind: "pi", model: "llama3.2", provider: "local" }, - memory: { enabled: false }, prompt: summarizePrompt("hi"), reply: { output_chars: 2, reply_given: true }, - schema: "daimon.turn_trace.v1", session: { dispose_after_wake: false, mode: "awake", thread_id: "t" }, - started_at: "2026-01-01T00:00:00.000Z", status: "completed", timings_ms: { total: 1 }, tools: [], - turn_id: "turn-1", wake: { event_id: "w", kind: "message" } - }); + await writeTurnTraceRecord(home, traceRecord); + assert.equal(await mode(path.join(home, "telemetry", "turns")), RUNTIME_HOME_SUBDIRECTORY_MODE); + }); +}); + +/** + * The half the mode argument never covered. + * + * `mkdir(..., { mode })` decides nothing for a directory that already exists, + * and `assertRuntimeDirectory` checks the home and not what Daimon creates + * inside it. So a `telemetry/` left at 0755 by a pre-branch Daimon — or + * pre-created by a deployment — stayed 0755 under a Grok agent's deliberately + * traversable 0710 home, where it is the sandboxed worker reading its own + * agent's prompts, replies and causal history. + * + * The boundary these assertions straddle: a fresh install against an existing + * one. Every writer below is reached through its real entry point, because the + * hole was never in the mode constant — it was in what the call sites did with + * it. + * + * Mutation: restore `mkdir(directory, { recursive: true, mode })` in + * `ensureRuntimeHomeDirectory` and every assertion here goes red while the + * fresh-install test above stays green, which is exactly how this shipped. + */ +test("a runtime-home subdirectory that already exists is made private, not left as it was found", async () => { + await withTraversableHome(async (home) => { + // Pre-created by a deployment, or by a Daimon that predates the mode. + await mkdir(path.join(home, "telemetry"), { mode: 0o755 }); + await appendCausalEvent(home, { + version: CAUSAL_EVENT_VERSION, id: "daimon:t1:turn.output.completed", type: "turn.output.completed", + occurred_at: "2026-01-01T00:00:00.000Z", actor: { kind: "agent", id: "a" }, subject: { kind: "turn", id: "t1" }, + causes: [], run_id: "run", seq: 1, payload: {} + } as never); + assert.equal(await mode(path.join(home, "telemetry")), RUNTIME_HOME_SUBDIRECTORY_MODE); + + // An existing ancestor is the same hole one level up: `telemetry/turns` can + // be created privately under a `telemetry/` that stays world-readable. + await chmod(path.join(home, "telemetry"), 0o755); + await mkdir(path.join(home, "telemetry", "turns"), { mode: 0o755 }); + await writeTurnTraceRecord(home, traceRecord); + assert.equal(await mode(path.join(home, "telemetry")), RUNTIME_HOME_SUBDIRECTORY_MODE, "the ancestor is corrected too"); assert.equal(await mode(path.join(home, "telemetry", "turns")), RUNTIME_HOME_SUBDIRECTORY_MODE); + + // And the home itself is never touched: which mode it should carry is + // `physicalReadiness.ts`'s judgement, and for a Grok agent it is 0710. + assert.equal(await mode(home), 0o710); + }); +}); + +/** + * Refuse rather than widen, and never follow a link to do it. + * + * A directory the runtime does not own cannot be made private by it, and + * writing an agent's telemetry into it anyway is the failure the correction + * exists to prevent. A symlink planted where a directory belongs is the same + * fault with an attacker attached, so the correction goes through an + * `O_DIRECTORY|O_NOFOLLOW` handle and the `fchmod` lands on the directory that + * was stat'd. + * + * Mutation: drop the owner check and the first case silently proceeds; drop + * `O_NOFOLLOW` and the second chmods the link's target instead of refusing. + */ +test("a runtime-home subdirectory owned by another user, or replaced by a symlink, is refused", async () => { + await withTraversableHome(async (home) => { + await mkdir(path.join(home, "telemetry"), { mode: 0o755 }); + const foreign = (process.getuid?.() ?? 0) + 4_242; + await assert.rejects(ensureRuntimeHomeDirectory(home, "telemetry", foreign), /owned by the runtime user/u); + assert.equal(await mode(path.join(home, "telemetry")), 0o755, "a refusal corrects nothing and widens nothing"); + + const elsewhere = path.join(home, "elsewhere"); + await mkdir(elsewhere, { mode: 0o755 }); + await symlink(elsewhere, path.join(home, "tool-state")); + await assert.rejects(ensureRuntimeHomeDirectory(home, "tool-state")); + assert.equal(await mode(elsewhere), 0o755, "the link's target must not be chmod'ed through it"); + }); +}); + +test("the home itself is created when absent and never re-moded when present", async () => { + await withTraversableHome(async (home) => { + const fresh = path.join(home, "fresh-home"); + assert.equal(await ensureRuntimeHome(fresh), fresh); + assert.equal(await mode(fresh), RUNTIME_HOME_SUBDIRECTORY_MODE); + await chmod(fresh, 0o710); + await ensureRuntimeHome(fresh); + assert.equal(await mode(fresh), 0o710, "a Grok agent's traversable home is not this helper's judgement"); }); }); +/** + * The rule that keeps the correction from being reintroducible. + * + * `mkdir(..., { mode })` reads like a guarantee and is one only for a + * directory that does not exist yet, so the mode constant stays private to + * this module: a call site that wants a private directory under a runtime home + * asks `ensureRuntimeHomeDirectory` for one and gets the assertion with it. + * + * Mutation: import the constant into any writer and pass it to `mkdir` again, + * and this goes red — which is the shape the 0755 hole had. + */ +test("the private mode is used only where it is also asserted", async () => { + const offenders: string[] = []; + const walk = async (directory: string): Promise => { + for (const entry of await readdir(directory, { withFileTypes: true })) { + const target = path.join(directory, entry.name); + if (entry.isDirectory()) { if (entry.name !== "fixtures" && entry.name !== "artifacts") await walk(target); continue; } + if (!entry.name.endsWith(".ts") || entry.name.endsWith(".test.ts") || target === "src/runtime/runtimeHomeLayout.ts") continue; + if ((await readFile(target, "utf8")).includes("RUNTIME_HOME_SUBDIRECTORY_MODE")) offenders.push(target); + } + }; + await Promise.all(["src/pi", "src/observability", "src/runtime", "src/mcp", "src/core"].map(walk)); + assert.deepEqual(offenders, []); +}); + // Creations that are not inside an agent's runtime home. const OUTSIDE_RUNTIME_HOME = [ "src/runtime/native/copyArtifact.ts", "src/observability/emitCausalFixture.ts", "src/pi/auth.ts", "src/runtime/cli.ts" diff --git a/src/runtime/runtimeHomeLayout.ts b/src/runtime/runtimeHomeLayout.ts index f8a280d..d1b5c1f 100644 --- a/src/runtime/runtimeHomeLayout.ts +++ b/src/runtime/runtimeHomeLayout.ts @@ -1,3 +1,7 @@ +import { constants } from "node:fs"; +import { mkdir, open } from "node:fs/promises"; +import path from "node:path"; + /** * Mode for every directory Daimon creates inside an agent's runtime home. * @@ -11,3 +15,70 @@ * readable one. */ export const RUNTIME_HOME_SUBDIRECTORY_MODE = 0o700; + +/** + * The home itself, created if it is absent and otherwise left exactly as it is. + * + * Create-only is the whole contract here. A brokered Grok agent's home is + * deliberately `0710` and an organization's may be `0700`; which of the two is + * correct is `physicalReadiness.ts`'s judgement, made against the agent's + * declared engine, and a layout helper that "corrected" a traversable home to + * `0700` would break the worker's only route to its own spills. + */ +export const ensureRuntimeHome = async (runtimeHomePath: string): Promise => { + await mkdir(runtimeHomePath, { recursive: true, mode: RUNTIME_HOME_SUBDIRECTORY_MODE }); + return runtimeHomePath; +}; + +/** + * One directory Daimon owns *below* a runtime home, private on every install. + * + * `mkdir(..., { mode })` decides nothing for a directory that already exists, + * and that is the common case rather than the exotic one: a `telemetry/` left + * at `0755` by a pre-branch Daimon, or pre-created by a deployment, stayed + * `0755` forever. Under a Grok agent's traversable `0710` home that is the + * worker reading its own agent's prompts, replies and causal history — the + * home is traverse-only precisely so that nothing but `tool-output/` is + * readable. `assertRuntimeDirectory` checks the home, and nothing checked what + * Daimon created inside it. + * + * So every level below the home is asserted and corrected on the way down, + * through a handle rather than a path: `O_DIRECTORY|O_NOFOLLOW` refuses a + * symlink planted where a directory belongs, and the `fchmod` that follows + * lands on the directory that was stat'd. A directory owned by anyone but the + * runtime user is **refused**, never widened and never silently accepted — the + * runtime cannot make someone else's directory private, and proceeding would + * write an agent's telemetry into it anyway. + * + * `owner` is a seam so both refusals are testable unprivileged, exactly as + * `physicalReadiness.ts`'s `RuntimeIdentity` is. + */ +export async function ensureRuntimeHomeDirectory(runtimeHomePath: string, relative: string, owner: number = process.getuid?.() ?? -1): Promise { + const segments = relative.split("/").filter((segment) => segment.length > 0); + if (segments.length === 0 || segments.some((segment) => segment === "." || segment === "..")) { + throw new Error(`runtime home subdirectory must name a path below the home: ${relative}`); + } + let current = await ensureRuntimeHome(runtimeHomePath); + for (const segment of segments) { + current = path.join(current, segment); + await mkdir(current, { mode: RUNTIME_HOME_SUBDIRECTORY_MODE }).catch((error: NodeJS.ErrnoException) => { + if (error.code !== "EEXIST") throw error; + }); + await assertPrivateDirectory(current, owner); + } + return current; +} + +const flag = (name: "O_DIRECTORY" | "O_NOFOLLOW"): number => (constants as typeof constants & Partial>)[name] ?? 0; + +async function assertPrivateDirectory(directory: string, owner: number): Promise { + const handle = await open(directory, constants.O_RDONLY | flag("O_DIRECTORY") | flag("O_NOFOLLOW")); + try { + const entry = await handle.stat(); + if (!entry.isDirectory()) throw new Error(`runtime home path is not a directory: ${directory}`); + if (entry.uid !== owner) throw new Error(`runtime home subdirectory must be owned by the runtime user: ${directory}`); + if ((entry.mode & 0o7777) !== RUNTIME_HOME_SUBDIRECTORY_MODE) await handle.chmod(RUNTIME_HOME_SUBDIRECTORY_MODE); + } finally { + await handle.close(); + } +} From f81c55ed69d743a8d1cf22bbd74e2ac0616168cf Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 15:08:13 +0200 Subject: [PATCH 64/69] fix: ensure the agent tool-state directory through the runtime-home layout --- src/runtime/AGENTS.md | 8 ++++++-- src/runtime/productionAgentTools.ts | 5 +++-- src/runtime/runtimeHomeLayout.test.ts | 22 ++++++++++++++++++++-- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index a84e495..57a657e 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -474,8 +474,12 @@ in `deny` (verified: `/tmp`, `/var/tmp`, `/run`, `/etc`, `sessions` all fail; directory belongs is refused rather than followed. The home itself is create-only (`ensureRuntimeHome`) — whether it should be `0700` or a Grok agent's `0710` is `physicalReadiness.ts`'s judgement, not the layout's. The - mode constant lives only in that module, and a test fails the build if any - writer imports it again; + mode constant lives only in that module, a test fails the build if any writer + imports it again, and the same test refuses any `mkdir` that names a runtime + home outside the layout — a `mode:` argument covers only the install where + the directory is new. `wakeAcceptanceFs.ts` is the one exception and closes + the hole the other way, by asserting the directory it found and refusing a + wider one; - spills (`toolResultSpill.ts`) are written `0640`; provision `/tool-output` as `2000: 2750` (setgid) under a runtime home the worker can traverse, so each spill carries that agent's diff --git a/src/runtime/productionAgentTools.ts b/src/runtime/productionAgentTools.ts index a23f304..1b7c702 100644 --- a/src/runtime/productionAgentTools.ts +++ b/src/runtime/productionAgentTools.ts @@ -1,7 +1,7 @@ import { spawn } from "node:child_process"; import { createHash } from "node:crypto"; import { constants } from "node:fs"; -import { lstat, mkdir, open, readdir, rename, unlink } from "node:fs/promises"; +import { lstat, open, readdir, rename, unlink } from "node:fs/promises"; import path from "node:path"; import type { ToolDefinition } from "@earendil-works/pi-coding-agent"; @@ -13,6 +13,7 @@ import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/ import type { OrganizationRuntimeAgentConfig, OrganizationRuntimeMcpServer } from "./organizationRuntime.js"; import { moltnetOperationResult, readMoltnetPages } from "./moltnetMachineRead.js"; import { McpToolCallError, MCP_TOOL_RESULT_MAX_BYTES, renderMcpToolResult, replayMcpReceipt, type McpUpstreamResult } from "./mcpToolResult.js"; +import { ensureRuntimeHomeDirectory } from "./runtimeHomeLayout.js"; import { capToolResult, resolveExemptToolNames, resolveToolResultMaxBytes, TOOL_OUTPUT_DIRECTORY_NAME } from "./toolResultSpill.js"; import { cliChildEnvironment } from "../pi/cliEnvironment.js"; import type { PiWakeEnvironmentContextRef } from "../pi/piAgentWakeSupport.js"; @@ -31,7 +32,7 @@ const MAX_RESULT = 65_536; const TIMEOUT = 10_000; const DAIMON_ACTION_ID_PREFIX = "daimon-"; export async function createProductionAgentTools(agent: OrganizationRuntimeAgentConfig, wakeContext: PiWakeEnvironmentContextRef = {}): Promise { - await mkdir(path.join(agent.runtimeHomePath, "tool-state"), { recursive: true, mode: 0o700 }); + await ensureRuntimeHomeDirectory(agent.runtimeHomePath, "tool-state"); // Resolved once, at agent start: a malformed bound is a configuration error // that should refuse the agent, not a surprise thrown from the middle of a // tool call the model is waiting on. diff --git a/src/runtime/runtimeHomeLayout.test.ts b/src/runtime/runtimeHomeLayout.test.ts index 7be7c9b..8b1a867 100644 --- a/src/runtime/runtimeHomeLayout.test.ts +++ b/src/runtime/runtimeHomeLayout.test.ts @@ -166,6 +166,16 @@ const OUTSIDE_RUNTIME_HOME = [ "src/runtime/native/copyArtifact.ts", "src/observability/emitCausalFixture.ts", "src/pi/auth.ts", "src/runtime/cli.ts" ]; +/** + * Files that may name a runtime home in a `mkdir` of their own. + * + * `runtimeHomeLayout.ts` is the correction itself. `wakeAcceptanceFs.ts` + * creates the home and its store directory and then *asserts* each one — + * refusing a directory it finds wider rather than correcting it — which closes + * the same hole the other way and is its own documented contract. + */ +const MAY_MKDIR_A_RUNTIME_HOME = ["src/runtime/runtimeHomeLayout.ts", "src/pi/wakeAcceptanceFs.ts"]; + const mkdirCalls = (source: string): string[] => { const calls: string[] = []; for (let index = source.indexOf("mkdir("); index !== -1; index = source.indexOf("mkdir(", index + 1)) { @@ -178,9 +188,10 @@ const mkdirCalls = (source: string): string[] => { return calls; }; -test("no runtime-home directory is created without an explicit private mode", async () => { +test("no runtime-home directory is created without an explicit private mode, and none is created outside the layout", async () => { // Source policy: a default `mkdir` under an agent's runtime home would be 0755, - // and the home of a brokered Grok agent is traversable by its worker uid. + // the home of a brokered Grok agent is traversable by its worker uid, and a + // `mode:` argument only covers the install where the directory is new. const offenders: string[] = []; const walk = async (directory: string): Promise => { for (const entry of await readdir(directory, { withFileTypes: true })) { @@ -189,6 +200,13 @@ test("no runtime-home directory is created without an explicit private mode", as if (!entry.name.endsWith(".ts") || entry.name.endsWith(".test.ts") || OUTSIDE_RUNTIME_HOME.includes(target)) continue; const source = await readFile(target, "utf8"); for (const call of mkdirCalls(source)) { + // A `mode:` argument is a create-only mode: it decides nothing for a + // directory that already exists, so naming a runtime home in a `mkdir` + // is the defect whether or not a mode is passed. + if (/runtimeHome/iu.test(call) && !MAY_MKDIR_A_RUNTIME_HOME.includes(target)) { + offenders.push(`${target}: ${call.replace(/\s+/gu, " ").slice(0, 90)}`); + continue; + } // The workspace is a caller-prepared root with its own contract (group-readable for Grok). if (call.includes("mode:") || call.includes("{ mode }") || call.includes("workspacePath")) continue; offenders.push(`${target}: ${call.replace(/\s+/gu, " ").slice(0, 90)}`); From 734372a50e30fcb4014b7403e6c53a21bd6f475e Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 15:12:34 +0200 Subject: [PATCH 65/69] fix: derive the MCP capability budget from the compiled worker turn bound --- src/runtime/AGENTS.md | 19 +++++-- src/runtime/engineBrokerMcpCallLog.ts | 10 ++-- src/runtime/engineBrokerMcpFacade.ts | 31 +++++++++--- .../engineBrokerMcpObservation.test.ts | 50 +++++++++++++------ src/runtime/engineBrokerSealLedger.test.ts | 5 +- src/runtime/engineBrokerSealLedger.ts | 6 +-- 6 files changed, 83 insertions(+), 38 deletions(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 57a657e..0b91035 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -339,10 +339,21 @@ byte-identical to a healthy turn. `EngineBrokerMcpCallLog.refuse` now counts each one by a closed reason class (`route`, `expired`, `exhausted`, `unrouted`, `oversized`), because the classes call for opposite fixes: an exhausted per-turn capability is a budget, an unserved route is a worker -asking for something that does not exist. The budget is reachable rather than -theoretical — `ENGINE_BROKER_MCP_CAPABILITY_REQUESTS` (128) covers every POST, -the GET tunnel and the DELETE, and a `search_tool`+`use_tool` round spends two, -so a 48-round wake asks for ~96 plus its handshake. Attribution comes from +asking for something that does not exist. That budget is *derived*, not +picked: `ENGINE_BROKER_MCP_CAPABILITY_REQUESTS` is `GROK_WORKER_MAX_TURNS` +times `ENGINE_BROKER_MCP_ROUND_REQUESTS` (3 — a round's `search_tool`, its +`use_tool`, and one spare for a retry or a second discovery) plus +`ENGINE_BROKER_MCP_SESSION_REQUESTS` (5 — `initialize`, +`notifications/initialized`, `tools/list`, the GET tunnel, the DELETE). It was +a literal 128 against a bound of 48 rounds whose legitimate traffic is ~101, so +the first round that also retried met a mid-turn 403 storm; the two numbers +that must agree now live in one place, and raising the turn bound can no longer +silently exhaust the budget. It stays a bound rather than a comfortable number +because the derivation is exact: the request *after* the worst-case legitimate +session is refused, so a compromised worker gets three MCP calls per round it +was compiled to take and not one more. `engineBrokerMcpObservation.test.ts` +drives that worst case through the real facade, computed from the turn bound +alone. Attribution comes from `EngineBrokerCapabilities.classifyToken`, which names the token's turn and why it would be refused *without spending its budget*; a bearer no grant matches names no turn and stays unattributed, because guessing an owner would be diff --git a/src/runtime/engineBrokerMcpCallLog.ts b/src/runtime/engineBrokerMcpCallLog.ts index 6654102..f8d8b8d 100644 --- a/src/runtime/engineBrokerMcpCallLog.ts +++ b/src/runtime/engineBrokerMcpCallLog.ts @@ -61,11 +61,11 @@ * every one of whose requests was 403'd sealed as `answered == started, * outstanding: []` — byte-identical to a healthy turn, which is precisely the * reading this instrument exists to make trustworthy. The reason class is what - * makes it actionable: an exhausted per-turn capability budget (a worker's - * `search_tool`+`use_tool` pair per round is two requests of the facade's - * `ENGINE_BROKER_MCP_CAPABILITY_REQUESTS`) is a different fault from a - * mount that was never registered, and both differ from a worker asking for a - * route the facade does not serve. Counts only, keyed by a closed vocabulary: + * makes it actionable: an exhausted per-turn capability budget — the facade's + * `ENGINE_BROKER_MCP_CAPABILITY_REQUESTS`, derived from the compiled turn + * bound times what a round may spend — is a different fault from a mount that + * was never registered, and both differ from a worker asking for a route the + * facade does not serve. Counts only, keyed by a closed vocabulary: * never the token, the capability, the URL or the body. A refusal the facade * cannot attribute to a turn — a bearer no live grant matches — is recorded * nowhere, because attributing it to a turn would be inventing the fact. diff --git a/src/runtime/engineBrokerMcpFacade.ts b/src/runtime/engineBrokerMcpFacade.ts index 44eccfe..68cd0d3 100644 --- a/src/runtime/engineBrokerMcpFacade.ts +++ b/src/runtime/engineBrokerMcpFacade.ts @@ -1,5 +1,6 @@ import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import { Readable } from "node:stream"; +import { GROK_WORKER_MAX_TURNS } from "../contracts/grokWorkerContract.js"; import { EngineBrokerCapabilities } from "./engineBrokerCapabilities.js"; import { EngineBrokerMcpCallLog, type EngineBrokerMcpCallObservation, type EngineBrokerMcpRefusalReason, type EngineBrokerMcpTunnelHandle } from "./engineBrokerMcpCallLog.js"; @@ -46,17 +47,31 @@ const FORWARDED_RESPONSE_HEADERS = ["content-type", "mcp-session-id", "mcp-proto const FORWARDED_METHODS = new Set(["POST", "GET", "DELETE"]); const MAX_REQUEST_BYTES = 1024 * 1024; export const ENGINE_BROKER_MCP_FACADE_PORT = 43_124; + +/** Requests one worker round may legitimately make: its `search_tool`, its `use_tool`, and one spare for a retry or a second discovery. */ +export const ENGINE_BROKER_MCP_ROUND_REQUESTS = 3; +/** The session's fixed cost, once per turn: `initialize`, `notifications/initialized`, `tools/list`, the standalone GET tunnel, the closing DELETE. */ +export const ENGINE_BROKER_MCP_SESSION_REQUESTS = 5; /** - * Requests one turn capability may spend, across all three methods. + * Requests one turn capability may spend, across all three methods — *derived* + * from the compiled turn bound rather than chosen. + * + * It was 128, and a legitimate 48-round wake needs ~101 of them: a worker's + * round is a `search_tool` and a `use_tool`, so the first round that also + * retries, or looks something up twice, eats the margin. A bound that can be + * predicted to bite mid-turn is not a bound, it is a 403 storm waiting for a + * real wake — and a number raised until it feels comfortable is not one + * either, because the budget exists to cap a *compromised* worker. * - * A worker's round is a `search_tool` and a `use_tool`, so a 48-turn wake is - * ~96 POSTs plus the handshake, the standalone GET tunnel and the closing - * DELETE: exhaustion is reachable rather than theoretical, and every request - * past it is a 403 the worker cannot explain. That is why the refusal is - * counted and sealed (`engineBrokerMcpCallLog.ts`) rather than being an - * absence in the turn's row. + * So the two numbers that must agree are kept in one place: the launcher's + * `--max-turns` backstop ({@link GROK_WORKER_MAX_TURNS}) times what a round + * may legitimately spend, plus the session's fixed cost. Raising the turn + * bound raises this with it, and a compromised worker still gets exactly three + * MCP calls per round it was compiled to take and not one more. The arithmetic + * is spelled out rather than folded into a literal so it can be audited: the + * two multiplicands above each say what they are. */ -export const ENGINE_BROKER_MCP_CAPABILITY_REQUESTS = 128; +export const ENGINE_BROKER_MCP_CAPABILITY_REQUESTS = GROK_WORKER_MAX_TURNS * ENGINE_BROKER_MCP_ROUND_REQUESTS + ENGINE_BROKER_MCP_SESSION_REQUESTS; export const ENGINE_BROKER_MCP_CAPABILITY_TTL_MS = 15 * 60_000; class FacadeRefusal extends Error {} diff --git a/src/runtime/engineBrokerMcpObservation.test.ts b/src/runtime/engineBrokerMcpObservation.test.ts index fc2b42d..86fdfeb 100644 --- a/src/runtime/engineBrokerMcpObservation.test.ts +++ b/src/runtime/engineBrokerMcpObservation.test.ts @@ -6,7 +6,8 @@ import test from "node:test"; import { ToolListChangedNotificationSchema } from "@modelcontextprotocol/sdk/types.js"; import type { EngineBrokerMcpTunnelObservation } from "./engineBrokerMcpCallLog.js"; -import { ENGINE_BROKER_MCP_CAPABILITY_REQUESTS, ENGINE_BROKER_MCP_FACADE_PORT } from "./engineBrokerMcpFacade.js"; +import { GROK_WORKER_MAX_TURNS } from "../contracts/grokWorkerContract.js"; +import { ENGINE_BROKER_MCP_FACADE_PORT, ENGINE_BROKER_MCP_ROUND_REQUESTS, ENGINE_BROKER_MCP_SESSION_REQUESTS } from "./engineBrokerMcpFacade.js"; import { connectClient, FACADE_URL, sharedFacade, startRig, withDeadline } from "./engineBrokerMcpFacadeRig.test.js"; /** @@ -161,18 +162,32 @@ test("the facade observes the standalone GET tunnel: open with its age, whether * `route()` refuses before the call log is ever touched, so a request the * facade 403'd never reached `started`, `undecoded` or `outstanding`: a turn * whose every request was refused sealed as `answered == started, - * outstanding: []`, which is exactly what a healthy turn seals as. The budget - * makes that reachable rather than theoretical — one capability buys - * {@link ENGINE_BROKER_MCP_CAPABILITY_REQUESTS} requests across all three - * methods, and a worker spends two of them per round. + * outstanding: []`, which is exactly what a healthy turn seals as. + * + * The middle of this test is a second guarantee and it is a *relationship*, + * not a number. The demand side is computed from the launcher's compiled + * `--max-turns` backstop alone — every round spending its full MCP allowance, + * plus the session's fixed cost — and the whole of it must be served on one + * capability, with the very next request refused. Two numbers that must agree + * and live apart drift: the budget was a literal 128 against a bound of 48 + * rounds, ~101 requests of legitimate traffic, and the first round that also + * retried would have met a mid-turn 403 storm. Deriving one from the other is + * what makes raising `GROK_WORKER_MAX_TURNS` unable to silently exhaust the + * budget — and asserting the refusal one past the worst case is what keeps the + * derivation a bound rather than a comfortable number. * * The boundary these assertions straddle: a turn served against a turn - * refused, and within the refusals, a spent capability against a route the - * facade does not serve — the first is a budget to raise, the second is a - * worker asking for something that does not exist. + * refused; within the refusals, a spent capability against a route the facade + * does not serve; and, for the budget, legitimate worst-case traffic against + * the first request beyond it. * * Mutation: restore `throw new FacadeRefusal()` in place of either `refuse` - * call in `route()`, and a 403'd turn reads as an idle one again. + * call in `route()`, and a 403'd turn reads as an idle one again. Pin + * `ENGINE_BROKER_MCP_CAPABILITY_REQUESTS` back to a literal `128` and this is + * already red at today's turn bound; raise `GROK_WORKER_MAX_TURNS` beside that + * literal and it stays red, which is the drift the derivation removes. Raising + * `GROK_WORKER_MAX_TURNS` *with* the derivation keeps it green, which is the + * guarantee itself. */ test("a request the facade refused is counted against its turn, by reason, and is never a call it served", async () => { const facade = await sharedFacade(); @@ -202,13 +217,18 @@ test("a request the facade refused is counted against its turn, by reason, and i assert.equal(await send(FACADE_URL, "wrong-token-abcdefghijklmnopqrstuvwxyz0123456789"), 403); assert.equal(facade.observe(turnId)?.refusals?.route, 2, "an unattributable refusal belongs to no turn"); - // Neither refusal spent the capability, so the budget is exactly what was - // issued — and spending it all is reachable: a 48-round worker asks for - // ~96 of these plus its handshake, tunnel and DELETE. - for (let spent = 0; spent < ENGINE_BROKER_MCP_CAPABILITY_REQUESTS; spent += 1) assert.equal(await send(FACADE_URL, token), 200); - assert.equal(served, ENGINE_BROKER_MCP_CAPABILITY_REQUESTS); + // Neither refusal spent the capability, so what follows is the whole of it. + // The demand is read off the compiled turn bound and nothing else: every + // round the launcher admits, each spending its full MCP allowance, plus the + // one-off session cost. All of it must be served. + const legitimate = GROK_WORKER_MAX_TURNS * ENGINE_BROKER_MCP_ROUND_REQUESTS + ENGINE_BROKER_MCP_SESSION_REQUESTS; + for (let spent = 0; spent < legitimate; spent += 1) assert.equal(await send(FACADE_URL, token), 200, `request ${spent + 1} of ${legitimate} was refused`); + assert.equal(served, legitimate, "every legitimate request must reach the mount"); + // And it is still a bound: the first request past the worst case is refused, + // so the budget caps a compromised worker at exactly the traffic the turn + // bound compiles for. assert.equal(await send(FACADE_URL, token), 403, "the capability's budget is spent"); - assert.equal(served, ENGINE_BROKER_MCP_CAPABILITY_REQUESTS, "an exhausted capability never reaches the mount"); + assert.equal(served, legitimate, "an exhausted capability never reaches the mount"); const observed = facade.observe(turnId); assert.deepEqual(observed?.refusals, { route: 2, expired: 0, exhausted: 1, unrouted: 0, oversized: 0 }); diff --git a/src/runtime/engineBrokerSealLedger.test.ts b/src/runtime/engineBrokerSealLedger.test.ts index b3d60c6..5639d94 100644 --- a/src/runtime/engineBrokerSealLedger.test.ts +++ b/src/runtime/engineBrokerSealLedger.test.ts @@ -163,9 +163,8 @@ test("a cancelled turn's GET tunnel is sealed open with its age, closed, or neve * The refusal, on the same durable route as the hang it looks like. * * A request the facade 403'd never reached the relay, so before it was counted - * a turn whose capability was spent — 128 requests, two per worker round — - * sealed `answered == started, outstanding: []`, which is exactly what a - * healthy turn seals. The row has to carry the reason class, because an + * a turn whose capability was spent sealed `answered == started, + * outstanding: []`, which is exactly what a healthy turn seals. The row has to carry the reason class, because an * exhausted budget and an unserved route are opposite fixes. * * Mutation: drop the `refusals` member from `renderBrokerTurnSealLine` and the diff --git a/src/runtime/engineBrokerSealLedger.ts b/src/runtime/engineBrokerSealLedger.ts index a1d119a..4d64893 100644 --- a/src/runtime/engineBrokerSealLedger.ts +++ b/src/runtime/engineBrokerSealLedger.ts @@ -105,9 +105,9 @@ export const renderBrokerTurnSealLine = (terminal: EngineBrokerTerminalResponse, .slice(0, ENGINE_BROKER_MCP_OUTSTANDING_MAX) .map((call) => ({ name: ENGINE_BROKER_MCP_CALL_NAME.test(call.name) ? call.name : "", outstanding_ms: call.outstandingMs })), // Every request the facade refused before it could relay it, by reason - // class. Without it a turn whose capability was spent — 128 requests, - // two per worker round — seals as `answered == started, outstanding: - // []`, which is what a healthy turn seals as. + // class. Without it a turn whose capability was spent seals as + // `answered == started, outstanding: []`, which is what a healthy turn + // seals as. ...(terminal.mcpCalls.refusals === undefined ? {} : { refusals: Object.fromEntries(ENGINE_BROKER_MCP_REFUSAL_REASONS.map((reason) => [reason, terminal.mcpCalls!.refusals![reason]])) }), From 36ce9f06246f32314bb4e7daa51b3897e734f36a Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 16:30:52 +0200 Subject: [PATCH 66/69] fix: keep the v2 activity closure query answerable after a control host stop --- src/contracts/runtimeContractManifest.ts | 2 +- src/runtime/AGENTS.md | 21 +++++ .../organizationRuntimeClosure.test.ts | 89 +++++++++++++++++++ src/runtime/organizationRuntimeControl.ts | 31 ++++++- src/runtime/wakeAcceptanceTypes.ts | 6 ++ 5 files changed, 144 insertions(+), 5 deletions(-) create mode 100644 src/runtime/organizationRuntimeClosure.test.ts diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 95b0dfc..687b104 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -185,5 +185,5 @@ export const RUNTIME_CONTRACT_MANIFEST = { ] }, healthResponseSchema: { type: "object", additionalProperties: false, required: ["version", "state", "agents"], properties: { version: { const: "noopolis.daimon.organization-runtime-health.v1" }, state: { enum: ["starting", "running", "stopping", "stopped"] }, agents: { type: "array", maxItems: ORGANIZATION_RUNTIME_MAX_AGENTS, items: { type: "object", additionalProperties: false, required: ["agentId", "state"], properties: { agentId: text, state: { enum: ["starting", "running", "stopping", "stopped", "idle", "failed"] } } } } } }, activityResponseSchema: { type: "object", additionalProperties: false, required: ["version", "items"], properties: { version: { const: "noopolis.daimon.organization-runtime-activity.v1" }, items: { type: "array", maxItems: 100, items: activityItem }, nextCursor: { type: "string", minLength: 1, maxLength: 16, pattern: "^(0|[1-9][0-9]{0,15})$" } } }, - activityV2ResponseSchema: { type: "object", additionalProperties: false, required: ["version", "items"], properties: { version: { const: ORGANIZATION_RUNTIME_ACTIVITY_V2_VERSION }, executions: { type: "array", maxItems: ORGANIZATION_RUNTIME_MAX_AGENTS, items: { type: "object", additionalProperties: false, required: ["agent_id", "execution_id", "state", "delivery_ids"], properties: { agent_id: text, execution_id: { type: "string" }, state: { const: "running" }, delivery_ids: { type: "array", maxItems: 32, items: text } } } }, items: { type: "array", maxItems: 2_112, items: { type: "object", additionalProperties: false, required: ["version", "acceptance_id", "agent_id", "delivery_id", "request_digest", "state", "accepted_at", "updated_at", "active"], properties: { version: { const: "noopolis.daimon.wake-receipt-status.v2" }, acceptance_id: { type: "string" }, agent_id: text, delivery_id: text, request_digest: { type: "string" }, state: { enum: ["accepted", "running", "completed", "failed", "stopped"] }, accepted_at: timestamp, updated_at: timestamp, active: { type: "boolean" }, execution_id: { type: "string" }, deferred: { type: "boolean" }, text: { type: "string", maxLength: 16384 }, queue_position: { type: "integer", minimum: 1 }, code: { enum: ["engine_failed", "host_stopped", "host_stopping", "queue_full", "unknown_agent"] } } } } } } + activityV2ResponseSchema: { type: "object", additionalProperties: false, required: ["version", "items"], properties: { version: { const: ORGANIZATION_RUNTIME_ACTIVITY_V2_VERSION }, state: { enum: ["running", "stopped"] }, executions: { type: "array", maxItems: ORGANIZATION_RUNTIME_MAX_AGENTS, items: { type: "object", additionalProperties: false, required: ["agent_id", "execution_id", "state", "delivery_ids"], properties: { agent_id: text, execution_id: { type: "string" }, state: { const: "running" }, delivery_ids: { type: "array", maxItems: 32, items: text } } } }, items: { type: "array", maxItems: 2_112, items: { type: "object", additionalProperties: false, required: ["version", "acceptance_id", "agent_id", "delivery_id", "request_digest", "state", "accepted_at", "updated_at", "active"], properties: { version: { const: "noopolis.daimon.wake-receipt-status.v2" }, acceptance_id: { type: "string" }, agent_id: text, delivery_id: text, request_digest: { type: "string" }, state: { enum: ["accepted", "running", "completed", "failed", "stopped"] }, accepted_at: timestamp, updated_at: timestamp, active: { type: "boolean" }, execution_id: { type: "string" }, deferred: { type: "boolean" }, text: { type: "string", maxLength: 16384 }, queue_position: { type: "integer", minimum: 1 }, code: { enum: ["engine_failed", "host_stopped", "host_stopping", "queue_full", "unknown_agent"] } } } } } } } as const; diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 0b91035..f53a605 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -725,3 +725,24 @@ unmarked/deferred deliveries wait for new input without a self-wake loop. Live turn authority is `activity.executions`, independent of receipt completion; its execution id must equal the engine wake id. Budget pauses retain acceptance, and operator stop remains a hard latch. + +That authority has to outlive the host, because the caller who needs it reads it +last. `activityV2` used to answer `undefined` once `stop()` closed the acceptance +store — HTTP 503 `native_host_unavailable` through a caller's route — and the one +caller that must prove an execution closed asks *after* the runtime stopped: a +harness worker stops its host the moment a delivery closes its execution and +stays deferred awaiting external input. So a trial whose subject really ran, +spent its budget and simply did not do the work could not be told from a hung or +crashed one, and reported as an unscorable infrastructure failure. `stop()` now +seals the projection between the dispatcher's own shutdown — which awaits every +admitted turn, so `executions` is settled rather than momentary — and the store's +close, and `activityV2` serves that seal afterwards with `state: "stopped"`. A +stopped host has *more* certainty about quiescence than a live poll, not less, +because nothing can be admitted after the seal. Three things it is not: a bypass +of the control token, a fabricated idle runtime (a host that never started and +one whose seal could not be read both still answer nothing, because absence must +stay absence), and a claim about the store-backed routes beside it — +`availability` and `wakeReceipt` keep answering `undefined` after a stop, since +neither settles a closure proof. `state` is optional on the wire for the reason +every additive member here is: a projection published before the seal existed +must still parse, and its absence means "not stated", never "running". diff --git a/src/runtime/organizationRuntimeClosure.test.ts b/src/runtime/organizationRuntimeClosure.test.ts new file mode 100644 index 0000000..8ec4178 --- /dev/null +++ b/src/runtime/organizationRuntimeClosure.test.ts @@ -0,0 +1,89 @@ +import assert from "node:assert/strict"; +import { chmod, mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { ORGANIZATION_RUNTIME_VERSION, type OrganizationRuntimeHost, type OrganizationRuntimeWakeRequest } from "./organizationRuntime.js"; +import { createOrganizationRuntimeControlHostWithCoreForTest } from "./organizationRuntimeControl.js"; +import { ACTIVITY_V2_VERSION } from "./wakeAcceptanceTypes.js"; + +const token = "control-secret"; +const config = { + version: ORGANIZATION_RUNTIME_VERSION, + host: { bindHost: "127.0.0.1", port: 4318, controlTokenEnv: "DAIMON_CONTROL_CLOSURE_TOKEN" }, + agents: [{ id: "alpha", name: "Alpha", instructions: "Act.", workspacePath: "/runtime/workspace", runtimeHomePath: "/runtime/home", engine: { kind: "codex" as const } }] +}; +const storeOptions = { processIdentity: async () => ({ pid: 1, process_start: "test-start", boot_id: "test-boot", pid_namespace_dev: 1, pid_namespace_ino: 1 }), ownerLiveness: async () => true }; +const delivery = (deliveryId = "delivery-1") => ({ token, agent_id: "alpha", delivery_id: deliveryId, event: { version: "noopolis.daimon.wake.v2", kind: "manual" as const, text: "hello", occurred_at: "2026-09-18T00:00:00.000Z" } }); + +const core = { + async start(): Promise {}, + async wake(request: OrganizationRuntimeWakeRequest) { return { version: "noopolis.daimon.wake-result.v1", status: "completed", agentId: request.agentId, wakeId: request.event.id, text: "private", durationMs: 1 } as const; }, + async health() { return { version: "noopolis.daimon.organization-runtime-health.v1" as const, state: "running" as const, agents: [{ agentId: "alpha", engine: "codex" as const, state: "idle" as const }] }; }, + async activity() { return { version: "noopolis.daimon.organization-runtime-activity.v1" as const, items: [] }; }, + async stop() { return { version: "noopolis.daimon.organization-runtime-stop.v1" as const, state: "stopped" as const }; } +} as unknown as OrganizationRuntimeHost; + +/** + * A caller proving that a native execution closed has exactly one authority to + * read — the v2 activity projection — and the runtime that owns it is stopped by + * the time the proof is taken: the worker stops its own host as soon as a delivery + * closes its execution and stays deferred. Before this, `activityV2` answered + * `undefined` there (HTTP 503 `native_host_unavailable` through the caller's + * worker route), so a trial whose subject really ran, spent its budget and simply + * did not do the work reported as an unscorable infrastructure failure. + */ +test("a stopped control host still answers the closure query it alone can settle", async () => { + const root = await privateRoot(); + const control = createOrganizationRuntimeControlHostWithCoreForTest(config, core, { acceptanceStorePath: root, controlToken: token, storeOptions }); + try { + // Nothing has started: there is no projection to seal and none is invented. + assert.equal(await control.activityV2(token), undefined); + await control.start(); + const accepted = await control.accept(delivery()); + assert.equal(accepted.state, "accepted"); + const live = await control.activityV2(token); + assert.equal(live?.state, "running"); + assert.equal(live?.items.length, 1); + + assert.equal((await control.stop()).state, "stopped"); + const sealed = await control.activityV2(token); + assert.equal(sealed?.version, ACTIVITY_V2_VERSION); + // The same projection, said to be final: nothing can be admitted after it, so + // an empty execution list is a stronger quiescence statement than a live poll. + assert.equal(sealed?.state, "stopped"); + assert.deepEqual(sealed?.executions, []); + assert.equal(sealed?.items.length, 1); + assert.equal(sealed?.items[0]?.delivery_id, "delivery-1"); + assert.equal(sealed?.items[0]?.active, false); + // Repeating the query repeats the seal rather than draining it. + assert.deepEqual(await control.activityV2(token), sealed); + // The seal is not a bypass of authentication, and the store-backed routes that + // have no post-stop answer still report absence instead of an empty runtime. + assert.equal(await control.activityV2("wrong-token"), undefined); + assert.equal(await control.availability(token), undefined); + assert.equal(await control.wakeReceipt(token, accepted.state === "accepted" ? accepted.acceptance_id : ""), undefined); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +/** A host that was never started cannot attest anything, and a second stop keeps the seal. */ +test("an unstarted host seals nothing and a repeated stop does not erase the seal", async () => { + const root = await privateRoot(); + const unstarted = createOrganizationRuntimeControlHostWithCoreForTest(config, core, { acceptanceStorePath: root, controlToken: token, storeOptions }); + try { + assert.equal((await unstarted.stop()).state, "stopped"); + assert.equal(await unstarted.activityV2(token), undefined); + + const control = createOrganizationRuntimeControlHostWithCoreForTest(config, core, { acceptanceStorePath: root, controlToken: token, storeOptions }); + await control.start(); + await control.accept(delivery("delivery-2")); + await control.stop(); + const sealed = await control.activityV2(token); + assert.equal(sealed?.state, "stopped"); + await control.stop(); + assert.deepEqual(await control.activityV2(token), sealed); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +async function privateRoot(): Promise { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-closure-")); await chmod(root, 0o700); return root; } diff --git a/src/runtime/organizationRuntimeControl.ts b/src/runtime/organizationRuntimeControl.ts index e4d9c5c..386b6c5 100644 --- a/src/runtime/organizationRuntimeControl.ts +++ b/src/runtime/organizationRuntimeControl.ts @@ -47,6 +47,19 @@ function createControl(config: OrganizationRuntimeConfig, host: OrganizationRunt let fusePoll: ReturnType | undefined; let started = false; let stopping = false; + let sealedActivity: OrganizationRuntimeActivityV2 | undefined; + + /** + * One projection, read the same way live and at shutdown. `active` is decided by + * the dispatcher's own execution authority rather than the record's flag alone, + * so a stopped host — whose dispatcher has already awaited every in-flight turn + * — reports exactly the executions that were still admitted when it stopped. + */ + const projectActivity = async (current: WakeAcceptanceStore, state: "running" | "stopped"): Promise => { + const executions = dispatcher?.activeExecutions() ?? []; + const items = (await current.activity()).map((item) => ({ ...item, active: item.active && executions.some((execution) => execution.agent_id === item.agent_id && execution.delivery_ids.includes(item.delivery_id)) })); + return { version: ACTIVITY_V2_VERSION, state, items, executions }; + }; const hardReason = (): BlockReason | undefined => { if (!started || stopping) return stopping ? "host_stopping" : "host_stopped"; @@ -132,10 +145,15 @@ function createControl(config: OrganizationRuntimeConfig, host: OrganizationRunt return await store.status(acceptanceId); }, async activityV2(token) { - if (!tokensEqual(expectedToken, token) || store === undefined) return undefined; - const executions = dispatcher?.activeExecutions() ?? []; - const items = (await store.activity()).map((item) => ({ ...item, active: item.active && executions.some((execution) => execution.agent_id === item.agent_id && execution.delivery_ids.includes(item.delivery_id)) })); - return { version: ACTIVITY_V2_VERSION, items, executions }; + if (!tokensEqual(expectedToken, token)) return undefined; + // A stopped host is not an unanswerable one. Its sealed projection is a + // *stronger* statement about quiescence than a live poll, because nothing + // can be admitted after it, and a caller proving that an execution closed + // has no other authority to read. A host that never started, or one whose + // seal could not be taken, still answers nothing: absence stays absence + // rather than becoming a fabricated idle runtime. + if (store === undefined) return sealedActivity; + return await projectActivity(store, "running"); }, async availability(token) { if (!tokensEqual(expectedToken, token) || !store || !fuse) return undefined; @@ -161,6 +179,11 @@ function createControl(config: OrganizationRuntimeConfig, host: OrganizationRunt await Promise.allSettled(persistence); const result = await host.stop(); await dispatcher?.stop(); + // The last moment the store can be read, and the only one at which the + // dispatcher has finished every admitted turn. Seal the projection here so + // the closure query keeps an accurate answer once the store is closed; a + // fault leaves it absent instead of inventing one. + if (store) { try { sealedActivity = await projectActivity(store, "stopped"); } catch { /* an unreadable final state stays absent */ } } await store?.close(); await fuse?.close(); store = undefined; fuse = undefined; schedules = undefined; started = false; return result; diff --git a/src/runtime/wakeAcceptanceTypes.ts b/src/runtime/wakeAcceptanceTypes.ts index 1bc8e40..e21de59 100644 --- a/src/runtime/wakeAcceptanceTypes.ts +++ b/src/runtime/wakeAcceptanceTypes.ts @@ -71,6 +71,12 @@ export type OrganizationRuntimeActivityV2Item = OrganizationRuntimeWakeReceiptSt }>; export type OrganizationRuntimeActivityV2 = Readonly<{ version: typeof ACTIVITY_V2_VERSION; + /** + * Whether this projection was read from a live runtime or sealed as the host + * stopped. Optional on the wire because a projection published before the seal + * existed must still parse; its absence means "not stated", never "running". + */ + state?: "running" | "stopped"; items: readonly OrganizationRuntimeActivityV2Item[]; executions?: readonly Readonly<{ agent_id: string; execution_id: string; state: "running"; delivery_ids: readonly string[] }>[]; }>; From 194580f2794db886909d9cdc9a9d2bff82b31bf1 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 19:15:51 +0200 Subject: [PATCH 67/69] fix: record the wake outcome that reclaimed an undisposed delivery --- src/contracts/runtimeContractManifest.ts | 2 +- src/runtime/AGENTS.md | 20 +++++ src/runtime/attentionDispatcher.ts | 11 ++- .../organizationRuntimeClosure.test.ts | 85 +++++++++++++++++++ src/runtime/wakeAcceptanceRecord.ts | 8 +- src/runtime/wakeAcceptanceStore.ts | 8 +- src/runtime/wakeAcceptanceTypes.ts | 13 ++- 7 files changed, 140 insertions(+), 7 deletions(-) diff --git a/src/contracts/runtimeContractManifest.ts b/src/contracts/runtimeContractManifest.ts index 687b104..7bed4e5 100644 --- a/src/contracts/runtimeContractManifest.ts +++ b/src/contracts/runtimeContractManifest.ts @@ -185,5 +185,5 @@ export const RUNTIME_CONTRACT_MANIFEST = { ] }, healthResponseSchema: { type: "object", additionalProperties: false, required: ["version", "state", "agents"], properties: { version: { const: "noopolis.daimon.organization-runtime-health.v1" }, state: { enum: ["starting", "running", "stopping", "stopped"] }, agents: { type: "array", maxItems: ORGANIZATION_RUNTIME_MAX_AGENTS, items: { type: "object", additionalProperties: false, required: ["agentId", "state"], properties: { agentId: text, state: { enum: ["starting", "running", "stopping", "stopped", "idle", "failed"] } } } } } }, activityResponseSchema: { type: "object", additionalProperties: false, required: ["version", "items"], properties: { version: { const: "noopolis.daimon.organization-runtime-activity.v1" }, items: { type: "array", maxItems: 100, items: activityItem }, nextCursor: { type: "string", minLength: 1, maxLength: 16, pattern: "^(0|[1-9][0-9]{0,15})$" } } }, - activityV2ResponseSchema: { type: "object", additionalProperties: false, required: ["version", "items"], properties: { version: { const: ORGANIZATION_RUNTIME_ACTIVITY_V2_VERSION }, state: { enum: ["running", "stopped"] }, executions: { type: "array", maxItems: ORGANIZATION_RUNTIME_MAX_AGENTS, items: { type: "object", additionalProperties: false, required: ["agent_id", "execution_id", "state", "delivery_ids"], properties: { agent_id: text, execution_id: { type: "string" }, state: { const: "running" }, delivery_ids: { type: "array", maxItems: 32, items: text } } } }, items: { type: "array", maxItems: 2_112, items: { type: "object", additionalProperties: false, required: ["version", "acceptance_id", "agent_id", "delivery_id", "request_digest", "state", "accepted_at", "updated_at", "active"], properties: { version: { const: "noopolis.daimon.wake-receipt-status.v2" }, acceptance_id: { type: "string" }, agent_id: text, delivery_id: text, request_digest: { type: "string" }, state: { enum: ["accepted", "running", "completed", "failed", "stopped"] }, accepted_at: timestamp, updated_at: timestamp, active: { type: "boolean" }, execution_id: { type: "string" }, deferred: { type: "boolean" }, text: { type: "string", maxLength: 16384 }, queue_position: { type: "integer", minimum: 1 }, code: { enum: ["engine_failed", "host_stopped", "host_stopping", "queue_full", "unknown_agent"] } } } } } } + activityV2ResponseSchema: { type: "object", additionalProperties: false, required: ["version", "items"], properties: { version: { const: ORGANIZATION_RUNTIME_ACTIVITY_V2_VERSION }, state: { enum: ["running", "stopped"] }, executions: { type: "array", maxItems: ORGANIZATION_RUNTIME_MAX_AGENTS, items: { type: "object", additionalProperties: false, required: ["agent_id", "execution_id", "state", "delivery_ids"], properties: { agent_id: text, execution_id: { type: "string" }, state: { const: "running" }, delivery_ids: { type: "array", maxItems: 32, items: text } } } }, items: { type: "array", maxItems: 2_112, items: { type: "object", additionalProperties: false, required: ["version", "acceptance_id", "agent_id", "delivery_id", "request_digest", "state", "accepted_at", "updated_at", "active"], properties: { version: { const: "noopolis.daimon.wake-receipt-status.v2" }, acceptance_id: { type: "string" }, agent_id: text, delivery_id: text, request_digest: { type: "string" }, state: { enum: ["accepted", "running", "completed", "failed", "stopped"] }, accepted_at: timestamp, updated_at: timestamp, active: { type: "boolean" }, execution_id: { type: "string" }, deferred: { type: "boolean" }, text: { type: "string", maxLength: 16384 }, queue_position: { type: "integer", minimum: 1 }, code: { enum: ["engine_failed", "host_stopped", "host_stopping", "queued_wake_stopped", "active_wake_aborted", "queue_full", "unknown_agent"] } } } } } } } as const; diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index f53a605..6c556c2 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -746,3 +746,23 @@ stay absence), and a claim about the store-backed routes beside it — neither settles a closure proof. `state` is optional on the wire for the reason every additive member here is: a projection published before the seal existed must still parse, and its absence means "not stated", never "running". + +A delivery returned to the inbox for restart now records the outcome that returned +it. `attentionDispatcher` reclaims an undisposed delivery to `accepted` on two +conditions — the dispatcher halting, and a wake that came back `stopped` — and it +was the one `transitionClaimed` caller that passed no code, so the receipt an +evaluator reads was identical for both; a live trial closed its execution, spent +real money, and reported an `accepted` delivery with no marker and no reason. +`WakeReceiptCode` therefore carries `queued_wake_stopped` and `active_wake_aborted` +beside the existing five, because those are the two shapes a shutdown really gives a +wake (`organizationRuntimeHost.ts` settles a queued job with the first and the +in-flight one with the second) and neither had an honest name. The wake's own code +is recorded exactly; a dispatcher merely halted has no wake outcome to name and the +record stays **silent**, because a plausible name for an undetermined cause gets +acted on and a missing one does not. Two consequences, both load bearing: `accepted` +is now the one non-terminal state a record may carry a code in, since it is the only +one reached *from* an ended execution (`running` and `completed` still refuse one), +and `transitionClaimed` no longer carries a code across a transition — it describes +the transition that produced the current state, and a reclaimed delivery is claimed +again later. Widening the enum rotates the contract manifest digest, so Spawnfile +must re-vendor `contract-manifest.json`/`.sha256` and its pinned constant. diff --git a/src/runtime/attentionDispatcher.ts b/src/runtime/attentionDispatcher.ts index ad204fc..ba1334c 100644 --- a/src/runtime/attentionDispatcher.ts +++ b/src/runtime/attentionDispatcher.ts @@ -122,7 +122,16 @@ export class AttentionDispatcher { const executionError = result.status === "rejected" ? `wake rejected: ${result.code}` : result.status === "failed" ? `engine_failed: ${result.detail ?? "engine execution failed"}` : null; for (const item of claimed.filter((value) => !value.done)) { - if (this.stopping || result.status === "stopped") await store.transitionClaimed(item.record.acceptance_id, item.claim, "accepted"); + // Returned to the inbox for restart, and now recording WHY. This was the one + // caller that never passed a code, so an evaluator reading the receipt could + // not tell a shutdown from a wake the host refused, and a live trial cost + // several investigations to the same undifferentiated record. The wake's own + // stopped code is exact; a dispatcher merely halted has no code to give, and + // it stays absent rather than borrowing a plausible one. + if (this.stopping || result.status === "stopped") { + await store.transitionClaimed(item.record.acceptance_id, item.claim, "accepted", + result.status === "stopped" ? result.code : undefined); + } else if (agent.attention !== undefined) { // Successful reading is not completion. A failed execution also keeps // unfinished deliveries, and its execution id for idempotent retry. diff --git a/src/runtime/organizationRuntimeClosure.test.ts b/src/runtime/organizationRuntimeClosure.test.ts index 8ec4178..7d1f7ed 100644 --- a/src/runtime/organizationRuntimeClosure.test.ts +++ b/src/runtime/organizationRuntimeClosure.test.ts @@ -87,3 +87,88 @@ test("an unstarted host seals nothing and a repeated stop does not erase the sea }); async function privateRoot(): Promise { const root = await mkdtemp(path.join(os.tmpdir(), "daimon-closure-")); await chmod(root, 0o700); return root; } + +/** + * A delivery returned to the inbox for restart must say which outcome returned it. + * + * `attentionDispatcher` reclaims an undisposed delivery to `accepted` on two + * conditions — the dispatcher halting, and a wake that came back `stopped` — and it + * recorded neither, so the receipt an evaluator reads was identical for both. A + * live trial closed its execution, spent real money and reported an `accepted` + * delivery with no marker and no reason, and four investigations went into telling + * those two apart from the outside. The wake's own code is exact and is now kept; + * a halt has no code of its own and stays absent, because a plausible name for an + * undetermined cause gets acted on and a missing one does not. + */ +test("a delivery reclaimed for restart records the stopped wake's own code", async () => { + const root = await privateRoot(); + const attention = { version: ORGANIZATION_RUNTIME_VERSION, host: config.host, + agents: [{ ...config.agents[0]!, attention: { maxBatchMessages: 4, maxBatchBytes: 4096, maxExecutions: 8, maxTokens: 100_000 } }] }; + let stopWake = false; + const stopping = { + ...core, + async wake(request: OrganizationRuntimeWakeRequest) { + if (!stopWake) return { version: "noopolis.daimon.wake-result.v1", status: "completed", agentId: request.agentId, wakeId: request.event.id, text: "private", durationMs: 1 } as const; + // Exactly what organizationRuntimeHost settles an in-flight wake with at shutdown. + return { version: "noopolis.daimon.wake-result.v1", status: "stopped", agentId: request.agentId, wakeId: request.event.id, code: "active_wake_aborted" } as const; + } + } as unknown as OrganizationRuntimeHost; + const control = createOrganizationRuntimeControlHostWithCoreForTest(attention, stopping, { acceptanceStorePath: root, controlToken: token, storeOptions }); + try { + await control.start(); + stopWake = true; + const accepted = await control.accept(delivery("restart-delivery")); + assert.equal(accepted.state, "accepted"); + await waitFor(async () => (await control.activityV2(token))?.items.some((item) => item.state === "accepted" && item.code !== undefined) === true); + const item = (await control.activityV2(token))?.items.find((row) => row.delivery_id === "restart-delivery"); + // Returned for restart, undisposed, and no longer silent about which outcome did it. + assert.equal(item?.state, "accepted"); + // Exactly the live shape: the running transition left deferred FALSE and the + // reclaim does not clear it, which is what distinguishes it from a real deferral. + assert.equal(item?.deferred, false); + assert.equal(item?.code, "active_wake_aborted"); + } finally { await control.stop().catch(() => undefined); await rm(root, { recursive: true, force: true }); } +}); + +async function waitFor(predicate: () => Promise, timeoutMs = 5_000): Promise { + const deadline = Date.now() + timeoutMs; + do { if (await predicate()) return; await new Promise((resolve) => setTimeout(resolve, 10)); } while (Date.now() < deadline); + throw new Error("timed out waiting for the reclaimed delivery"); +} + +/** + * The other half, and the one that keeps the field trustworthy. A dispatcher halted + * mid-turn reclaims the same way, and has no wake outcome to name: the record must + * stay silent rather than borrow `host_stopping`, which reads as an account of the + * cause and would be acted on as one. Absence is the honest answer here. + */ +test("a reclaim with no wake outcome of its own records no code at all", async () => { + const root = await privateRoot(); + const attention = { version: ORGANIZATION_RUNTIME_VERSION, host: config.host, + agents: [{ ...config.agents[0]!, attention: { maxBatchMessages: 4, maxBatchBytes: 4096, maxExecutions: 8, maxTokens: 100_000 } }] }; + let release!: () => void; + const held = new Promise((resolve) => { release = resolve; }); + let arrived!: () => void; + const waking = new Promise((resolve) => { arrived = resolve; }); + const blocking = { ...core, + async wake(request: OrganizationRuntimeWakeRequest) { + arrived(); + await held; + return { version: "noopolis.daimon.wake-result.v1", status: "completed", agentId: request.agentId, wakeId: request.event.id, text: "private", durationMs: 1 } as const; + } + } as unknown as OrganizationRuntimeHost; + const control = createOrganizationRuntimeControlHostWithCoreForTest(attention, blocking, { acceptanceStorePath: root, controlToken: token, storeOptions }); + try { + await control.start(); + await control.accept(delivery("halted-delivery")); + await waking; + const stopping = control.stop(); + release(); + await stopping; + const item = (await control.activityV2(token))?.items.find((row) => row.delivery_id === "halted-delivery"); + // Reclaimed exactly as above — and saying nothing it cannot know. + assert.equal(item?.state, "accepted"); + assert.equal(item?.deferred, false); + assert.equal(item?.code, undefined); + } finally { await control.stop().catch(() => undefined); await rm(root, { recursive: true, force: true }); } +}); diff --git a/src/runtime/wakeAcceptanceRecord.ts b/src/runtime/wakeAcceptanceRecord.ts index ddf544b..7e5ae32 100644 --- a/src/runtime/wakeAcceptanceRecord.ts +++ b/src/runtime/wakeAcceptanceRecord.ts @@ -40,8 +40,12 @@ export function parseStoredWakeAcceptance(value: unknown): StoredWakeAcceptanceR if (executionError === "" || record.execution_error !== undefined && Buffer.byteLength(string(record.execution_error)) > MAX_EXECUTION_ERROR_BYTES) throw new Error("wake acceptance execution error is invalid"); const completionText = record.text === undefined ? undefined : sanitizeWakeCompletionText(string(record.text)); if (claimGeneration !== undefined && !uuid(claimGeneration)) throw new Error("wake acceptance record is invalid"); - if (code !== undefined && !(["engine_failed", "host_stopped", "host_stopping", "queue_full", "unknown_agent"] as const).includes(code)) throw new Error("wake acceptance record is invalid"); - if ((state === "accepted" || state === "running" || state === "completed") && code !== undefined) throw new Error("wake acceptance record is invalid"); + if (code !== undefined && !(["engine_failed", "host_stopped", "host_stopping", "queued_wake_stopped", "active_wake_aborted", "queue_full", "unknown_agent"] as const).includes(code)) throw new Error("wake acceptance record is invalid"); + // `accepted` is the one non-terminal state a record can be arrived at FROM an ended + // execution: the dispatcher returns an undisposed delivery there for restart, and the + // outcome that returned it is the only account of why. `running` and `completed` still + // refuse a code, where one would be nonsense rather than evidence. + if ((state === "running" || state === "completed") && code !== undefined) throw new Error("wake acceptance record is invalid"); if ((state === "failed" || state === "stopped") && code === undefined) throw new Error("wake acceptance record is invalid"); if ((state !== "completed" && state !== "failed" && completionText !== undefined) || completionText !== record.text) throw new Error("wake acceptance record is invalid"); if (string(record.request_digest) !== wakeAcceptanceDigest(parsed) || !uuid(string(record.acceptance_id))) throw new Error("wake acceptance record is invalid"); diff --git a/src/runtime/wakeAcceptanceStore.ts b/src/runtime/wakeAcceptanceStore.ts index bad2dd6..7c456ed 100644 --- a/src/runtime/wakeAcceptanceStore.ts +++ b/src/runtime/wakeAcceptanceStore.ts @@ -198,7 +198,13 @@ export class WakeAcceptanceStore { await this.afterFinalLockAssertion?.(); await this.assertTransitionLock(record, lock); const target = this.fileFor(record.agent_id, record.delivery_id); - const next: Stored = { ...record, state, updated_at: new Date().toISOString(), claim_generation: claim.generation, ...(code === undefined ? {} : { code }), ...(completedText === undefined ? {} : { text: sanitizeWakeCompletionText(completedText) }), ...(attention === undefined ? {} : { execution_id: attention.clear_execution ? undefined : attention.execution_id ?? record.execution_id, deferred: attention.deferred ?? record.deferred, execution_error: attention.execution_error === null ? undefined : attention.execution_error === undefined ? record.execution_error : sanitizeExecutionError(attention.execution_error) || undefined }) }; + // The code explains the transition that produced the CURRENT state, so a + // transition that names none clears it. While codes existed only on terminal + // records this could not matter — a terminal record returns above and is never + // rewritten — but a delivery reclaimed to `accepted` with its wake's outcome is + // claimed again later, and a carried-over code would describe the wrong state. + const { code: _replaced, ...carried } = record; + const next: Stored = { ...carried, state, updated_at: new Date().toISOString(), claim_generation: claim.generation, ...(code === undefined ? {} : { code }), ...(completedText === undefined ? {} : { text: sanitizeWakeCompletionText(completedText) }), ...(attention === undefined ? {} : { execution_id: attention.clear_execution ? undefined : attention.execution_id ?? record.execution_id, deferred: attention.deferred ?? record.deferred, execution_error: attention.execution_error === null ? undefined : attention.execution_error === undefined ? record.execution_error : sanitizeExecutionError(attention.execution_error) || undefined }) }; await this.replace(target, next); if (claim.acceptance_ids === undefined && (isTerminal(state) || state === "accepted")) { const currentClaim = await this.readClaimOptional(this.claimFor(record)); diff --git a/src/runtime/wakeAcceptanceTypes.ts b/src/runtime/wakeAcceptanceTypes.ts index e21de59..8c6ad3a 100644 --- a/src/runtime/wakeAcceptanceTypes.ts +++ b/src/runtime/wakeAcceptanceTypes.ts @@ -25,12 +25,21 @@ export const WAKE_ACCEPTANCE_REQUEST_SCHEMA = { export const WAKE_RECEIPT_STATUS_SCHEMA = { $schema: "https://json-schema.org/draft/2020-12/schema", $id: WAKE_RECEIPT_STATUS_VERSION, type: "object", additionalProperties: false, required: ["version", "acceptance_id", "agent_id", "delivery_id", "request_digest", "state", "accepted_at", "updated_at"], properties: { - version: { const: WAKE_RECEIPT_STATUS_VERSION }, acceptance_id: { type: "string", pattern: "^[0-9a-f-]{36}$" }, agent_id: { type: "string" }, delivery_id: { type: "string" }, request_digest: { type: "string", pattern: "^[a-f0-9]{64}$" }, state: { enum: ["accepted", "running", "completed", "failed", "stopped"] }, accepted_at: { type: "string" }, updated_at: { type: "string" }, execution_id: { type: "string", pattern: "^[0-9a-f-]{36}$" }, deferred: { type: "boolean" }, code: { enum: ["engine_failed", "host_stopped", "host_stopping", "queue_full", "unknown_agent"] }, text: { type: "string", maxLength: MAX_WAKE_COMPLETION_TEXT_BYTES } + version: { const: WAKE_RECEIPT_STATUS_VERSION }, acceptance_id: { type: "string", pattern: "^[0-9a-f-]{36}$" }, agent_id: { type: "string" }, delivery_id: { type: "string" }, request_digest: { type: "string", pattern: "^[a-f0-9]{64}$" }, state: { enum: ["accepted", "running", "completed", "failed", "stopped"] }, accepted_at: { type: "string" }, updated_at: { type: "string" }, execution_id: { type: "string", pattern: "^[0-9a-f-]{36}$" }, deferred: { type: "boolean" }, code: { enum: ["engine_failed", "host_stopped", "host_stopping", "queued_wake_stopped", "active_wake_aborted", "queue_full", "unknown_agent"] }, text: { type: "string", maxLength: MAX_WAKE_COMPLETION_TEXT_BYTES } } } as const; export type WakeReceiptState = "accepted" | "running" | "completed" | "failed" | "stopped"; -export type WakeReceiptCode = "engine_failed" | "host_stopped" | "host_stopping" | "queue_full" | "unknown_agent"; +/** + * Why a receipt reached its state, and every member is a state the runtime really + * produces: `queued_wake_stopped` and `active_wake_aborted` are the two shapes a + * shutdown gives a wake (`organizationRuntimeHost.ts` settles a queued job with the + * first and the in-flight one with the second), and a delivery reclaimed for restart + * used to record neither, because the only caller that could name them passed no + * code at all. A code that cannot be determined stays ABSENT: a plausible name for + * an undetermined cause is worse than no name, because it is acted on. + */ +export type WakeReceiptCode = "engine_failed" | "host_stopped" | "host_stopping" | "queued_wake_stopped" | "active_wake_aborted" | "queue_full" | "unknown_agent"; export type OrganizationRuntimeWakeAcceptanceRequest = Readonly<{ token: string | undefined; agent_id: string; From 8e0cb1ac667ed281a4143f058037d6209652d715 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 20:25:29 +0200 Subject: [PATCH 68/69] fix: decide a reclaimed delivery on the wake outcome alone --- src/runtime/AGENTS.md | 49 ++++++++++++------- src/runtime/attentionDispatcher.ts | 19 +++---- .../organizationRuntimeClosure.test.ts | 27 +++++++--- 3 files changed, 60 insertions(+), 35 deletions(-) diff --git a/src/runtime/AGENTS.md b/src/runtime/AGENTS.md index 6c556c2..08250d9 100644 --- a/src/runtime/AGENTS.md +++ b/src/runtime/AGENTS.md @@ -747,22 +747,33 @@ neither settles a closure proof. `state` is optional on the wire for the reason every additive member here is: a projection published before the seal existed must still parse, and its absence means "not stated", never "running". -A delivery returned to the inbox for restart now records the outcome that returned -it. `attentionDispatcher` reclaims an undisposed delivery to `accepted` on two -conditions — the dispatcher halting, and a wake that came back `stopped` — and it -was the one `transitionClaimed` caller that passed no code, so the receipt an -evaluator reads was identical for both; a live trial closed its execution, spent -real money, and reported an `accepted` delivery with no marker and no reason. -`WakeReceiptCode` therefore carries `queued_wake_stopped` and `active_wake_aborted` -beside the existing five, because those are the two shapes a shutdown really gives a -wake (`organizationRuntimeHost.ts` settles a queued job with the first and the -in-flight one with the second) and neither had an honest name. The wake's own code -is recorded exactly; a dispatcher merely halted has no wake outcome to name and the -record stays **silent**, because a plausible name for an undetermined cause gets -acted on and a missing one does not. Two consequences, both load bearing: `accepted` -is now the one non-terminal state a record may carry a code in, since it is the only -one reached *from* an ended execution (`running` and `completed` still refuse one), -and `transitionClaimed` no longer carries a code across a transition — it describes -the transition that produced the current state, and a reclaimed delivery is claimed -again later. Widening the enum rotates the contract manifest digest, so Spawnfile -must re-vendor `contract-manifest.json`/`.sha256` and its pinned constant. +A delivery returned to the inbox for restart records the outcome that returned it, +and **only a wake outcome can return one**. `attentionDispatcher` reclaims an +undisposed delivery to `accepted` on exactly one condition — a wake result of +`stopped`, which is also the shape an aborted in-flight wake arrives in +(`organizationRuntimeHost.ts` settles a queued job `queued_wake_stopped` and the +in-flight one `active_wake_aborted`). The dispatcher's own `stopping` latch used to +share that condition, and it is a HOST-LIFECYCLE fact, not a wake outcome: a wake +that *completed* had its evidence discarded because the dispatcher happened to be +halting, and the delivery was recorded `accepted, deferred: false, execution id +retained, no code` — byte-identical to "never ran" and to "ran but forgotten". +Production tolerated that because a restart re-delivers and the agent redoes the +work; a one-shot isolated trial has no restart, so the information was simply lost +and a subject that ran and made a choice reported as an infrastructure failure. It +is the wrong record for production too: an agent that read a delivery and declined +to dispose of it is **deferred**, whichever way the host is heading, and a restart +must not re-deliver it as fresh work. So a completed or failed wake takes the +deferred path regardless of dispatcher state, and `stopping` guards only the +pre-wake path, which is where it belongs — it must never be restored to the +post-wake decision. `WakeReceiptCode` carries `queued_wake_stopped` and +`active_wake_aborted` beside the existing five, because those are the two shapes a +shutdown really gives a wake and neither had an honest name. The wake's own code is +recorded exactly; nothing else names a reclaim, because a plausible name for an +undetermined cause gets acted on and a missing one does not. Two consequences, both +load bearing: `accepted` is the one non-terminal state a record may carry a code in, +since it is the only one reached *from* an ended execution (`running` and +`completed` still refuse one), and `transitionClaimed` no longer carries a code +across a transition — it describes the transition that produced the current state, +and a reclaimed delivery is claimed again later. Widening the enum rotates the +contract manifest digest, so Spawnfile must re-vendor +`contract-manifest.json`/`.sha256` and its pinned constant. diff --git a/src/runtime/attentionDispatcher.ts b/src/runtime/attentionDispatcher.ts index ba1334c..a63c9eb 100644 --- a/src/runtime/attentionDispatcher.ts +++ b/src/runtime/attentionDispatcher.ts @@ -122,15 +122,16 @@ export class AttentionDispatcher { const executionError = result.status === "rejected" ? `wake rejected: ${result.code}` : result.status === "failed" ? `engine_failed: ${result.detail ?? "engine execution failed"}` : null; for (const item of claimed.filter((value) => !value.done)) { - // Returned to the inbox for restart, and now recording WHY. This was the one - // caller that never passed a code, so an evaluator reading the receipt could - // not tell a shutdown from a wake the host refused, and a live trial cost - // several investigations to the same undifferentiated record. The wake's own - // stopped code is exact; a dispatcher merely halted has no code to give, and - // it stays absent rather than borrowing a plausible one. - if (this.stopping || result.status === "stopped") { - await store.transitionClaimed(item.record.acceptance_id, item.claim, "accepted", - result.status === "stopped" ? result.code : undefined); + // Returned to the inbox for restart, and recording WHY. Only a WAKE OUTCOME + // decides this: `stopped` — which an aborted in-flight wake also carries, with + // its own code — is the runtime reclaiming work nobody read. The dispatcher's + // own halt is not an outcome and must not stand in for one; it guards the + // pre-wake path, where it belongs. Keying on it here discarded a completed + // wake's evidence because the host happened to be halting, leaving a record + // byte-identical to "never ran". Production tolerated it because a restart + // re-delivers; a one-shot isolated trial has no restart and simply lost it. + if (result.status === "stopped") { + await store.transitionClaimed(item.record.acceptance_id, item.claim, "accepted", result.code); } else if (agent.attention !== undefined) { // Successful reading is not completion. A failed execution also keeps diff --git a/src/runtime/organizationRuntimeClosure.test.ts b/src/runtime/organizationRuntimeClosure.test.ts index 7d1f7ed..67b1a48 100644 --- a/src/runtime/organizationRuntimeClosure.test.ts +++ b/src/runtime/organizationRuntimeClosure.test.ts @@ -137,12 +137,19 @@ async function waitFor(predicate: () => Promise, timeoutMs = 5_000): Pr } /** - * The other half, and the one that keeps the field trustworthy. A dispatcher halted - * mid-turn reclaims the same way, and has no wake outcome to name: the record must - * stay silent rather than borrow `host_stopping`, which reads as an account of the - * cause and would be acted on as one. Absence is the honest answer here. + * The other half, and the one the reclaim path kept getting wrong. A wake that + * COMPLETED is a wake outcome; the dispatcher happening to be halting when it + * lands is not. Keying the reclaim on the host's own `stopping` latch discarded + * that outcome and wrote `accepted, deferred: false, execution retained, no code` + * — a record byte-identical to "never ran" and to "ran but forgotten". Production + * survived it because a restart re-delivers and the agent redoes the work; a + * one-shot isolated trial has no restart, so the evidence was simply lost and a + * subject that really ran and made a choice reported as infrastructure failure. + * An agent that read a delivery and declined to dispose of it is DEFERRED, + * whichever way the host is heading, and a restart must not re-deliver it as + * fresh work. */ -test("a reclaim with no wake outcome of its own records no code at all", async () => { +test("a completed wake under a halting dispatcher is deferred, not reclaimed for restart", async () => { const root = await privateRoot(); const attention = { version: ORGANIZATION_RUNTIME_VERSION, host: config.host, agents: [{ ...config.agents[0]!, attention: { maxBatchMessages: 4, maxBatchBytes: 4096, maxExecutions: 8, maxTokens: 100_000 } }] }; @@ -162,13 +169,19 @@ test("a reclaim with no wake outcome of its own records no code at all", async ( await control.start(); await control.accept(delivery("halted-delivery")); await waking; + // The halt lands while the wake is in flight; the wake then completes anyway. const stopping = control.stop(); release(); await stopping; const item = (await control.activityV2(token))?.items.find((row) => row.delivery_id === "halted-delivery"); - // Reclaimed exactly as above — and saying nothing it cannot know. assert.equal(item?.state, "accepted"); - assert.equal(item?.deferred, false); + // The wake's own outcome decides the record: read, undisposed, deferred. + assert.equal(item?.deferred, true); + // A completed wake releases its execution, so a restart waits for new input + // instead of replaying the delivery as work nobody has seen. + assert.equal(item?.execution_id, undefined); + // Still silent: a completed wake is no more a named reclaim outcome than a + // halt is, and a plausible name for an undetermined cause gets acted on. assert.equal(item?.code, undefined); } finally { await control.stop().catch(() => undefined); await rm(root, { recursive: true, force: true }); } }); From cff564aef285ecd9a8f60a625be6135c50b839c7 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Fri, 18 Sep 2026 22:19:14 +0200 Subject: [PATCH 69/69] test: await the published offline-reconciliation lease instead of a 5 ms sleep --- src/runtime/wakeAcceptanceReconciliation.test.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/runtime/wakeAcceptanceReconciliation.test.ts b/src/runtime/wakeAcceptanceReconciliation.test.ts index fb2bd4e..f7ab14c 100644 --- a/src/runtime/wakeAcceptanceReconciliation.test.ts +++ b/src/runtime/wakeAcceptanceReconciliation.test.ts @@ -48,9 +48,16 @@ test("offline reconciliation blocks untrusted proof, identity mismatch, and conc const mismatch = await reconcileOfflineWakeTransition({ ...request, lock: { ...request.lock, ino: request.lock.ino + 1 } }, { storePath: root, ...testLeaseOptions, verifyDeploymentAttestation: async (context) => ({ request_digest: context.request_digest, nonce: context.nonce, exclusive_store: true, authorized_registration_digests: [] }) }); assert.equal(mismatch.state, "blocked"); let release!: () => void; + let leaseCreated!: () => void; const paused = new Promise((resolve) => { release = resolve; }); - const first = reconcileOfflineWakeTransition(request, { storePath: root, ...testLeaseOptions, verifyDeploymentAttestation: async (context) => { await paused; return { request_digest: context.request_digest, nonce: context.nonce, exclusive_store: true, authorized_registration_digests: [] }; } }); - await new Promise((resolve) => setTimeout(resolve, 5)); + // `verifyDeploymentAttestation` runs only after `acquireLease` has published the + // lease, so signalling from inside it is a real happens-after of that publication. + // A sleep is not: publishing the lease is several fsynced filesystem operations and + // takes ~5 ms even on an idle machine, so a 5 ms timer raced it and the store then + // opened against a store no one had reserved yet. + const leased = new Promise((resolve) => { leaseCreated = resolve; }); + const first = reconcileOfflineWakeTransition(request, { storePath: root, ...testLeaseOptions, verifyDeploymentAttestation: async (context) => { leaseCreated(); await paused; return { request_digest: context.request_digest, nonce: context.nonce, exclusive_store: true, authorized_registration_digests: [] }; } }); + await leased; await assert.rejects(WakeAcceptanceStore.open(root, testStoreOptions), /reserved for offline reconciliation/); const concurrent = await reconcileOfflineWakeTransition(request, { storePath: root, ...testLeaseOptions, verifyDeploymentAttestation: async (context) => ({ request_digest: context.request_digest, nonce: context.nonce, exclusive_store: true, authorized_registration_digests: [] }) }); assert.equal(concurrent.state, "blocked");